From 564c4ff5c1cd10e305a5304da64efbfef312fcbc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 11:37:44 +0000 Subject: [PATCH 001/199] Remove the zstd C library from the build The build no longer fetches, patches, compiles or links facebook/zstd. Nothing replaces it yet, so the build fails: bun_icu_decompress.cpp and ProcessBindingConstants.cpp include zstd.h, BunProcess.cpp reads BUN_VERSION_ZSTD_HASH, and bun_zstd declares zstd's symbols. --- .github/workflows/update-zstd.yml | 106 ------------------ .gitignore | 1 - CLAUDE.md | 2 +- patches/zstd/bmi2-probe-once.patch | 37 ------ .../free-workers-before-dictionaries.patch | 28 ----- scripts/build/CLAUDE.md | 10 +- scripts/build/compile.ts | 2 +- scripts/build/deps/index.ts | 3 - scripts/build/deps/zstd.ts | 96 ---------------- scripts/build/flags.ts | 2 +- scripts/build/source.ts | 13 +-- scripts/build/stream.ts | 2 +- scripts/verify-baseline-static/CLAUDE.md | 1 - .../verify-baseline-static/allowlist-x64.txt | 25 ----- 14 files changed, 15 insertions(+), 313 deletions(-) delete mode 100644 .github/workflows/update-zstd.yml delete mode 100644 patches/zstd/bmi2-probe-once.patch delete mode 100644 patches/zstd/free-workers-before-dictionaries.patch delete mode 100644 scripts/build/deps/zstd.ts diff --git a/.github/workflows/update-zstd.yml b/.github/workflows/update-zstd.yml deleted file mode 100644 index c19fc9092a51..000000000000 --- a/.github/workflows/update-zstd.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: Update zstd - -permissions: - contents: read - -on: - schedule: - - cron: "0 1 * * 0" - workflow_dispatch: - -jobs: - check-update: - if: github.repository == 'oven-sh/bun' - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Check zstd version - id: check-version - run: | - set -euo pipefail - - DEP_FILE=scripts/build/deps/zstd.ts - CURRENT_VERSION=$(sed -nE 's/^const ZSTD_COMMIT = "([0-9a-f]{40})";$/\1/p' "$DEP_FILE") - - if [ -z "$CURRENT_VERSION" ]; then - echo "Error: Could not find ZSTD_COMMIT in $DEP_FILE" - exit 1 - fi - - # Validate that it looks like a git hash - if ! [[ $CURRENT_VERSION =~ ^[0-9a-f]{40}$ ]]; then - echo "Error: Invalid git hash format in $DEP_FILE" - echo "Found: $CURRENT_VERSION" - echo "Expected: 40 character hexadecimal string" - exit 1 - fi - - echo "current=$CURRENT_VERSION" >> $GITHUB_OUTPUT - - LATEST_RELEASE=$(curl -sL https://api.github.com/repos/facebook/zstd/releases/latest) - if [ -z "$LATEST_RELEASE" ]; then - echo "Error: Failed to fetch latest release from GitHub API" - exit 1 - fi - - LATEST_TAG=$(echo "$LATEST_RELEASE" | jq -r '.tag_name') - if [ -z "$LATEST_TAG" ] || [ "$LATEST_TAG" = "null" ]; then - echo "Error: Could not extract tag name from GitHub API response" - exit 1 - fi - - LATEST_TAG_SHA=$(curl -sL "https://api.github.com/repos/facebook/zstd/git/refs/tags/$LATEST_TAG" | jq -r '.object.sha') - if [ -z "$LATEST_TAG_SHA" ] || [ "$LATEST_TAG_SHA" = "null" ]; then - echo "Error: Could not fetch SHA for tag $LATEST_TAG" - exit 1 - fi - LATEST_SHA=$(curl -sL "https://api.github.com/repos/facebook/zstd/git/tags/$LATEST_TAG_SHA" | jq -r '.object.sha') - if [ -z "$LATEST_SHA" ] || [ "$LATEST_SHA" = "null" ]; then - echo "Error: Could not fetch SHA for tag $LATEST_TAG @ $LATEST_TAG_SHA" - exit 1 - fi - - if ! [[ $LATEST_SHA =~ ^[0-9a-f]{40}$ ]]; then - echo "Error: Invalid SHA format received from GitHub" - echo "Found: $LATEST_SHA" - echo "Expected: 40 character hexadecimal string" - exit 1 - fi - - echo "latest=$LATEST_SHA" >> $GITHUB_OUTPUT - echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT - - - name: Update version if needed - if: success() && steps.check-version.outputs.current != steps.check-version.outputs.latest - env: - LATEST: ${{ steps.check-version.outputs.latest }} - run: | - set -euo pipefail - sed -i -E 's/^(const ZSTD_COMMIT = ")[0-9a-f]{40}(";)$/\1'"$LATEST"'\2/' scripts/build/deps/zstd.ts - - - name: Create Pull Request - if: success() && steps.check-version.outputs.current != steps.check-version.outputs.latest - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - add-paths: | - scripts/build/deps/zstd.ts - commit-message: "deps: update zstd to ${{ steps.check-version.outputs.tag }} (${{ steps.check-version.outputs.latest }})" - title: "deps: update zstd to ${{ steps.check-version.outputs.tag }}" - delete-branch: true - branch: deps/update-zstd - body: | - ## What does this PR do? - - Updates zstd to version ${{ steps.check-version.outputs.tag }} - - Compare: https://github.com/facebook/zstd/compare/${{ steps.check-version.outputs.current }}...${{ steps.check-version.outputs.latest }} - - Auto-updated by [this workflow](https://github.com/oven-sh/bun/actions/workflows/update-zstd.yml) diff --git a/.gitignore b/.gitignore index 4b09e2f7fd5a..0fb10c382889 100644 --- a/.gitignore +++ b/.gitignore @@ -163,7 +163,6 @@ codegen-for-zig-team.tar.gz /src/deps/WebKit /src/deps/zig /src/deps/zlib -/src/deps/zstd # Generated files diff --git a/CLAUDE.md b/CLAUDE.md index 8feccff501bc..81a19f9c2c31 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,7 +147,7 @@ The Rust side is a Cargo workspace of ~200 crates rooted at `Cargo.toml`. The ke #### Vendored Dependencies (`vendor/`) -Third-party native libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), highway (SIMD), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), picohttpparser, rust-argon2 (argon2 for `Bun.password`, Rust), tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng), zstd. Build configuration for these is in `scripts/build/deps/*.ts`. +Third-party native libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), highway (SIMD), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), picohttpparser, rust-argon2 (argon2 for `Bun.password`, Rust), tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng). Build configuration for these is in `scripts/build/deps/*.ts`. ### JavaScript Class Implementation (C++) diff --git a/patches/zstd/bmi2-probe-once.patch b/patches/zstd/bmi2-probe-once.patch deleted file mode 100644 index 603ccfe44678..000000000000 --- a/patches/zstd/bmi2-probe-once.patch +++ /dev/null @@ -1,37 +0,0 @@ -Probe BMI2 support once instead of once per context. - -ZSTD_cpuSupportsBmi2() runs on every CCtx and DCtx init when DYNAMIC_BMI2 is -on, which it is for Bun's x64 builds (they target nehalem, so __BMI2__ is not -defined). It executes CPUID twice. Under a hypervisor each CPUID is a VM exit -of about 2 us, so a create + free of a DCtx costs 3.6 us instead of 30 ns, and -a one-shot ZSTD_decompress() of 1 KB costs 3.9 us instead of 0.4 us. - -facebook/zstd#4767 removed the per-context probe for Linux kernel builds only. -Userspace builds still run it. - ---- a/lib/common/zstd_internal.h -+++ b/lib/common/zstd_internal.h -@@ -314,11 +314,21 @@ - - /** - * @returns true iff the CPU supports dynamic BMI2 dispatch. -+ * -+ * Bun: every CCtx and DCtx init calls this, and CPUID is a VM exit under a -+ * hypervisor (about 2 us each, two per call). The answer cannot change, so -+ * probe once per translation unit. 0: not probed yet, 1: no BMI2, 2: BMI2. - */ - MEM_STATIC int ZSTD_cpuSupportsBmi2(void) - { -- ZSTD_cpuid_t cpuid = ZSTD_cpuid(); -- return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); -+ static int cached = 0; -+ int state = __atomic_load_n(&cached, __ATOMIC_SEQ_CST); -+ if (state == 0) { -+ ZSTD_cpuid_t const cpuid = ZSTD_cpuid(); -+ state = 1 + (ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid)); -+ __atomic_store_n(&cached, state, __ATOMIC_SEQ_CST); -+ } -+ return state - 1; - } - - #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/patches/zstd/free-workers-before-dictionaries.patch b/patches/zstd/free-workers-before-dictionaries.patch deleted file mode 100644 index c954c98e7d61..000000000000 --- a/patches/zstd/free-workers-before-dictionaries.patch +++ /dev/null @@ -1,28 +0,0 @@ -Free the worker threads of a context before the dictionaries they read. - -ZSTD_freeCCtxContent() calls ZSTD_clearAllDicts() and then ZSTDMT_freeCCtx(). -With nbWorkers >= 1, ZSTD_compressStream2() posts a job to a worker thread and -returns. The first job of a frame reads the dictionary of the context through -a pointer that it does not own (job->cdict). A job stops only in -ZSTDMT_freeCCtx(), which joins the threads. So ZSTD_freeCCtx() of a context that -has a dictionary and a running job frees the dictionary under that job. - -ZSTD_CCtx_reset() with parameters, ZSTD_CCtx_loadDictionary(), ZSTD_CCtx_refCDict() -and ZSTD_CCtx_refPrefix() free the dictionaries too. Bun does not call them on a -context whose jobs still run. - ---- a/lib/compress/zstd_compress.c -+++ b/lib/compress/zstd_compress.c -@@ -171,10 +171,11 @@ - { - assert(cctx != NULL); - assert(cctx->staticSize == 0); -- ZSTD_clearAllDicts(cctx); - #ifdef ZSTD_MULTITHREAD - ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; - #endif -+ /* Bun: after the join above. A job that still runs reads these dictionaries. */ -+ ZSTD_clearAllDicts(cctx); - ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); - } - diff --git a/scripts/build/CLAUDE.md b/scripts/build/CLAUDE.md index 55f29955c068..45e6d92eaeb4 100644 --- a/scripts/build/CLAUDE.md +++ b/scripts/build/CLAUDE.md @@ -22,7 +22,7 @@ This directory generates `build.ninja`. The scripts **describe** the build; ninj **Deps in our graph by default; native build systems when needed.** `BuildSpec` variants: -- `direct` — list the dep's sources explicitly; each becomes a first-class `cc`/`cxx` edge in our graph and the `.o`s go straight into bun's link. The default for the C/C++ deps (zlib, zstd, boringssl, mimalloc, …). Skips a sub-process configure entirely and lets LTO see across the dep boundary. +- `direct` — list the dep's sources explicitly; each becomes a first-class `cc`/`cxx` edge in our graph and the `.o`s go straight into bun's link. The default for the C/C++ deps (zlib, boringssl, mimalloc, …). Skips a sub-process configure entirely and lets LTO see across the dep boundary. - `nested-cmake` — invoke the dep's own cmake configure + build as ninja edges. For deps whose build is too entangled to list by hand. Flags forwarded via `-DCMAKE_C_FLAGS`; cmake's own dependency tracking handles incrementality inside. - `cargo` — (rule kept for out-of-workspace cargo deps; none today — lolhtml and rust-argon2 are path dependencies compiled as units of bun's own Rust graph). - `prebuilt` — skip build entirely, download compiled `.a`/`.lib` (WebKit, nodejs-headers). @@ -49,7 +49,7 @@ rule cc depfile = $out.d deps = gcc -build obj/src/foo.c.o: cc ../../src/foo.c | deps/zstd/libzstd.a || codegen/generated.h +build obj/src/foo.c.o: cc ../../src/foo.c | deps/cares/libcares.a || codegen/generated.h cflags = -O2 -I... ``` @@ -61,9 +61,9 @@ rule dep_fetch restat = 1 pool = dep -build ../../vendor/zstd/.ref: dep_fetch | ../../scripts/build/fetch-cli.ts - name = zstd - repo = facebook/zstd +build ../../vendor/zlib/.ref: dep_fetch | ../../scripts/build/fetch-cli.ts + name = zlib + repo = zlib-ng/zlib-ng commit = abc123... ``` diff --git a/scripts/build/compile.ts b/scripts/build/compile.ts index 9125a649db71..742e94d60050 100644 --- a/scripts/build/compile.ts +++ b/scripts/build/compile.ts @@ -256,7 +256,7 @@ export function cxx(n: Ninja, cfg: Config, src: string, opts: CompileOpts): stri * * `.S` (preprocessed assembly) is also accepted — clang dispatches on the * extension and runs cpp + as. Used by deps that ship hand-tuned kernels - * (e.g. zstd's huf_decompress_amd64.S). + * (e.g. boringssl's aesni-x86_64-linux.S). */ export function cc(n: Ninja, cfg: Config, src: string, opts: Omit): string { const ext = extname(src); diff --git a/scripts/build/deps/index.ts b/scripts/build/deps/index.ts index 334931b8683f..766a89d65b11 100644 --- a/scripts/build/deps/index.ts +++ b/scripts/build/deps/index.ts @@ -32,7 +32,6 @@ import { sqlite } from "./sqlite.ts"; import { tinycc } from "./tinycc.ts"; import { webkit } from "./webkit.ts"; import { zlib } from "./zlib.ts"; -import { zstd } from "./zstd.ts"; /** * All deps in dependency-resolution + link order. @@ -47,7 +46,6 @@ export const allDeps: readonly Dependency[] = [ nodejsHeaders, zlib, - zstd, brotli, libdeflate, // Image codecs — libspng has fetchDeps:["zlib"], so after zlib. @@ -97,5 +95,4 @@ export { tinycc, webkit, zlib, - zstd, }; diff --git a/scripts/build/deps/zstd.ts b/scripts/build/deps/zstd.ts deleted file mode 100644 index 351bb62c9fc9..000000000000 --- a/scripts/build/deps/zstd.ts +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Zstandard — fast compression with a good ratio/speed tradeoff. Backs - * bun's install cache and the `zstd` Content-Encoding in fetch. - * - * DirectBuild: globbed common/compress/decompress/dictBuilder. Legacy-format - * decoders (zstd v0.5-v0.7, pre-1.0) are NOT built — Bun never reads - * pre-1.0 frames (install cache, fetch Content-Encoding, and Bun.zstd all - * use the current format). The amd64 - * Huffman kernel ships as a .S file that clang assembles directly; on other - * targets ZSTD_DISABLE_ASM falls through to the C implementation. - */ - -import type { Dependency, DirectBuild } from "../source.ts"; - -const ZSTD_COMMIT = "f8745da6ff1ad1e7bab384bd1f9d742439278e99"; - -// prettier-ignore -const SOURCES = [ - "common/debug", "common/entropy_common", "common/error_private", - "common/fse_decompress", "common/pool", "common/threading", "common/xxhash", - "common/zstd_common", - "compress/fse_compress", "compress/hist", "compress/huf_compress", - "compress/zstd_compress", "compress/zstd_compress_literals", - "compress/zstd_compress_sequences", "compress/zstd_compress_superblock", - "compress/zstd_double_fast", "compress/zstd_fast", "compress/zstd_lazy", - "compress/zstd_ldm", "compress/zstd_opt", "compress/zstd_preSplit", - "compress/zstdmt_compress", - "decompress/huf_decompress", "decompress/zstd_ddict", - "decompress/zstd_decompress", "decompress/zstd_decompress_block", - "dictBuilder/cover", "dictBuilder/divsufsort", "dictBuilder/fastcover", - "dictBuilder/zdict", -].map(s => `lib/${s}.c`); - -export const zstd: Dependency = { - name: "zstd", - versionMacro: "ZSTD_HASH", - - source: () => ({ - kind: "github-archive", - repo: "facebook/zstd", - commit: ZSTD_COMMIT, - }), - - patches: [ - // x64 targets nehalem, so zstd picks its BMI2 kernels at run time and - // probes CPUID in every CCtx/DCtx init. CPUID is a VM exit under a - // hypervisor (about 2 us each, two per init). Probe once instead. - "patches/zstd/bmi2-probe-once.patch", - // ZSTD_freeCCtx frees a context's dictionaries before joining the worker - // threads, so a job that's still running can read freed memory. - "patches/zstd/free-workers-before-dictionaries.patch", - ], - - build: cfg => { - const sources = [...SOURCES]; - const defines: Record = { - ZSTD_MULTITHREAD: true, - ZSTD_LEGACY_SUPPORT: 0, - }; - - // Upstream's if(MSVC) block sets these for the static target. - // ZSTD_HEAPMODE=0 makes the one-shot ZSTD_decompress() (used by - // src/zstd/lib.rs) stack-allocate its DCtx instead of malloc/free - // per call; the source default is 1. - if (cfg.windows) { - defines.ZSTD_HEAPMODE = 0; - defines._CRT_SECURE_NO_WARNINGS = true; - } - - // huf_decompress_amd64.S is GNU-as syntax. clang assembles it on - // posix x64; clang-cl can't, so Windows takes the C path. - if (cfg.x64 && !cfg.windows) { - sources.push("lib/decompress/huf_decompress_amd64.S"); - } else { - defines.ZSTD_DISABLE_ASM = 1; - } - - const spec: DirectBuild = { - kind: "direct", - sources, - defines, - includes: ["lib", "lib/common"], - pic: true, - // XXH_NAMESPACE must be a bare token prefix (xxhash pastes it onto - // symbol names), not a string literal — DirectBuild.defines would - // wrap it in quotes. Avoids clashes with lshpack's copy. - cflags: ["-DXXH_NAMESPACE=ZSTD_"], - }; - return spec; - }, - - provides: () => ({ - libs: [], - includes: ["lib"], - }), -}; diff --git a/scripts/build/flags.ts b/scripts/build/flags.ts index fb8e2c38448d..dd323d40cac2 100644 --- a/scripts/build/flags.ts +++ b/scripts/build/flags.ts @@ -251,7 +251,7 @@ export const globalFlags: Flag[] = [ // cmake's Release/RelWithDebInfo build types append this to // CMAKE__FLAGS_ automatically; nested-cmake deps got it // from there. Direct deps only see globalFlags, so it must be here - // too — otherwise every assert() in zstd/boringssl/mimalloc/etc. + // too — otherwise every assert() in zlib/boringssl/mimalloc/etc. // stays live in release. (bun's own NDEBUG in `defines` below is // redundant after this, but harmless.) flag: "-DNDEBUG", diff --git a/scripts/build/source.ts b/scripts/build/source.ts index f9db40929ff7..9911ec8ad249 100644 --- a/scripts/build/source.ts +++ b/scripts/build/source.ts @@ -333,7 +333,7 @@ export interface NestedCmakeBuild { libSubdir?: string; /** * Subdirectory within the SOURCE dir containing CMakeLists.txt. - * E.g. zstd's cmake files live at `build/cmake/`, not the repo root. + * Some projects keep their cmake files in e.g. `build/cmake/`, not the repo root. * Becomes the `-S` arg to cmake. Default: source root. */ sourceSubdir?: string; @@ -462,8 +462,7 @@ export type DepName = | "rust-argon2" | "sqlite" | "tinycc" - | "zlib" - | "zstd"; + | "zlib"; /** * A vendored dependency definition. Lives in scripts/build/deps/.ts. @@ -869,7 +868,7 @@ export function resolveDep( } else { // Local/in-tree: no .ref to write. Use the build system's manifest file // as the stamp — touching it triggers reconfigure/rebuild. - // cmake deps → CMakeLists.txt (in sourceSubdir if set, e.g. zstd) + // cmake deps → CMakeLists.txt (in sourceSubdir if set) // cargo deps → Cargo.toml (in manifestDir) // direct/header-only → none: the sources are on disk before ninja // starts, so the compiler depfiles see edits directly. (Stamping the @@ -958,8 +957,8 @@ export function resolveDep( // ─── Resolve include paths ─── // Includes are relative to the SOURCE dir (in-tree or vendor). Not the - // cmake subdir — e.g. zstd's headers are at vendor/zstd/lib/, not - // vendor/zstd/build/cmake/lib/. + // cmake subdir — e.g. headers are at vendor//lib/, not + // vendor//build/cmake/lib/. // // Includes CAN be absolute — for deps whose headers land in the BUILD dir // (generated during configure), the `provides` function computes absolute @@ -1153,7 +1152,7 @@ function emitNestedCmake( const hostWin = cfg.host.os === "windows"; // cmake source dir (where CMakeLists.txt lives). Usually srcDir, but - // some projects nest it (zstd: vendor/zstd/build/cmake/). + // some projects nest it (vendor//build/cmake/). const cmakeSrcDir = spec.sourceSubdir ? resolve(srcDir, spec.sourceSubdir) : srcDir; // ─── Assemble cmake configure args ─── diff --git a/scripts/build/stream.ts b/scripts/build/stream.ts index 2c3c59972e25..7a60d4e8d4e5 100644 --- a/scripts/build/stream.ts +++ b/scripts/build/stream.ts @@ -69,7 +69,7 @@ export const STREAM_FD = 3; /** * Produce a colored `[name] ` prefix. Color derived from a hash of the - * name — deterministic, so zstd is always the same shade across runs. + * name — deterministic, so zlib is always the same shade across runs. * * Palette: 12 ANSI 256-color codes from gold → green → cyan → blue → * purple. No reds/pinks (error connotation), no dark blues (illegible diff --git a/scripts/verify-baseline-static/CLAUDE.md b/scripts/verify-baseline-static/CLAUDE.md index bb911b65fd15..b1e27a88c25e 100644 --- a/scripts/verify-baseline-static/CLAUDE.md +++ b/scripts/verify-baseline-static/CLAUDE.md @@ -159,7 +159,6 @@ table, an HWCAP test. Known patterns: | simdutf | `set_best()` — CPUID first call, cached atomic ptr | `vendor/` or WebKit's bundled copy | | Highway (Bun) | `BUN_HWY_DISPATCH` → `hwy::SupportedTargets()` | `src/jsc/bindings/highway_dispatch.h` | | BoringSSL | `OPENSSL_ia32cap_P` global, set at init | `vendor/boringssl/crypto/cpu_intel.c` | -| zstd | `ZSTD_cpuid()` | `vendor/zstd/lib/common/cpu.h` | | libdeflate | `libdeflate_init_x86_cpu_features()` / `HWCAP_ASIMDDP` | `vendor/libdeflate/lib/x86/cpu_features.c` | | Rust `memchr` | `is_x86_feature_detected!()` | (via lolhtml dep) | | compiler-rt outline-atomics (aarch64) | `__aarch64_have_lse_atomics` (= `AT_HWCAP & HWCAP_ATOMICS`) | compiler-rt builtin | diff --git a/scripts/verify-baseline-static/allowlist-x64.txt b/scripts/verify-baseline-static/allowlist-x64.txt index 8cc12b70ff91..09ff362c00ea 100644 --- a/scripts/verify-baseline-static/allowlist-x64.txt +++ b/scripts/verify-baseline-static/allowlist-x64.txt @@ -897,31 +897,6 @@ CRYPTO_rdrand [RDRAND] CRYPTO_rdrand_multiple8_buf [RDRAND] -# ---------------------------------------------------------------------------- -# zstd Huffman/FSE BMI2. Gate: ZSTD_cpuid(). -# (19 symbols) -# ---------------------------------------------------------------------------- -FSE_decompress_wksp_body_bmi2 [BMI2, LZCNT] -FSE_readNCount_body_bmi2 [BMI2, LZCNT] -HUF_compress1X_usingCTable_internal_bmi2 [BMI2] -HUF_decompress1X1_usingDTable_internal_bmi2 [BMI1, BMI2, LZCNT] -HUF_decompress1X2_usingDTable_internal_bmi2 [BMI1, BMI2, LZCNT] -HUF_decompress4X1_usingDTable_internal_bmi2 [BMI1, BMI2, LZCNT] -HUF_decompress4X1_usingDTable_internal_fast [BMI1, BMI2] -HUF_decompress4X1_usingDTable_internal_fast_asm_loop [BMI2] -HUF_decompress4X1_usingDTable_internal_fast_c_loop [BMI2] -HUF_decompress4X2_usingDTable_internal_bmi2 [BMI1, BMI2, LZCNT] -HUF_decompress4X2_usingDTable_internal_fast [BMI1, BMI2] -HUF_decompress4X2_usingDTable_internal_fast_asm_loop [BMI2] -HUF_decompress4X2_usingDTable_internal_fast_c_loop [BMI2] -HUF_readStats_body_bmi2 [BMI2, LZCNT] -ZSTD_buildFSETable_body_bmi2 [BMI2] -ZSTD_decompressSequencesLong_bmi2 [BMI1, BMI2, LZCNT] -ZSTD_decompressSequencesSplitLitBuffer_bmi2 [BMI1, BMI2, LZCNT] -ZSTD_decompressSequences_bmi2 [BMI1, BMI2, LZCNT] -ZSTD_encodeSequences_bmi2 [BMI2] - - # ---------------------------------------------------------------------------- # libdeflate. Gate: libdeflate_init_x86_cpu_features. # (10 symbols) From 8c197b5ecfeb3fc575feff12327afcbc697ff9d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 11:58:36 +0000 Subject: [PATCH 002/199] zstd: add the decoder in safe Rust bun_zstd no longer declares zstd's C symbols. It now holds a port of the decoder of zstd 1.5.7 with no unsafe code: frames, skippable frames, dictionaries, prefixes, one call and streaming. For every input it is meant to give the output, the error code and the stream positions that the C decoder gives. bun_pod gets try_zeroed_vec, for buffers whose size the input dictates. The compressor is still missing and nothing is moved to the new API yet, so the build still fails. --- Cargo.lock | 3 +- src/pod/lib.rs | 19 + src/pod/tests.rs | 13 + src/zstd/Cargo.toml | 3 +- src/zstd/common/bitstream.rs | 247 ++++++ src/zstd/common/entropy.rs | 235 ++++++ src/zstd/common/fse_decompress.rs | 230 ++++++ src/zstd/common/mod.rs | 145 ++++ src/zstd/common/xxhash.rs | 139 ++++ src/zstd/decompress/block.rs | 999 ++++++++++++++++++++++ src/zstd/decompress/huf.rs | 1023 +++++++++++++++++++++++ src/zstd/decompress/mod.rs | 1285 +++++++++++++++++++++++++++++ src/zstd/decompress/seq_table.rs | 241 ++++++ src/zstd/error.rs | 122 +++ src/zstd/lib.rs | 731 +++------------- src/zstd/util.rs | 176 ++++ 16 files changed, 5003 insertions(+), 608 deletions(-) create mode 100644 src/zstd/common/bitstream.rs create mode 100644 src/zstd/common/entropy.rs create mode 100644 src/zstd/common/fse_decompress.rs create mode 100644 src/zstd/common/mod.rs create mode 100644 src/zstd/common/xxhash.rs create mode 100644 src/zstd/decompress/block.rs create mode 100644 src/zstd/decompress/huf.rs create mode 100644 src/zstd/decompress/mod.rs create mode 100644 src/zstd/decompress/seq_table.rs create mode 100644 src/zstd/error.rs create mode 100644 src/zstd/util.rs diff --git a/Cargo.lock b/Cargo.lock index 036baf436302..255a0da486f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2089,8 +2089,7 @@ dependencies = [ name = "bun_zstd" version = "0.0.0" dependencies = [ - "bun_core", - "bun_opaque", + "bun_pod", "strum", ] diff --git a/src/pod/lib.rs b/src/pod/lib.rs index 8101df9337bf..2957bea55592 100644 --- a/src/pod/lib.rs +++ b/src/pod/lib.rs @@ -327,5 +327,24 @@ pub fn vec_from_bytes(bytes: &[u8]) -> Option(len: usize) -> Option> { + let layout = core::alloc::Layout::array::(len).ok()?; + if layout.size() == 0 { + // Nothing to allocate: `len` is 0, or `T` has no bytes. + return Some(core::iter::repeat_with(zeroed::).take(len).collect()); + } + // SAFETY: the size of `layout` is not zero. + let ptr = unsafe { alloc::alloc::alloc_zeroed(layout) }.cast::(); + if ptr.is_null() { + return None; + } + // SAFETY: `ptr` comes from the global allocator with the layout of `[T; len]`, and all-zero bytes are `len` valid + // `T`s because `T: Zeroable`. + Some(unsafe { alloc::vec::Vec::from_raw_parts(ptr, len, len) }) +} + #[cfg(test)] mod tests; diff --git a/src/pod/tests.rs b/src/pod/tests.rs index 4602d307f5ec..b6182416e206 100644 --- a/src/pod/tests.rs +++ b/src/pod/tests.rs @@ -122,6 +122,19 @@ fn vec_from_bytes_copies_from_any_address() { assert_eq!(vec_from_bytes::(&[]), Some(Vec::new())); } +#[test] +fn try_zeroed_vec_is_zeros_or_none() { + assert_eq!(try_zeroed_vec::(5).unwrap(), [0; 5]); + assert_eq!(try_zeroed_vec::<[u16; 3]>(2).unwrap(), [[0; 3]; 2]); + assert_eq!(try_zeroed_vec::(0).unwrap().capacity(), 0); + assert_eq!(try_zeroed_vec::<()>(7).unwrap().len(), 7); + assert!(try_zeroed_vec::(usize::MAX / 8).is_none()); + assert!(try_zeroed_vec::(usize::MAX).is_none()); + let mut grows = try_zeroed_vec::(3).unwrap(); + grows.extend_from_slice(&[1; 100]); + assert_eq!(grows.len(), 103); +} + #[test] fn read_prefix_takes_the_front() { let bytes = &BYTES.0; diff --git a/src/zstd/Cargo.toml b/src/zstd/Cargo.toml index 23999d7aaf09..31e702c51bec 100644 --- a/src/zstd/Cargo.toml +++ b/src/zstd/Cargo.toml @@ -10,6 +10,5 @@ path = "lib.rs" workspace = true [dependencies] -bun_opaque.workspace = true strum.workspace = true -bun_core.workspace = true +bun_pod = { workspace = true, features = ["alloc"] } diff --git a/src/zstd/common/bitstream.rs b/src/zstd/common/bitstream.rs new file mode 100644 index 000000000000..035d5badeff9 --- /dev/null +++ b/src/zstd/common/bitstream.rs @@ -0,0 +1,247 @@ +//! `bitstream.h`: a bit stream is written forward and read backward. + +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, highbit32, read_le64}; + +const CONTAINER_BYTES: usize = size_of::(); +const CONTAINER_BITS: u32 = u64::BITS; + +/// `BIT_CStream_t` +pub(crate) struct BitCStream<'a> { + dst: &'a mut [u8], + container: u64, + bit_pos: u32, + pos: usize, + /// The last position a whole container can be written to. + end: usize, +} + +impl<'a> BitCStream<'a> { + /// `BIT_initCStream` + #[inline] + pub(crate) fn new(dst: &'a mut [u8]) -> Result { + if dst.len() <= CONTAINER_BYTES { + return Err(ErrorCode::DstSizeTooSmall); + } + let end = dst.len() - CONTAINER_BYTES; + Ok(Self { + dst, + container: 0, + bit_pos: 0, + pos: 0, + end, + }) + } + + /// `BIT_addBits`: `nb_bits` is less than 32, and there is room for them in the container. + #[inline(always)] + pub(crate) fn add_bits(&mut self, value: u64, nb_bits: u32) { + self.add_bits_fast(value & ((1u64 << (nb_bits & 31)) - 1), nb_bits); + } + + /// `BIT_addBitsFast`: the bits of `value` above `nb_bits` are 0. + #[inline(always)] + pub(crate) fn add_bits_fast(&mut self, value: u64, nb_bits: u32) { + debug_assert!(nb_bits + self.bit_pos < CONTAINER_BITS); + self.container |= value << (self.bit_pos & (CONTAINER_BITS - 1)); + self.bit_pos += nb_bits; + } + + #[inline(always)] + fn flush(&mut self) { + let nb_bytes = (self.bit_pos >> 3) as usize; + if let Ok(out) = self.dst.array_at_mut(self.pos) { + *out = self.container.to_le_bytes(); + } + self.pos += nb_bytes; + self.bit_pos &= 7; + // All 8 bytes leave only if `add_bits` was given more than fits. + self.container = self.container.checked_shr(nb_bytes as u32 * 8).unwrap_or(0); + } + + /// `BIT_flushBitsFast`: the caller knows that the output is large enough. + #[inline(always)] + pub(crate) fn flush_bits_fast(&mut self) { + self.flush(); + } + + /// `BIT_flushBits`: at the end of the output the stream stops advancing, which [`Self::close`] reports. + #[inline(always)] + pub(crate) fn flush_bits(&mut self) { + self.flush(); + if self.pos > self.end { + self.pos = self.end; + } + } + + /// `BIT_closeCStream`: the size of the stream, or 0 if it did not fit. + #[inline] + pub(crate) fn close(mut self) -> usize { + self.add_bits_fast(1, 1); + self.flush_bits(); + if self.pos >= self.end { + return 0; + } + self.pos + usize::from(self.bit_pos > 0) + } +} + +/// `BIT_DStream_status` +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum DStreamStatus { + Unfinished, + EndOfBuffer, + Completed, + Overflow, +} + +/// `BIT_DStream_t` +#[derive(Clone, Copy)] +pub(crate) struct BitDStream<'a> { + src: &'a [u8], + pub(crate) container: u64, + pub(crate) bits_consumed: u32, + /// Where `container` was read from. + pos: usize, +} + +impl<'a> BitDStream<'a> { + /// `BIT_initDStream` + #[inline] + pub(crate) fn new(src: &'a [u8]) -> Result { + let Some(&last_byte) = src.last() else { + return Err(ErrorCode::SrcSizeWrong); + }; + let padding = 8 - highbit32(u32::from(last_byte)); + if let Some(tail) = src.last_chunk::() { + if last_byte == 0 { + return Err(ErrorCode::Generic); + } + return Ok(Self { + src, + container: u64::from_le_bytes(*tail), + bits_consumed: padding, + pos: src.len() - CONTAINER_BYTES, + }); + } + if last_byte == 0 { + return Err(ErrorCode::CorruptionDetected); + } + let mut bytes = [0u8; CONTAINER_BYTES]; + bytes.write_at(0, src)?; + Ok(Self { + src, + container: u64::from_le_bytes(bytes), + bits_consumed: padding + (CONTAINER_BYTES - src.len()) as u32 * 8, + pos: 0, + }) + } + + /// A stream in the state that the 4-stream Huffman loops leave it in. `src` has at least 8 bytes from `pos` on. + #[inline] + pub(crate) fn resume(src: &'a [u8], pos: usize, bits_consumed: u32) -> Result { + Ok(Self { + src, + container: read_le64(src, pos)?, + bits_consumed, + pos, + }) + } + + /// `BIT_lookBits`: `nb_bits` is less than 32. + #[inline(always)] + pub(crate) fn look_bits(&self, nb_bits: u32) -> u64 { + let start = CONTAINER_BITS + .wrapping_sub(self.bits_consumed) + .wrapping_sub(nb_bits); + (self.container >> (start & (CONTAINER_BITS - 1))) & ((1u64 << (nb_bits & 31)) - 1) + } + + /// `BIT_lookBitsFast`: `nb_bits` is at least 1. + #[inline(always)] + pub(crate) fn look_bits_fast(&self, nb_bits: u32) -> u64 { + const MASK: u32 = CONTAINER_BITS - 1; + (self.container << (self.bits_consumed & MASK)) + >> (CONTAINER_BITS.wrapping_sub(nb_bits) & MASK) + } + + /// `BIT_skipBits` + #[inline(always)] + pub(crate) fn skip_bits(&mut self, nb_bits: u32) { + self.bits_consumed = self.bits_consumed.wrapping_add(nb_bits); + } + + /// `BIT_readBits` + #[inline(always)] + pub(crate) fn read_bits(&mut self, nb_bits: u32) -> u64 { + let value = self.look_bits(nb_bits); + self.skip_bits(nb_bits); + value + } + + /// `BIT_readBitsFast` + #[inline(always)] + pub(crate) fn read_bits_fast(&mut self, nb_bits: u32) -> u64 { + let value = self.look_bits_fast(nb_bits); + self.skip_bits(nb_bits); + value + } + + #[inline(always)] + fn load(&mut self, pos: usize) { + self.pos = pos; + if let Ok(container) = read_le64(self.src, pos) { + self.container = container; + } + } + + /// `BIT_reloadDStreamFast` + #[inline(always)] + pub(crate) fn reload_fast(&mut self) -> DStreamStatus { + if self.pos < CONTAINER_BYTES { + return DStreamStatus::Overflow; + } + self.reload_internal() + } + + /// `BIT_reloadDStream_internal`: `pos` is at least 8 and at most 64 bits are consumed. + #[inline(always)] + fn reload_internal(&mut self) -> DStreamStatus { + let pos = self.pos.wrapping_sub((self.bits_consumed >> 3) as usize); + self.bits_consumed &= 7; + self.load(pos); + DStreamStatus::Unfinished + } + + /// `BIT_reloadDStream` + #[inline(always)] + pub(crate) fn reload(&mut self) -> DStreamStatus { + if self.bits_consumed > CONTAINER_BITS { + return DStreamStatus::Overflow; + } + if self.pos >= CONTAINER_BYTES { + return self.reload_internal(); + } + if self.pos == 0 { + if self.bits_consumed < CONTAINER_BITS { + return DStreamStatus::EndOfBuffer; + } + return DStreamStatus::Completed; + } + let mut nb_bytes = (self.bits_consumed >> 3) as usize; + let mut result = DStreamStatus::Unfinished; + if self.pos < nb_bytes { + nb_bytes = self.pos; + result = DStreamStatus::EndOfBuffer; + } + self.bits_consumed -= nb_bytes as u32 * 8; + self.load(self.pos - nb_bytes); + result + } + + /// `BIT_endOfDStream` + #[inline(always)] + pub(crate) fn is_at_end(&self) -> bool { + self.pos == 0 && self.bits_consumed == CONTAINER_BITS + } +} diff --git a/src/zstd/common/entropy.rs b/src/zstd/common/entropy.rs new file mode 100644 index 000000000000..83502cfbec66 --- /dev/null +++ b/src/zstd/common/entropy.rs @@ -0,0 +1,235 @@ +//! `entropy_common.c`: the headers of FSE and Huffman tables. + +use super::fse_decompress::fse_decompress_weights; +use super::{FSE_MIN_TABLELOG, FSE_TABLELOG_ABSOLUTE_MAX, HUF_TABLELOG_MAX}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, highbit32, read_le32}; + +/// What [`read_ncount`] found besides the counts. +pub(crate) struct NCount { + /// How many bytes of the input the header takes. + pub(crate) size: usize, + pub(crate) max_symbol_value: u32, + pub(crate) table_log: u32, +} + +/// `FSE_readNCount`: `normalized_counter` has room for the symbols up to `max_symbol_value`. +pub(crate) fn read_ncount( + normalized_counter: &mut [i16], + max_symbol_value: u32, + header: &[u8], +) -> Result { + if header.len() < 8 { + let mut buffer = [0u8; 8]; + buffer.write_at(0, header)?; + let ncount = read_ncount(normalized_counter, max_symbol_value, &buffer)?; + if ncount.size > header.len() { + return Err(ErrorCode::CorruptionDetected); + } + return Ok(ncount); + } + + let iend = header.len() as isize; + let mut ip: isize = 0; + let load = |ip: isize, bit_count: i32| -> Result { + Ok(read_le32(header, ip as usize)? >> (bit_count & 31)) + }; + let max_sv1 = max_symbol_value + 1; + let mut charnum: u32 = 0; + let mut previous0 = false; + + normalized_counter.sub_mut(..max_sv1 as usize)?.fill(0); + let mut bit_stream = read_le32(header, 0)?; + let mut nb_bits = ((bit_stream & 0xF) + FSE_MIN_TABLELOG) as i32; + if nb_bits > FSE_TABLELOG_ABSOLUTE_MAX as i32 { + return Err(ErrorCode::TableLogTooLarge); + } + bit_stream >>= 4; + let mut bit_count: i32 = 4; + let table_log = nb_bits as u32; + let mut remaining: i32 = (1 << nb_bits) + 1; + let mut threshold: i32 = 1 << nb_bits; + nb_bits += 1; + + loop { + if previous0 { + // Each pair of set bits stands for three more symbols of count 0. + let mut repeats = ((!bit_stream | 0x8000_0000).trailing_zeros() >> 1) as i32; + while repeats >= 12 { + charnum += 3 * 12; + if ip <= iend - 7 { + ip += 3; + } else { + bit_count -= (8 * (iend - 7 - ip)) as i32; + bit_count &= 31; + ip = iend - 4; + } + bit_stream = load(ip, bit_count)?; + repeats = ((!bit_stream | 0x8000_0000).trailing_zeros() >> 1) as i32; + } + charnum += 3 * repeats as u32; + bit_stream >>= 2 * repeats; + bit_count += 2 * repeats; + + charnum += bit_stream & 3; + bit_count += 2; + + if charnum >= max_sv1 { + break; + } + + if ip <= iend - 7 || ip + (bit_count >> 3) as isize <= iend - 4 { + ip += (bit_count >> 3) as isize; + bit_count &= 7; + } else { + bit_count -= (8 * (iend - 4 - ip)) as i32; + bit_count &= 31; + ip = iend - 4; + } + bit_stream = load(ip, bit_count)?; + } + { + let max = (2 * threshold - 1) - remaining; + let mut count: i32; + + if (bit_stream & (threshold - 1) as u32) < max as u32 { + count = (bit_stream & (threshold - 1) as u32) as i32; + bit_count += nb_bits - 1; + } else { + count = (bit_stream & (2 * threshold - 1) as u32) as i32; + if count >= threshold { + count -= max; + } + bit_count += nb_bits; + } + + // The stored value is the count plus one, and a count of -1 takes one slot. + count -= 1; + if count >= 0 { + remaining -= count; + } else { + remaining += count; + } + normalized_counter.put(charnum as usize, count as i16)?; + charnum += 1; + previous0 = count == 0; + + if remaining < threshold { + if remaining <= 1 { + break; + } + nb_bits = highbit32(remaining as u32) as i32 + 1; + threshold = 1 << (nb_bits - 1); + } + if charnum >= max_sv1 { + break; + } + + if ip <= iend - 7 || ip + (bit_count >> 3) as isize <= iend - 4 { + ip += (bit_count >> 3) as isize; + bit_count &= 7; + } else { + bit_count -= (8 * (iend - 4 - ip)) as i32; + bit_count &= 31; + ip = iend - 4; + } + bit_stream = load(ip, bit_count)?; + } + } + if remaining != 1 { + return Err(ErrorCode::CorruptionDetected); + } + if charnum > max_sv1 { + return Err(ErrorCode::MaxSymbolValueTooSmall); + } + if bit_count > 32 { + return Err(ErrorCode::CorruptionDetected); + } + ip += ((bit_count + 7) >> 3) as isize; + Ok(NCount { + size: ip as usize, + max_symbol_value: charnum - 1, + table_log, + }) +} + +/// What [`huf_read_stats`] found besides the weights and how many symbols have each. +pub(crate) struct HufStats { + /// How many bytes of the input the header takes. + pub(crate) size: usize, + pub(crate) nb_symbols: u32, + pub(crate) table_log: u32, +} + +/// `HUF_readStats` +pub(crate) fn huf_read_stats( + huff_weight: &mut [u8; 256], + rank_stats: &mut [u32; HUF_TABLELOG_MAX as usize + 1], + src: &[u8], +) -> Result { + let Some((&first, rest)) = src.split_first() else { + return Err(ErrorCode::SrcSizeWrong); + }; + let mut i_size = usize::from(first); + let o_size; + + if i_size >= 128 { + // Not compressed: two weights to a byte. + o_size = i_size - 127; + i_size = o_size.div_ceil(2); + if i_size + 1 > src.len() { + return Err(ErrorCode::SrcSizeWrong); + } + if o_size >= huff_weight.len() { + return Err(ErrorCode::CorruptionDetected); + } + let (pairs, _) = huff_weight.as_chunks_mut::<2>(); + for (pair, &byte) in pairs.iter_mut().zip(rest.sub(..i_size)?) { + *pair = [byte >> 4, byte & 15]; + } + } else { + if i_size + 1 > src.len() { + return Err(ErrorCode::SrcSizeWrong); + } + o_size = fse_decompress_weights(huff_weight.sub_mut(..255)?, rest.sub(..i_size)?)?; + } + + rank_stats.fill(0); + let mut weight_total: u32 = 0; + for &weight in huff_weight.sub(..o_size)? { + let Some(rank) = rank_stats.get_mut(usize::from(weight)) else { + return Err(ErrorCode::CorruptionDetected); + }; + *rank += 1; + weight_total += (1 << weight) >> 1; + } + if weight_total == 0 { + return Err(ErrorCode::CorruptionDetected); + } + + // The last symbol's weight is not stored: it completes the total to a power of 2. + let table_log = highbit32(weight_total) + 1; + if table_log > HUF_TABLELOG_MAX { + return Err(ErrorCode::CorruptionDetected); + } + let total = 1u32 << table_log; + let rest = total - weight_total; + let verif = 1u32 << highbit32(rest); + let last_weight = highbit32(rest) + 1; + if verif != rest { + return Err(ErrorCode::CorruptionDetected); + } + huff_weight.put(o_size, last_weight as u8)?; + *rank_stats.at_mut(last_weight as usize)? += 1; + + let [_, rank1, ..] = *rank_stats; + if rank1 < 2 || rank1 & 1 != 0 { + return Err(ErrorCode::CorruptionDetected); + } + + Ok(HufStats { + size: i_size + 1, + nb_symbols: o_size as u32 + 1, + table_log, + }) +} diff --git a/src/zstd/common/fse_decompress.rs b/src/zstd/common/fse_decompress.rs new file mode 100644 index 000000000000..a6fd62b9c6e1 --- /dev/null +++ b/src/zstd/common/fse_decompress.rs @@ -0,0 +1,230 @@ +//! `fse_decompress.c`. zstd only decodes the weights of a Huffman table this way: sequences have their own decoder. + +use super::bitstream::{BitDStream, DStreamStatus}; +use super::entropy::read_ncount; +use super::{FSE_MAX_SYMBOL_VALUE, FSE_MAX_TABLELOG, fse_table_step}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, highbit32}; + +/// The largest table log of the weights of a Huffman table. +const MAX_LOG: u32 = 6; + +/// `FSE_decode_t` +#[derive(Clone, Copy, Default)] +struct FseDecode { + new_state: u16, + symbol: u8, + nb_bits: u8, +} + +/// `FSE_DTable` +struct FseDTable { + table_log: u32, + /// Every state reads at least one bit. + fast_mode: bool, + table: [FseDecode; 1 << MAX_LOG], +} + +// The C code carves its tables out of a workspace of `HUF_READ_STATS_WORKSPACE_SIZE` bytes, and refuses the headers +// whose tables do not fit. +const fn dtable_size_u32(table_log: u32) -> usize { + 1 + (1 << table_log) +} +const fn build_dtable_wksp_size(table_log: u32, max_symbol_value: u32) -> usize { + size_of::() * (max_symbol_value as usize + 1) + (1 << table_log) + 8 +} +const fn decompress_wksp_size(table_log: u32, max_symbol_value: u32) -> usize { + (dtable_size_u32(table_log) + + 1 + + build_dtable_wksp_size(table_log, max_symbol_value).div_ceil(size_of::()) + + NCOUNT_SIZE / size_of::() + + 1) + * size_of::() +} +const WKSP_SIZE: usize = decompress_wksp_size(MAX_LOG, super::HUF_TABLELOG_MAX - 1); +const NCOUNT_SIZE: usize = size_of::() * (FSE_MAX_SYMBOL_VALUE as usize + 1); + +/// `FSE_buildDTable_internal` +fn build_dtable( + dt: &mut FseDTable, + normalized_counter: &[i16], + max_symbol_value: u32, + table_log: u32, + wksp_size: usize, +) -> Result<()> { + let mut symbol_next = [0u16; FSE_MAX_SYMBOL_VALUE as usize + 1]; + let max_sv1 = max_symbol_value as usize + 1; + let table_size = 1usize << table_log; + let mut high_threshold = table_size - 1; + + if build_dtable_wksp_size(table_log, max_symbol_value) > wksp_size { + return Err(ErrorCode::MaxSymbolValueTooLarge); + } + if max_symbol_value > FSE_MAX_SYMBOL_VALUE { + return Err(ErrorCode::MaxSymbolValueTooLarge); + } + if table_log > FSE_MAX_TABLELOG { + return Err(ErrorCode::TableLogTooLarge); + } + let counts = normalized_counter.sub(..max_sv1)?; + + dt.table_log = table_log; + dt.fast_mode = true; + let large_limit = 1i16 << (table_log - 1); + for (s, (&count, next)) in counts.iter().zip(&mut symbol_next).enumerate() { + if count == -1 { + dt.table.at_mut(high_threshold)?.symbol = s as u8; + high_threshold = high_threshold.wrapping_sub(1); + *next = 1; + } else { + if count >= large_limit { + dt.fast_mode = false; + } + *next = count as u16; + } + } + + let table_mask = table_size - 1; + let step = fse_table_step(table_size); + let mut position = 0usize; + for (s, &count) in counts.iter().enumerate() { + for _ in 0..count { + dt.table.at_mut(position)?.symbol = s as u8; + position = (position + step) & table_mask; + while position > high_threshold { + position = (position + step) & table_mask; + } + } + } + if position != 0 { + return Err(ErrorCode::Generic); + } + + for entry in dt.table.sub_mut(..table_size)? { + let next = symbol_next.at_mut(usize::from(entry.symbol))?; + let next_state = u32::from(*next); + *next = next.wrapping_add(1); + entry.nb_bits = (table_log - highbit32(next_state)) as u8; + entry.new_state = ((next_state << entry.nb_bits) as usize).wrapping_sub(table_size) as u16; + } + Ok(()) +} + +/// `FSE_DState_t` +struct FseDState { + state: usize, +} + +impl FseDState { + /// `FSE_initDState` + #[inline] + fn new(bit_d: &mut BitDStream<'_>, dt: &FseDTable) -> Self { + let state = bit_d.read_bits(dt.table_log) as usize; + bit_d.reload(); + Self { state } + } + + /// `FSE_decodeSymbol` and `FSE_decodeSymbolFast` + #[inline(always)] + fn decode_symbol( + &mut self, + bit_d: &mut BitDStream<'_>, + dt: &FseDTable, + ) -> Result { + let info = dt.table.at(self.state)?; + let nb_bits = u32::from(info.nb_bits); + let low_bits = if FAST { + bit_d.read_bits_fast(nb_bits) + } else { + bit_d.read_bits(nb_bits) + }; + self.state = usize::from(info.new_state) + low_bits as usize; + Ok(info.symbol) + } +} + +/// `FSE_decompress_usingDTable_generic` +fn decompress_using_dtable( + dst: &mut [u8], + src: &[u8], + dt: &FseDTable, +) -> Result { + let omax = dst.len(); + let mut op = 0usize; + + let mut bit_d = BitDStream::new(src)?; + let mut state1 = FseDState::new(&mut bit_d, dt); + let mut state2 = FseDState::new(&mut bit_d, dt); + if bit_d.reload() == DStreamStatus::Overflow { + return Err(ErrorCode::CorruptionDetected); + } + + while (bit_d.reload() == DStreamStatus::Unfinished) & (op + 3 < omax) { + *dst.array_at_mut(op)? = [ + state1.decode_symbol::(&mut bit_d, dt)?, + state2.decode_symbol::(&mut bit_d, dt)?, + state1.decode_symbol::(&mut bit_d, dt)?, + state2.decode_symbol::(&mut bit_d, dt)?, + ]; + op += 4; + } + + loop { + if op + 2 > omax { + return Err(ErrorCode::DstSizeTooSmall); + } + dst.put(op, state1.decode_symbol::(&mut bit_d, dt)?)?; + op += 1; + if bit_d.reload() == DStreamStatus::Overflow { + dst.put(op, state2.decode_symbol::(&mut bit_d, dt)?)?; + op += 1; + break; + } + + if op + 2 > omax { + return Err(ErrorCode::DstSizeTooSmall); + } + dst.put(op, state2.decode_symbol::(&mut bit_d, dt)?)?; + op += 1; + if bit_d.reload() == DStreamStatus::Overflow { + dst.put(op, state1.decode_symbol::(&mut bit_d, dt)?)?; + op += 1; + break; + } + } + Ok(op) +} + +/// `FSE_decompress_wksp_bmi2`, as `HUF_readStats` calls it. Returns how many weights there are. +pub(crate) fn fse_decompress_weights(dst: &mut [u8], src: &[u8]) -> Result { + let mut ncount = [0i16; FSE_MAX_SYMBOL_VALUE as usize + 1]; + let header = read_ncount(&mut ncount, FSE_MAX_SYMBOL_VALUE, src)?; + if header.table_log > MAX_LOG { + return Err(ErrorCode::TableLogTooLarge); + } + let body = src.sub(header.size..)?; + + if decompress_wksp_size(header.table_log, header.max_symbol_value) > WKSP_SIZE { + return Err(ErrorCode::TableLogTooLarge); + } + let wksp_size = WKSP_SIZE - NCOUNT_SIZE - dtable_size_u32(header.table_log) * size_of::(); + + let mut dtable = FseDTable { + table_log: 0, + fast_mode: false, + table: [FseDecode::default(); 1 << MAX_LOG], + }; + build_dtable( + &mut dtable, + &ncount, + header.max_symbol_value, + header.table_log, + wksp_size, + )?; + + if dtable.fast_mode { + decompress_using_dtable::(dst, body, &dtable) + } else { + decompress_using_dtable::(dst, body, &dtable) + } +} diff --git a/src/zstd/common/mod.rs b/src/zstd/common/mod.rs new file mode 100644 index 000000000000..dd38865b1200 --- /dev/null +++ b/src/zstd/common/mod.rs @@ -0,0 +1,145 @@ +//! What the compressor and the decompressor share: `zstd_internal.h` and `lib/common`. + +pub(crate) mod bitstream; +pub(crate) mod entropy; +pub(crate) mod fse_decompress; +pub(crate) mod xxhash; + +pub(crate) const ZSTD_MAGICNUMBER: u32 = 0xFD2F_B528; +pub(crate) const ZSTD_MAGIC_DICTIONARY: u32 = 0xEC30_A437; +pub(crate) const ZSTD_MAGIC_SKIPPABLE_START: u32 = 0x184D_2A50; +pub(crate) const ZSTD_MAGIC_SKIPPABLE_MASK: u32 = 0xFFFF_FFF0; +pub(crate) const ZSTD_SKIPPABLEHEADERSIZE: usize = 8; + +pub(crate) const ZSTD_BLOCKSIZELOG_MAX: u32 = 17; +pub(crate) const ZSTD_BLOCKSIZE_MAX: usize = 1 << ZSTD_BLOCKSIZELOG_MAX; + +pub(crate) const ZSTD_WINDOWLOG_MAX: u32 = 31; +pub(crate) const ZSTD_WINDOWLOG_ABSOLUTEMIN: u32 = 10; +pub(crate) const ZSTD_WINDOWLOG_LIMIT_DEFAULT: u32 = 27; + +pub const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX; +pub const ZSTD_CONTENTSIZE_ERROR: u64 = u64::MAX - 1; + +pub(crate) const ZSTD_REP_NUM: usize = 3; +pub(crate) const REP_START_VALUE: [u32; ZSTD_REP_NUM] = [1, 4, 8]; + +pub(crate) const ZSTD_FCS_FIELD_SIZE: [usize; 4] = [0, 2, 4, 8]; +pub(crate) const ZSTD_DID_FIELD_SIZE: [usize; 4] = [0, 1, 2, 4]; + +pub(crate) const ZSTD_FRAMEIDSIZE: usize = 4; +pub(crate) const ZSTD_BLOCKHEADERSIZE: usize = 3; + +/// `blockType_e` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum BlockType { + Raw, + Rle, + Compressed, + Reserved, +} + +/// `SymbolEncodingType_e` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum SymbolEncodingType { + Basic, + Rle, + Compressed, + Repeat, +} + +impl SymbolEncodingType { + #[inline] + pub(crate) const fn from_bits(bits: u8) -> Self { + match bits & 3 { + 0 => Self::Basic, + 1 => Self::Rle, + 2 => Self::Compressed, + _ => Self::Repeat, + } + } +} + +pub(crate) const MIN_SEQUENCES_SIZE: usize = 1; +pub(crate) const MIN_CBLOCK_SIZE: usize = 1 + 1; +pub(crate) const MIN_LITERALS_FOR_4_STREAMS: usize = 6; + +pub(crate) const LONGNBSEQ: usize = 0x7F00; + +pub(crate) const MINMATCH: u32 = 3; + +pub(crate) const MAX_ML: u32 = 52; +pub(crate) const MAX_LL: u32 = 35; +pub(crate) const DEFAULT_MAX_OFF: u32 = 28; +pub(crate) const MAX_OFF: u32 = 31; +pub(crate) const MAX_SEQ: u32 = MAX_ML; +pub(crate) const ML_FSE_LOG: u32 = 9; +pub(crate) const LL_FSE_LOG: u32 = 9; +pub(crate) const OFF_FSE_LOG: u32 = 8; + +#[rustfmt::skip] +pub(crate) const LL_BITS: [u8; MAX_LL as usize + 1] = [ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 6, 7, 8, 9,10,11,12, + 13,14,15,16, +]; + +#[rustfmt::skip] +pub(crate) const LL_DEFAULT_NORM: [i16; MAX_LL as usize + 1] = [ + 4, 3, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, + 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1, +]; +pub(crate) const LL_DEFAULT_NORM_LOG: u32 = 6; + +#[rustfmt::skip] +pub(crate) const ML_BITS: [u8; MAX_ML as usize + 1] = [ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 7, 8, 9,10,11, + 12,13,14,15,16, +]; + +#[rustfmt::skip] +pub(crate) const ML_DEFAULT_NORM: [i16; MAX_ML as usize + 1] = [ + 1, 4, 3, 2, 2, 2, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1, +]; +pub(crate) const ML_DEFAULT_NORM_LOG: u32 = 6; + +#[rustfmt::skip] +pub(crate) const OF_DEFAULT_NORM: [i16; DEFAULT_MAX_OFF as usize + 1] = [ + 1, 1, 1, 1, 1, 1, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + -1,-1,-1,-1,-1, +]; +pub(crate) const OF_DEFAULT_NORM_LOG: u32 = 5; + +pub(crate) const WILDCOPY_OVERLENGTH: usize = 32; + +pub(crate) const FSE_MAX_SYMBOL_VALUE: u32 = 255; +pub(crate) const FSE_MAX_TABLELOG: u32 = 12; +pub(crate) const FSE_MIN_TABLELOG: u32 = 5; +pub(crate) const FSE_TABLELOG_ABSOLUTE_MAX: u32 = 15; + +/// `FSE_TABLESTEP` +#[inline(always)] +pub(crate) const fn fse_table_step(table_size: usize) -> usize { + (table_size >> 1) + (table_size >> 3) + 3 +} + +pub(crate) const HUF_TABLELOG_MAX: u32 = 12; +pub(crate) const HUF_SYMBOLVALUE_MAX: u32 = 255; diff --git a/src/zstd/common/xxhash.rs b/src/zstd/common/xxhash.rs new file mode 100644 index 000000000000..c45636fd457d --- /dev/null +++ b/src/zstd/common/xxhash.rs @@ -0,0 +1,139 @@ +//! XXH64, which is the checksum of a frame. + +const PRIME64_1: u64 = 0x9E37_79B1_85EB_CA87; +const PRIME64_2: u64 = 0xC2B2_AE3D_27D4_EB4F; +const PRIME64_3: u64 = 0x1656_67B1_9E37_79F9; +const PRIME64_4: u64 = 0x85EB_CA77_C2B2_AE63; +const PRIME64_5: u64 = 0x27D4_EB2F_1656_67C5; + +const STRIPE: usize = 32; + +#[inline(always)] +fn round(acc: u64, input: u64) -> u64 { + acc.wrapping_add(input.wrapping_mul(PRIME64_2)) + .rotate_left(31) + .wrapping_mul(PRIME64_1) +} + +#[inline(always)] +fn merge_round(acc: u64, val: u64) -> u64 { + (acc ^ round(0, val)) + .wrapping_mul(PRIME64_1) + .wrapping_add(PRIME64_4) +} + +/// `XXH64_state_t`, with a seed of 0. +#[derive(Clone)] +pub(crate) struct Xxh64 { + total_len: u64, + acc: [u64; 4], + buffer: [u8; STRIPE], + buffered: usize, +} + +impl Xxh64 { + /// `XXH64_reset` + pub(crate) const fn new() -> Self { + Self { + total_len: 0, + acc: [ + PRIME64_1.wrapping_add(PRIME64_2), + PRIME64_2, + 0, + 0u64.wrapping_sub(PRIME64_1), + ], + buffer: [0; STRIPE], + buffered: 0, + } + } + + #[inline(always)] + fn consume(acc: &mut [u64; 4], stripe: &[u8; STRIPE]) { + let (lanes, _) = stripe.as_chunks::<8>(); + for (acc, lane) in acc.iter_mut().zip(lanes) { + *acc = round(*acc, u64::from_le_bytes(*lane)); + } + } + + /// `XXH64_update` + pub(crate) fn update(&mut self, mut input: &[u8]) { + self.total_len = self.total_len.wrapping_add(input.len() as u64); + + if self.buffered > 0 { + let Some(free) = self.buffer.get_mut(self.buffered..) else { + return; + }; + let Some((head, tail)) = input.split_at_checked(free.len()) else { + if let Some(free) = free.get_mut(..input.len()) { + free.copy_from_slice(input); + } + self.buffered += input.len(); + return; + }; + free.copy_from_slice(head); + Self::consume(&mut self.acc, &self.buffer); + self.buffered = 0; + input = tail; + } + + let (stripes, rest) = input.as_chunks::(); + let mut acc = self.acc; + for stripe in stripes { + Self::consume(&mut acc, stripe); + } + self.acc = acc; + + if let Some(buffer) = self.buffer.get_mut(..rest.len()) { + buffer.copy_from_slice(rest); + } + self.buffered = rest.len(); + } + + /// `XXH64_digest` + pub(crate) fn digest(&self) -> u64 { + let [v1, v2, v3, v4] = self.acc; + let mut hash = if self.total_len >= STRIPE as u64 { + let mut hash = v1 + .rotate_left(1) + .wrapping_add(v2.rotate_left(7)) + .wrapping_add(v3.rotate_left(12)) + .wrapping_add(v4.rotate_left(18)); + for v in self.acc { + hash = merge_round(hash, v); + } + hash + } else { + v3.wrapping_add(PRIME64_5) + }; + hash = hash.wrapping_add(self.total_len); + + let tail = self.buffer.get(..self.buffered).unwrap_or_default(); + let (lanes, tail) = tail.as_chunks::<8>(); + for lane in lanes { + hash ^= round(0, u64::from_le_bytes(*lane)); + hash = hash + .rotate_left(27) + .wrapping_mul(PRIME64_1) + .wrapping_add(PRIME64_4); + } + let (words, tail) = tail.as_chunks::<4>(); + for word in words { + hash ^= u64::from(u32::from_le_bytes(*word)).wrapping_mul(PRIME64_1); + hash = hash + .rotate_left(23) + .wrapping_mul(PRIME64_2) + .wrapping_add(PRIME64_3); + } + for &byte in tail { + hash ^= u64::from(byte).wrapping_mul(PRIME64_5); + hash = hash.rotate_left(11).wrapping_mul(PRIME64_1); + } + + hash ^= hash >> 33; + hash = hash.wrapping_mul(PRIME64_2); + hash ^= hash >> 29; + hash = hash.wrapping_mul(PRIME64_3); + hash ^= hash >> 32; + hash + } +} diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs new file mode 100644 index 000000000000..5d53cb4cbef2 --- /dev/null +++ b/src/zstd/decompress/block.rs @@ -0,0 +1,999 @@ +//! `zstd_decompress_block.c`: one compressed block, which is literals and the sequences that mix them with matches. + +use super::huf::{self, HufDTable}; +use super::seq_table::{ + LL_BASE, LL_DEFAULT_DTABLE, ML_BASE, ML_DEFAULT_DTABLE, OF_BASE, OF_BITS, OF_DEFAULT_DTABLE, + SeqTable, +}; +use crate::common::bitstream::BitDStream; +use crate::common::entropy::read_ncount; +use crate::common::{ + BlockType, LL_BITS, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MIN_CBLOCK_SIZE, + MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, ML_BITS, ML_FSE_LOG, OFF_FSE_LOG, + REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, read_le16, read_le24, read_le32}; + +/// `blockProperties_t` +pub(crate) struct BlockProperties { + pub(crate) block_type: BlockType, + pub(crate) last_block: bool, + pub(crate) orig_size: u32, +} + +/// `ZSTD_getcBlockSize`: reads a block header, and returns how many bytes of input the block takes after it. +pub(crate) fn get_c_block_size(src: &[u8]) -> Result<(usize, BlockProperties)> { + if src.len() < ZSTD_BLOCKHEADERSIZE { + return Err(ErrorCode::SrcSizeWrong); + } + let c_block_header = read_le24(src, 0)?; + let c_size = c_block_header >> 3; + let block_type = match (c_block_header >> 1) & 3 { + 0 => BlockType::Raw, + 1 => BlockType::Rle, + 2 => BlockType::Compressed, + _ => BlockType::Reserved, + }; + let properties = BlockProperties { + block_type, + last_block: c_block_header & 1 != 0, + orig_size: c_size, + }; + match block_type { + BlockType::Rle => Ok((1, properties)), + BlockType::Reserved => Err(ErrorCode::CorruptionDetected), + _ => Ok((c_size as usize, properties)), + } +} + +/// `ZSTD_entropyDTables_t` +#[derive(Clone)] +pub(crate) struct EntropyDTables { + pub(crate) ll_table: SeqTable<{ 1 << LL_FSE_LOG }>, + pub(crate) of_table: SeqTable<{ 1 << OFF_FSE_LOG }>, + pub(crate) ml_table: SeqTable<{ 1 << ML_FSE_LOG }>, + pub(crate) huf_table: HufDTable, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +impl EntropyDTables { + pub(crate) const fn new() -> Self { + Self { + ll_table: SeqTable::new(), + of_table: SeqTable::new(), + ml_table: SeqTable::new(), + huf_table: HufDTable::new(), + rep: REP_START_VALUE, + } + } +} + +/// Which table `LLTptr`, `MLTptr`, `OFTptr` or `HUFptr` points to. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TableSource { + /// The context's own. + Local, + /// The dictionary's. + Dict, + /// The one the format predefines. There is none for literals. + Predefined, +} + +/// What precedes the contiguous output, for matches to reach into: `dictEnd` and `virtualStart`. +#[derive(Clone, Copy)] +pub(crate) enum ExtDict<'a> { + /// The content of a dictionary. Empty if there is none. + Dict(&'a [u8]), + /// The `len` bytes up to `end` of the buffer that the output goes to, which is the streaming decoder's and has + /// wrapped around. + Within { end: usize, len: usize }, +} + +impl ExtDict<'_> { + #[inline(always)] + fn len(&self) -> usize { + match *self { + ExtDict::Dict(dict) => dict.len(), + ExtDict::Within { len, .. } => len, + } + } + + /// `memmove(&buf[dst], dictEnd - back, len)` + #[inline] + fn copy_to(&self, buf: &mut [u8], dst: usize, back: usize, len: usize) -> Result<()> { + match *self { + ExtDict::Dict(dict) => buf.write_at(dst, dict.span(dict.len() - back, len)?), + ExtDict::Within { end, .. } => buf.move_within(end - back, dst, len), + } + } +} + +/// Where a block is written. +pub(crate) struct BlockOutput<'a> { + /// Holds the contiguous output so far, from `prefix_start` to `start`. + pub(crate) buf: &'a mut [u8], + pub(crate) prefix_start: usize, + pub(crate) start: usize, + /// How many bytes from `start` on may be written. + pub(crate) capacity: usize, + pub(crate) ext_dict: ExtDict<'a>, +} + +/// `streaming_operation` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Streaming { + No, + Yes, +} + +/// `ZSTD_LITBUFFEREXTRASIZE` +const LIT_BUFFER_EXTRA_SIZE: usize = 1 << 16; + +/// `ZSTD_litLocation_e`. The C code saves memory by keeping the literals of a block in the part of the output that the +/// block has not reached yet. Here they never are, but where they would have been limits how far the block may write. +#[derive(Clone, Copy, PartialEq, Eq)] +enum LitLocation { + NotInDst, + /// Past the largest block, so that is as far as the block may write. + InDst, + /// All but the last `LIT_BUFFER_EXTRA_SIZE` at the end of the block, where the output must not catch up with them. + Split, +} + +/// `ZSTD_allocateLiteralsBuffer` +fn lit_location( + block_size_max: usize, + dst_capacity: usize, + lit_size: usize, + streaming: Streaming, +) -> LitLocation { + if streaming == Streaming::No + && dst_capacity > block_size_max + WILDCOPY_OVERLENGTH + lit_size + WILDCOPY_OVERLENGTH + { + LitLocation::InDst + } else if lit_size <= LIT_BUFFER_EXTRA_SIZE { + LitLocation::NotInDst + } else { + LitLocation::Split + } +} + +/// The literals of a block, once decoded. +struct Literals { + /// How many bytes of the block they take. + c_size: usize, + size: usize, + /// Where they start in the block, if they are stored there as they are with enough after them to copy in chunks. + /// Otherwise they are in the context's buffer. + in_src: Option, + location: LitLocation, +} + +/// The state of a `ZSTD_DCtx` that lasts from block to block. +pub(crate) struct BlockDecoder { + pub(crate) entropy: Box, + pub(crate) ll_source: TableSource, + pub(crate) of_source: TableSource, + pub(crate) ml_source: TableSource, + pub(crate) huf_source: TableSource, + /// There is a Huffman table to repeat. + pub(crate) lit_entropy: bool, + /// There are sequence tables to repeat. + pub(crate) fse_entropy: bool, + pub(crate) ddict_is_cold: bool, + /// Grows to the literals of the largest block so far, and `WILDCOPY_OVERLENGTH`. + lit_buffer: Vec, +} + +impl BlockDecoder { + pub(crate) fn new() -> Self { + Self { + entropy: Box::new(EntropyDTables::new()), + ll_source: TableSource::Local, + of_source: TableSource::Local, + ml_source: TableSource::Local, + huf_source: TableSource::Local, + lit_entropy: false, + fse_entropy: false, + ddict_is_cold: false, + lit_buffer: Vec::new(), + } + } + + fn lit_buffer(&mut self, lit_size: usize) -> Result<&mut [u8]> { + let needed = lit_size + WILDCOPY_OVERLENGTH; + if self.lit_buffer.len() < needed { + self.lit_buffer + .try_reserve(needed - self.lit_buffer.len()) + .map_err(|_| ErrorCode::MemoryAllocation)?; + self.lit_buffer.resize(needed, 0); + } + self.lit_buffer.sub_mut(..lit_size) + } + + /// `ZSTD_decodeLiteralsBlock` + fn decode_literals_block( + &mut self, + src: &[u8], + dict_entropy: Option<&EntropyDTables>, + block_size_max: usize, + dst_capacity: usize, + streaming: Streaming, + ) -> Result { + if src.len() < MIN_CBLOCK_SIZE { + return Err(ErrorCode::CorruptionDetected); + } + let first = src.at(0)?; + let lit_enc_type = SymbolEncodingType::from_bits(first); + let lhl_code = (first >> 2) & 3; + let expected_write_size = block_size_max.min(dst_capacity); + + match lit_enc_type { + SymbolEncodingType::Repeat | SymbolEncodingType::Compressed => { + if lit_enc_type == SymbolEncodingType::Repeat && !self.lit_entropy { + return Err(ErrorCode::DictionaryCorrupted); + } + if src.len() < 5 { + return Err(ErrorCode::CorruptionDetected); + } + let lhc = read_le32(src, 0)? as usize; + let (single_stream, lh_size, lit_size, lit_c_size) = match lhl_code { + 2 => (false, 4, (lhc >> 4) & 0x3FFF, lhc >> 18), + 3 => ( + false, + 5, + (lhc >> 4) & 0x3FFFF, + (lhc >> 22) + (usize::from(src.at(4)?) << 10), + ), + _ => (lhl_code == 0, 3, (lhc >> 4) & 0x3FF, (lhc >> 14) & 0x3FF), + }; + if lit_size > block_size_max { + return Err(ErrorCode::CorruptionDetected); + } + if !single_stream && lit_size < MIN_LITERALS_FOR_4_STREAMS { + return Err(ErrorCode::LiteralsHeaderWrong); + } + if lit_c_size + lh_size > src.len() { + return Err(ErrorCode::CorruptionDetected); + } + if expected_write_size < lit_size { + return Err(ErrorCode::DstSizeTooSmall); + } + let location = lit_location(block_size_max, dst_capacity, lit_size, streaming); + let compressed = src.span(lh_size, lit_c_size)?; + + let huf_source = self.huf_source; + self.lit_buffer(lit_size)?; + let Self { + entropy, + lit_buffer, + .. + } = self; + let dst = lit_buffer.sub_mut(..lit_size)?; + let huf_result = if lit_enc_type == SymbolEncodingType::Repeat { + let table = match (huf_source, dict_entropy) { + (TableSource::Dict, Some(dict)) => &dict.huf_table, + _ => &entropy.huf_table, + }; + if single_stream { + huf::decompress_1x_using_dtable(dst, compressed, table) + } else { + huf::decompress_4x_using_dtable(dst, compressed, table) + } + } else if single_stream { + huf::decompress_1x1(&mut entropy.huf_table, dst, compressed) + } else { + huf::decompress_4x_huf_only(&mut entropy.huf_table, dst, compressed) + }; + if huf_result.is_err() { + return Err(ErrorCode::CorruptionDetected); + } + + self.lit_entropy = true; + if lit_enc_type == SymbolEncodingType::Compressed { + self.huf_source = TableSource::Local; + } + Ok(Literals { + c_size: lit_c_size + lh_size, + size: lit_size, + in_src: None, + location, + }) + } + + SymbolEncodingType::Basic => { + let (lh_size, lit_size) = match lhl_code { + 1 => (2, usize::from(read_le16(src, 0)? >> 4)), + 3 => { + if src.len() < 3 { + return Err(ErrorCode::CorruptionDetected); + } + (3, (read_le24(src, 0)? >> 4) as usize) + } + _ => (1, usize::from(first >> 3)), + }; + if lit_size > block_size_max { + return Err(ErrorCode::CorruptionDetected); + } + if expected_write_size < lit_size { + return Err(ErrorCode::DstSizeTooSmall); + } + if lh_size + lit_size + WILDCOPY_OVERLENGTH > src.len() { + if lit_size + lh_size > src.len() { + return Err(ErrorCode::CorruptionDetected); + } + self.lit_buffer(lit_size)? + .copy_from_slice(src.span(lh_size, lit_size)?); + return Ok(Literals { + c_size: lh_size + lit_size, + size: lit_size, + in_src: None, + location: lit_location(block_size_max, dst_capacity, lit_size, streaming), + }); + } + Ok(Literals { + c_size: lh_size + lit_size, + size: lit_size, + in_src: Some(lh_size), + location: LitLocation::NotInDst, + }) + } + + SymbolEncodingType::Rle => { + let (lh_size, lit_size) = match lhl_code { + 1 => { + if src.len() < 3 { + return Err(ErrorCode::CorruptionDetected); + } + (2, usize::from(read_le16(src, 0)? >> 4)) + } + 3 => { + if src.len() < 4 { + return Err(ErrorCode::CorruptionDetected); + } + (3, (read_le24(src, 0)? >> 4) as usize) + } + _ => (1, usize::from(first >> 3)), + }; + if lit_size > block_size_max { + return Err(ErrorCode::CorruptionDetected); + } + if expected_write_size < lit_size { + return Err(ErrorCode::DstSizeTooSmall); + } + self.lit_buffer(lit_size)?.fill(src.at(lh_size)?); + Ok(Literals { + c_size: lh_size + 1, + size: lit_size, + in_src: None, + location: lit_location(block_size_max, dst_capacity, lit_size, streaming), + }) + } + } + } + + /// `ZSTD_decodeSeqHeaders`: returns how many sequences there are, and the size of the header. + fn decode_seq_headers(&mut self, src: &[u8]) -> Result<(usize, usize)> { + if src.len() < MIN_SEQUENCES_SIZE { + return Err(ErrorCode::SrcSizeWrong); + } + let mut ip = 0usize; + let mut nb_seq = usize::from(src.at(ip)?); + ip += 1; + if nb_seq > 0x7F { + if nb_seq == 0xFF { + if ip + 2 > src.len() { + return Err(ErrorCode::SrcSizeWrong); + } + nb_seq = usize::from(read_le16(src, ip)?) + LONGNBSEQ; + ip += 2; + } else { + if ip >= src.len() { + return Err(ErrorCode::SrcSizeWrong); + } + nb_seq = ((nb_seq - 0x80) << 8) + usize::from(src.at(ip)?); + ip += 1; + } + } + + if nb_seq == 0 { + if ip != src.len() { + return Err(ErrorCode::CorruptionDetected); + } + return Ok((0, ip)); + } + + if ip + 1 > src.len() { + return Err(ErrorCode::SrcSizeWrong); + } + let modes = src.at(ip)?; + if modes & 3 != 0 { + return Err(ErrorCode::CorruptionDetected); + } + ip += 1; + + let fse_entropy = self.fse_entropy; + ip += build_seq_table( + &mut self.entropy.ll_table, + &mut self.ll_source, + SymbolEncodingType::from_bits(modes >> 6), + MAX_LL, + LL_FSE_LOG, + src.sub(ip..)?, + &LL_BASE, + &LL_BITS, + fse_entropy, + ) + .map_err(|_| ErrorCode::CorruptionDetected)?; + ip += build_seq_table( + &mut self.entropy.of_table, + &mut self.of_source, + SymbolEncodingType::from_bits(modes >> 4), + MAX_OFF, + OFF_FSE_LOG, + src.sub(ip..)?, + &OF_BASE, + &OF_BITS, + fse_entropy, + ) + .map_err(|_| ErrorCode::CorruptionDetected)?; + ip += build_seq_table( + &mut self.entropy.ml_table, + &mut self.ml_source, + SymbolEncodingType::from_bits(modes >> 2), + MAX_ML, + ML_FSE_LOG, + src.sub(ip..)?, + &ML_BASE, + &ML_BITS, + fse_entropy, + ) + .map_err(|_| ErrorCode::CorruptionDetected)?; + + Ok((nb_seq, ip)) + } + + /// `ZSTD_decompressBlock_internal`: returns the size of the block once decompressed. + pub(crate) fn decompress_block( + &mut self, + out: BlockOutput<'_>, + src: &[u8], + dict_entropy: Option<&EntropyDTables>, + block_size_max: usize, + streaming: Streaming, + ) -> Result { + let BlockOutput { + buf, + prefix_start, + start, + capacity, + ext_dict, + } = out; + if src.len() > block_size_max { + return Err(ErrorCode::SrcSizeWrong); + } + + let literals = + self.decode_literals_block(src, dict_entropy, block_size_max, capacity, streaming)?; + let sequences = src.sub(literals.c_size..)?; + + let total_history_size = + start - prefix_start + ext_dict.len() + capacity.min(block_size_max); + let mut use_prefetch_decoder = self.ddict_is_cold; + let (nb_seq, seq_h_size) = self.decode_seq_headers(sequences)?; + let sequences = sequences.sub(seq_h_size..)?; + + if capacity == 0 && nb_seq > 0 { + return Err(ErrorCode::DstSizeTooSmall); + } + + let Self { + entropy, + lit_buffer, + .. + } = self; + let EntropyDTables { + ll_table, + of_table, + ml_table, + rep, + .. + } = &mut **entropy; + let ll_table = match (self.ll_source, dict_entropy) { + (TableSource::Predefined, _) => &LL_DEFAULT_DTABLE, + (TableSource::Dict, Some(dict)) => &dict.ll_table, + _ => &*ll_table, + }; + let of_table = match (self.of_source, dict_entropy) { + (TableSource::Predefined, _) => &OF_DEFAULT_DTABLE, + (TableSource::Dict, Some(dict)) => &dict.of_table, + _ => &*of_table, + }; + let ml_table = match (self.ml_source, dict_entropy) { + (TableSource::Predefined, _) => &ML_DEFAULT_DTABLE, + (TableSource::Dict, Some(dict)) => &dict.ml_table, + _ => &*ml_table, + }; + + // With matches far back in a long history, it pays to know the sequences ahead of time. Nothing is prefetched + // here, but which of two problems in a damaged block is reported depends on the order. + if !use_prefetch_decoder && total_history_size > (1 << 24) && nb_seq > 8 { + const MIN_SHARE: u32 = 7; + use_prefetch_decoder = long_offset_share(of_table) >= MIN_SHARE; + } + self.ddict_is_cold = false; + if nb_seq > 0 { + self.fse_entropy = true; + } + + let block_end = start + capacity; + let oend = if literals.location == LitLocation::InDst { + start + block_size_max + WILDCOPY_OVERLENGTH + } else { + block_end + }; + let split = literals.location == LitLocation::Split; + let in_dst_size = literals.size.saturating_sub(LIT_BUFFER_EXTRA_SIZE); + let mut exec = Executor { + buf: buf.sub_mut(..block_end)?, + op: start, + oend, + prefix_start, + ext_dict, + lits: match literals.in_src { + Some(start) => src.sub(start..)?, + None => lit_buffer.as_slice(), + }, + lit_pos: 0, + lit_limit: if split { in_dst_size } else { literals.size }, + lit_size: literals.size, + split, + // `litBufferEnd` is `WILDCOPY_OVERLENGTH` short of how far the block is expected to write. + lit_in_dst: (start + capacity.min(block_size_max)) + .wrapping_sub(WILDCOPY_OVERLENGTH + in_dst_size), + }; + + if nb_seq > 0 { + let mut seq_state = SeqState::new(sequences, ll_table, of_table, ml_table, rep)?; + if use_prefetch_decoder { + exec.run_long(&mut seq_state, nb_seq)?; + } else { + exec.run(&mut seq_state, nb_seq)?; + } + *rep = seq_state.prev_offset.map(|offset| offset as u32); + } + exec.last_literals()?; + Ok(exec.op - start) + } +} + +/// `ZSTD_buildSeqTable`: returns the size of the table's description in `src`. +fn build_seq_table( + table: &mut SeqTable, + source: &mut TableSource, + encoding_type: SymbolEncodingType, + max: u32, + max_log: u32, + src: &[u8], + base_value: &[u32], + nb_additional_bits: &[u8], + flag_repeat_table: bool, +) -> Result { + match encoding_type { + SymbolEncodingType::Rle => { + let Some(&symbol) = src.first() else { + return Err(ErrorCode::SrcSizeWrong); + }; + if u32::from(symbol) > max { + return Err(ErrorCode::CorruptionDetected); + } + let symbol = usize::from(symbol); + table.build_rle(base_value.at(symbol)?, nb_additional_bits.at(symbol)?); + *source = TableSource::Local; + Ok(1) + } + SymbolEncodingType::Basic => { + *source = TableSource::Predefined; + Ok(0) + } + SymbolEncodingType::Repeat => { + if !flag_repeat_table { + return Err(ErrorCode::CorruptionDetected); + } + Ok(0) + } + SymbolEncodingType::Compressed => { + let mut norm = [0i16; MAX_SEQ as usize + 1]; + let header = + read_ncount(&mut norm, max, src).map_err(|_| ErrorCode::CorruptionDetected)?; + if header.table_log > max_log { + return Err(ErrorCode::CorruptionDetected); + } + table.build( + norm.sub(..header.max_symbol_value as usize + 1)?, + base_value, + nb_additional_bits, + header.table_log, + )?; + *source = TableSource::Local; + Ok(header.size) + } + } +} + +/// `ZSTD_getOffsetInfo`: how many of 256 offset codes have more than 22 bits. +fn long_offset_share(of_table: &SeqTable<{ 1 << OFF_FSE_LOG }>) -> u32 { + let count = of_table + .symbols() + .iter() + .filter(|symbol| symbol.nb_additional_bits > 22) + .count() as u32; + count << (OFF_FSE_LOG - of_table.table_log) +} + +/// `seq_t` +#[derive(Clone, Copy, Default)] +struct Sequence { + lit_length: usize, + match_length: usize, + offset: usize, +} + +/// `seqState_t` +struct SeqState<'a> { + bit_d: BitDStream<'a>, + state_ll: usize, + state_off: usize, + state_ml: usize, + ll_table: &'a SeqTable<{ 1 << LL_FSE_LOG }>, + of_table: &'a SeqTable<{ 1 << OFF_FSE_LOG }>, + ml_table: &'a SeqTable<{ 1 << ML_FSE_LOG }>, + prev_offset: [usize; ZSTD_REP_NUM], +} + +impl<'a> SeqState<'a> { + fn new( + src: &'a [u8], + ll_table: &'a SeqTable<{ 1 << LL_FSE_LOG }>, + of_table: &'a SeqTable<{ 1 << OFF_FSE_LOG }>, + ml_table: &'a SeqTable<{ 1 << ML_FSE_LOG }>, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result { + let mut bit_d = BitDStream::new(src).map_err(|_| ErrorCode::CorruptionDetected)?; + // `ZSTD_initFseState` + let mut init = |table_log: u32| { + let state = bit_d.read_bits(table_log) as usize; + bit_d.reload(); + state + }; + let state_ll = init(ll_table.table_log); + let state_off = init(of_table.table_log); + let state_ml = init(ml_table.table_log); + Ok(Self { + bit_d, + state_ll, + state_off, + state_ml, + ll_table, + of_table, + ml_table, + prev_offset: rep.map(|offset| offset as usize), + }) + } + + /// `ZSTD_decodeSequence` + #[inline(always)] + fn decode_sequence(&mut self, is_last_seq: bool) -> Sequence { + let ll_d_info = self.ll_table.get(self.state_ll); + let ml_d_info = self.ml_table.get(self.state_ml); + let of_d_info = self.of_table.get(self.state_off); + let mut match_length = ml_d_info.base_value as usize; + let mut lit_length = ll_d_info.base_value as usize; + let of_base = of_d_info.base_value as usize; + let ll_bits = u32::from(ll_d_info.nb_additional_bits); + let ml_bits = u32::from(ml_d_info.nb_additional_bits); + let of_bits = u32::from(of_d_info.nb_additional_bits); + let total_bits = ll_bits + ml_bits + of_bits; + + let [prev0, prev1, prev2] = self.prev_offset; + let offset; + if of_bits > 1 { + offset = of_base + self.bit_d.read_bits_fast(of_bits) as usize; + self.prev_offset = [offset, prev0, prev1]; + } else { + let ll0 = ll_d_info.base_value == 0; + if of_bits == 0 { + if ll0 { + offset = prev1; + self.prev_offset = [prev1, prev0, prev2]; + } else { + offset = prev0; + } + } else { + let code = of_base + usize::from(ll0) + self.bit_d.read_bits_fast(1) as usize; + let temp = match code { + 1 => prev1, + 2 => prev2, + _ => prev0.wrapping_sub(1), + }; + // 0 is not valid: it becomes an offset that no history is long enough for. + offset = temp.wrapping_sub(usize::from(temp == 0)); + self.prev_offset = [offset, prev0, if code != 1 { prev1 } else { prev2 }]; + } + } + + if ml_bits > 0 { + match_length += self.bit_d.read_bits_fast(ml_bits) as usize; + } + const ACCUMULATOR_MIN: u32 = 57; + if total_bits >= ACCUMULATOR_MIN - (LL_FSE_LOG + ML_FSE_LOG + OFF_FSE_LOG) { + self.bit_d.reload(); + } + if ll_bits > 0 { + lit_length += self.bit_d.read_bits_fast(ll_bits) as usize; + } + + if !is_last_seq { + // `ZSTD_updateFseStateWithDInfo` + self.state_ll = usize::from(ll_d_info.next_state) + + self.bit_d.read_bits(u32::from(ll_d_info.nb_bits)) as usize; + self.state_ml = usize::from(ml_d_info.next_state) + + self.bit_d.read_bits(u32::from(ml_d_info.nb_bits)) as usize; + self.state_off = usize::from(of_d_info.next_state) + + self.bit_d.read_bits(u32::from(of_d_info.nb_bits)) as usize; + self.bit_d.reload(); + } + + Sequence { + lit_length, + match_length, + offset, + } + } +} + +#[inline(always)] +fn copy_within(buf: &mut [u8], src: usize, dst: usize) -> Result<()> { + let chunk: [u8; N] = *buf.array_at(src)?; + *buf.array_at_mut(dst)? = chunk; + Ok(()) +} + +/// `buf[dst + i] = buf[src + i]`, one after the other, so that a match repeats itself where it overlaps its copy. +/// `src` is less than `dst`. +fn copy_match_exact(buf: &mut [u8], src: usize, dst: usize, len: usize) -> Result<()> { + let offset = dst - src; + if offset >= len { + return buf.move_within(src, dst, len); + } + // What is copied so far is whole periods, so it can be copied again from the start, twice as much each time. + let mut done = 0usize; + while done < len { + let chunk = (offset + done).min(len - done); + buf.move_within(src, dst + done, chunk)?; + done += chunk; + } + Ok(()) +} + +/// Turns sequences into output. +struct Executor<'a> { + /// Ends where the block has to. + buf: &'a mut [u8], + op: usize, + /// How far the C code would let the block write. + oend: usize, + prefix_start: usize, + ext_dict: ExtDict<'a>, + /// At least `WILDCOPY_OVERLENGTH` longer than `lit_size`. + lits: &'a [u8], + lit_pos: usize, + /// `litBufferEnd` + lit_limit: usize, + lit_size: usize, + /// `litBufferLocation == ZSTD_split`: the literals up to `lit_limit` would be in the output buffer. + split: bool, + /// Where in `buf` they would start. + lit_in_dst: usize, +} + +impl Executor<'_> { + /// The match of a sequence whose literals end at `o_lit_end`. `FAST`: 32 bytes after the match may be written. + #[inline(always)] + fn copy_match(&mut self, o_lit_end: usize, sequence: Sequence) -> Result<()> { + let Sequence { + mut match_length, + offset, + .. + } = sequence; + let mut op = o_lit_end; + let in_prefix = o_lit_end - self.prefix_start; + let mut match_pos = o_lit_end.wrapping_sub(offset); + + if offset > in_prefix { + if offset > in_prefix + self.ext_dict.len() { + return Err(ErrorCode::CorruptionDetected); + } + let back = offset - in_prefix; + if back >= match_length { + return self.ext_dict.copy_to(self.buf, op, back, match_length); + } + // The match spans the two segments. + self.ext_dict.copy_to(self.buf, op, back, back)?; + op += back; + match_length -= back; + match_pos = self.prefix_start; + } + + if !FAST { + return copy_match_exact(self.buf, match_pos, op, match_length); + } + + if offset >= 16 { + // `ZSTD_wildcopy(ZSTD_no_overlap)` + let mut done = 0usize; + loop { + copy_within::<16>(self.buf, match_pos + done, op + done)?; + done += 16; + if done >= match_length { + return Ok(()); + } + } + } + + // `ZSTD_overlapCopy8`: after 8 bytes, the source is at least 8 bytes behind. + if offset < 8 { + const DEC32_TABLE: [usize; 8] = [0, 1, 2, 1, 4, 4, 4, 4]; + const DEC64_TABLE: [usize; 8] = [8, 8, 8, 7, 8, 9, 10, 11]; + for i in 0..4 { + let byte = self.buf.at(match_pos + i)?; + self.buf.put(op + i, byte)?; + } + match_pos += DEC32_TABLE.at(offset)?; + copy_within::<4>(self.buf, match_pos, op + 4)?; + match_pos += 8; + match_pos -= DEC64_TABLE.at(offset)?; + } else { + copy_within::<8>(self.buf, match_pos, op)?; + match_pos += 8; + } + op += 8; + // `ZSTD_wildcopy(ZSTD_overlap_src_before_dst)` + let mut done = 8usize; + while done < match_length { + copy_within::<8>(self.buf, match_pos, op)?; + match_pos += 8; + op += 8; + done += 8; + } + Ok(()) + } + + /// `ZSTD_execSequenceEnd` and `ZSTD_execSequenceEndSplitLitBuffer`: near the end of a buffer. + #[cold] + #[inline(never)] + fn exec_sequence_end(&mut self, sequence: Sequence) -> Result<()> { + let op = self.op; + let o_lit_end = op + sequence.lit_length; + let sequence_length = sequence.lit_length + sequence.match_length; + + if sequence_length > self.oend - op { + return Err(ErrorCode::DstSizeTooSmall); + } + if sequence.lit_length > self.lit_limit - self.lit_pos { + return Err(ErrorCode::CorruptionDetected); + } + if self.split { + let lit_ptr = self.lit_in_dst.wrapping_add(self.lit_pos); + if op > lit_ptr && op < lit_ptr + sequence.lit_length { + return Err(ErrorCode::DstSizeTooSmall); + } + } + + self.buf + .write_at(op, self.lits.span(self.lit_pos, sequence.lit_length)?)?; + self.lit_pos += sequence.lit_length; + self.copy_match::(o_lit_end, sequence)?; + self.op = op + sequence_length; + Ok(()) + } + + /// `ZSTD_execSequence` and `ZSTD_execSequenceSplitLitBuffer` + #[inline(always)] + fn exec_sequence(&mut self, sequence: Sequence) -> Result<()> { + let op = self.op; + let o_lit_end = op + sequence.lit_length; + let o_match_end = o_lit_end + sequence.match_length; + let i_lit_end = self.lit_pos + sequence.lit_length; + // How far a copy in chunks may write, plus `WILDCOPY_OVERLENGTH`. + let limit = if self.split { + self.lit_in_dst.wrapping_add(i_lit_end) + } else { + self.oend + }; + + if i_lit_end > self.lit_limit || o_match_end + WILDCOPY_OVERLENGTH > limit { + return self.exec_sequence_end(sequence); + } + + *self.buf.array_at_mut::<16>(op)? = *self.lits.array_at(self.lit_pos)?; + if sequence.lit_length > 16 { + self.buf + .write_at(op + 16, self.lits.sub(self.lit_pos + 16..i_lit_end)?)?; + } + self.lit_pos = i_lit_end; + self.copy_match::(o_lit_end, sequence)?; + self.op = o_match_end; + Ok(()) + } + + /// A sequence, whose literals may start among those that would be in the output buffer and end among the others. + #[inline(always)] + fn exec_sequence_maybe_split(&mut self, mut sequence: Sequence) -> Result<()> { + if self.split && self.lit_pos + sequence.lit_length > self.lit_limit { + let leftover_lit = self.lit_limit - self.lit_pos; + if leftover_lit > 0 { + if leftover_lit > self.oend - self.op { + return Err(ErrorCode::DstSizeTooSmall); + } + self.buf + .write_at(self.op, self.lits.span(self.lit_pos, leftover_lit)?)?; + sequence.lit_length -= leftover_lit; + self.op += leftover_lit; + self.lit_pos += leftover_lit; + } + self.split = false; + self.lit_limit = self.lit_size; + } + self.exec_sequence(sequence) + } + + /// `ZSTD_decompressSequences_body` and `ZSTD_decompressSequences_bodySplitLitBuffer` + fn run(&mut self, seq_state: &mut SeqState<'_>, nb_seq: usize) -> Result<()> { + for remaining in (1..=nb_seq).rev() { + let sequence = seq_state.decode_sequence(remaining == 1); + self.exec_sequence_maybe_split(sequence)?; + } + if !seq_state.bit_d.is_at_end() { + return Err(ErrorCode::CorruptionDetected); + } + Ok(()) + } + + /// `ZSTD_decompressSequencesLong_body`: sequences are decoded 8 ahead of the one that is executed. + fn run_long(&mut self, seq_state: &mut SeqState<'_>, nb_seq: usize) -> Result<()> { + const STORED_SEQS: usize = 8; + const STORED_SEQS_MASK: usize = STORED_SEQS - 1; + let mut sequences = [Sequence::default(); STORED_SEQS]; + let seq_advance = nb_seq.min(STORED_SEQS); + + for (seq_nb, stored) in sequences.iter_mut().enumerate().take(seq_advance) { + *stored = seq_state.decode_sequence(seq_nb == nb_seq - 1); + } + for seq_nb in seq_advance..nb_seq { + let sequence = seq_state.decode_sequence(seq_nb == nb_seq - 1); + let stored = sequences.at_mut(seq_nb & STORED_SEQS_MASK)?; + self.exec_sequence_maybe_split(core::mem::replace(stored, sequence))?; + } + if !seq_state.bit_d.is_at_end() { + return Err(ErrorCode::CorruptionDetected); + } + for seq_nb in nb_seq - seq_advance..nb_seq { + self.exec_sequence_maybe_split(sequences.at(seq_nb & STORED_SEQS_MASK)?)?; + } + Ok(()) + } + + /// The literals after the last sequence. + fn last_literals(&mut self) -> Result<()> { + let last_ll_size = self.lit_size - self.lit_pos; + if last_ll_size > self.oend - self.op { + return Err(ErrorCode::DstSizeTooSmall); + } + self.buf + .write_at(self.op, self.lits.span(self.lit_pos, last_ll_size)?)?; + self.op += last_ll_size; + Ok(()) + } +} diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs new file mode 100644 index 000000000000..dabaefe5ca2c --- /dev/null +++ b/src/zstd/decompress/huf.rs @@ -0,0 +1,1023 @@ +//! `huf_decompress.c`: Huffman decoding of literals, one symbol (X1) or up to two symbols (X2) per table lookup. + +use crate::common::bitstream::{BitDStream, DStreamStatus}; +use crate::common::entropy::huf_read_stats; +use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; +use crate::error::{ErrorCode, Result}; +use crate::util::{BUG, SliceExt, highbit32, read_le16, read_le64}; + +/// Tables of this log are decoded by the loops that keep four streams in registers. +const HUF_DECODER_FAST_TABLELOG: u32 = 11; + +const TABLE_SIZE: usize = 1 << HUF_TABLELOG_MAX; +const TABLE_MASK: usize = TABLE_SIZE - 1; +const NB_SYMBOLS: usize = HUF_SYMBOLVALUE_MAX as usize + 1; +const NB_RANKS: usize = HUF_TABLELOG_MAX as usize + 1; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum TableType { + X1, + X2, +} + +/// `HUF_DTable`, of capacity `ZSTD_HUFFDTABLE_CAPACITY_LOG`. +#[derive(Clone)] +pub(crate) struct HufDTable { + table_type: TableType, + table_log: u32, + /// `HUF_DEltX1`: the number of bits in the low byte, and the symbol above it. + x1: [u16; TABLE_SIZE], + /// `HUF_DEltX2`: two symbols, then the number of bits, then how many of the symbols count. + x2: [u32; TABLE_SIZE], +} + +impl HufDTable { + pub(crate) const fn new() -> Self { + Self { + table_type: TableType::X1, + table_log: 0, + x1: [0; TABLE_SIZE], + x2: [0; TABLE_SIZE], + } + } +} + +#[inline(always)] +fn entry_x1(dt: &[u16; TABLE_SIZE], index: u64) -> u16 { + dt.get(index as usize & TABLE_MASK).copied().unwrap_or(0) +} + +#[inline(always)] +fn entry_x2(dt: &[u32; TABLE_SIZE], index: u64) -> u32 { + dt.get(index as usize & TABLE_MASK).copied().unwrap_or(0) +} + +// ── the four streams ────────────────────────────────────────────────────── + +/// Where the four streams of `src` start, after the jump table, and where the last one ends. +struct Streams { + start: [usize; 4], + length: [usize; 4], +} + +impl Streams { + /// `src` has at least 10 bytes. + fn new(src: &[u8]) -> Result { + let length1 = usize::from(read_le16(src, 0)?); + let length2 = usize::from(read_le16(src, 2)?); + let length3 = usize::from(read_le16(src, 4)?); + let length4 = src.len().wrapping_sub(length1 + length2 + length3 + 6); + let start1 = 6; + let start2 = start1 + length1; + let start3 = start2 + length2; + let start4 = start3 + length3; + Ok(Self { + start: [start1, start2, start3, start4], + length: [length1, length2, length3, length4], + }) + } + + fn open<'a>(&self, src: &'a [u8]) -> Result<[BitDStream<'a>; 4]> { + let [s1, s2, s3, s4] = self.start; + let [l1, l2, l3, l4] = self.length; + Ok([ + BitDStream::new(src.span(s1, l1)?)?, + BitDStream::new(src.span(s2, l2)?)?, + BitDStream::new(src.span(s3, l3)?)?, + BitDStream::new(src.span(s4, l4)?)?, + ]) + } +} + +/// `HUF_DecompressFastArgs`, with positions in the input and the output for pointers. +struct FastArgs { + ip: [usize; 4], + op: [usize; 4], + bits: [u64; 4], + /// Where each stream starts. + iend: [usize; 4], +} + +/// `HUF_initFastDStream`: the lowest set bit marks how much of the 64 bits is consumed. +#[inline] +fn init_fast_dstream(src: &[u8], ip: usize) -> Result { + let value = read_le64(src, ip)?; + let last_byte = (value >> 56) as u32; + let bits_consumed = if last_byte != 0 { + 8 - highbit32(last_byte) + } else { + 0 + }; + Ok((value | 1) << bits_consumed) +} + +impl FastArgs { + /// `HUF_DecompressFastArgs_init`. `None` if the input has to be decoded by the other loops. + fn new(dst_size: usize, src: &[u8], table_log: u32) -> Result> { + if dst_size == 0 { + return Ok(None); + } + if src.len() < 10 { + return Err(ErrorCode::CorruptionDetected); + } + if table_log != HUF_DECODER_FAST_TABLELOG { + return Ok(None); + } + let streams = Streams::new(src)?; + if streams.length.iter().any(|&length| length < 8) { + return Ok(None); + } + let [_, _, _, length4] = streams.length; + if length4 > src.len() { + return Err(ErrorCode::CorruptionDetected); + } + let [_, start2, start3, start4] = streams.start; + let ip = [start2 - 8, start3 - 8, start4 - 8, src.len() - 8]; + let segment = dst_size.div_ceil(4); + let op = [0, segment, 2 * segment, 3 * segment]; + if 3 * segment >= dst_size { + return Ok(None); + } + let [ip0, ip1, ip2, ip3] = ip; + Ok(Some(Self { + ip, + op, + bits: [ + init_fast_dstream(src, ip0)?, + init_fast_dstream(src, ip1)?, + init_fast_dstream(src, ip2)?, + init_fast_dstream(src, ip3)?, + ], + iend: streams.start, + })) + } + + /// `HUF_initRemainingDStream` + fn remaining_dstream<'a>( + &self, + src: &'a [u8], + stream: usize, + segment_end: usize, + ) -> Result> { + if self.op.at(stream)? > segment_end { + return Err(ErrorCode::CorruptionDetected); + } + let ip = self.ip.at(stream)?; + if ip + 8 < self.iend.at(stream)? { + return Err(ErrorCode::CorruptionDetected); + } + BitDStream::resume(src, ip, self.bits.at(stream)?.trailing_zeros()) + } + + /// How many rounds of at most 7 bytes the first stream has input for. The others start above it. + #[inline] + fn input_rounds(&self) -> usize { + let [ip0, ..] = self.ip; + ip0 / 7 + } + + #[inline] + fn streams_are_ordered(&self) -> bool { + self.ip.is_sorted() + } +} + +/// What both fast loops do to a stream after a round. +#[inline(always)] +fn reload_fast(src: &[u8], ip: &mut usize, bits: &mut u64) -> Result<()> { + let ctz = bits.trailing_zeros(); + let nb_bits = ctz & 7; + let nb_bytes = (ctz >> 3) as usize; + *ip = ip.wrapping_sub(nb_bytes); + *bits = (read_le64(src, *ip)? | 1) << nb_bits; + Ok(()) +} + +/// Three parts of `segment_size` bytes, and the rest. +fn split_in_4(dst: &mut [u8], segment_size: usize) -> Result<[&mut [u8]; 4]> { + let (part0, rest) = dst.split_at_mut_checked(segment_size).ok_or(BUG)?; + let (part1, rest) = rest.split_at_mut_checked(segment_size).ok_or(BUG)?; + let (part2, part3) = rest.split_at_mut_checked(segment_size).ok_or(BUG)?; + Ok([part0, part1, part2, part3]) +} + +/// The end of each of the four parts of an output of `dst_size` bytes. +fn segment_ends(dst_size: usize) -> [usize; 4] { + let segment = dst_size.div_ceil(4); + [1, 2, 3, 4].map(|n| (n * segment).min(dst_size)) +} + +// ── X1 ──────────────────────────────────────────────────────────────────── + +/// `HUF_rescaleStats` +fn rescale_stats( + huff_weight: &mut [u8], + rank_val: &mut [u32; NB_RANKS], + table_log: u32, + target_table_log: u32, +) -> Result { + if table_log > target_table_log { + return Ok(table_log); + } + if table_log < target_table_log { + let scale = (target_table_log - table_log) as usize; + for weight in huff_weight { + if *weight != 0 { + *weight += scale as u8; + } + } + let target = target_table_log as usize; + rank_val.move_within(1, scale + 1, target - scale)?; + rank_val.span_mut(1, scale)?.fill(0); + } + Ok(target_table_log) +} + +/// `HUF_readDTableX1_wksp`. Returns the size of the table's description in `src`. +fn read_dtable_x1(dtable: &mut HufDTable, src: &[u8]) -> Result { + let mut rank_val = [0u32; NB_RANKS]; + let mut rank_start = [0u32; NB_RANKS]; + let mut symbols = [0u8; NB_SYMBOLS]; + let mut huff_weight = [0u8; NB_SYMBOLS]; + + let stats = huf_read_stats(&mut huff_weight, &mut rank_val, src)?; + let weights = huff_weight.sub_mut(..stats.nb_symbols as usize)?; + + // An X1 table takes half the room of an X2 table, so its log may be one more than the capacity. + let max_table_log = HUF_TABLELOG_MAX + 1; + let target_table_log = max_table_log.min(HUF_DECODER_FAST_TABLELOG); + let table_log = rescale_stats(weights, &mut rank_val, stats.table_log, target_table_log)?; + if table_log > max_table_log { + return Err(ErrorCode::TableLogTooLarge); + } + dtable.table_type = TableType::X1; + dtable.table_log = table_log; + + // Sort the symbols by weight. + let mut next_rank_start = 0u32; + for (start, &count) in rank_start + .iter_mut() + .zip(&rank_val) + .take(table_log as usize + 1) + { + *start = next_rank_start; + next_rank_start += count; + } + for (symbol, &weight) in weights.iter().enumerate() { + let start = rank_start.at_mut(usize::from(weight))?; + symbols.put(*start as usize, symbol as u8)?; + *start += 1; + } + + let [rank0, ..] = rank_val; + let mut symbol = rank0 as usize; + let mut u_start = 0usize; + for w in 1..=table_log { + let symbol_count = rank_val.at(w as usize)? as usize; + let length = (1usize << w) >> 1; + let nb_bits = (table_log + 1 - w) as u16; + for &byte in symbols.span(symbol, symbol_count)? { + dtable + .x1 + .span_mut(u_start, length)? + .fill((u16::from(byte) << 8) | nb_bits); + u_start += length; + } + symbol += symbol_count; + } + Ok(stats.size) +} + +/// `HUF_decodeSymbolX1` +#[inline(always)] +fn decode_symbol_x1(bit_d: &mut BitDStream<'_>, dt: &[u16; TABLE_SIZE], dt_log: u32) -> u8 { + let entry = entry_x1(dt, bit_d.look_bits_fast(dt_log)); + bit_d.skip_bits(u32::from(entry & 0xFF)); + (entry >> 8) as u8 +} + +/// `HUF_decodeStreamX1`: fills `out`. +#[inline] +fn decode_stream_x1( + out: &mut [u8], + bit_d: &mut BitDStream<'_>, + dt: &[u16; TABLE_SIZE], + dt_log: u32, +) { + let end = out.len(); + let mut p = 0usize; + if end > 3 { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 3 < end) { + if let Ok(four) = out.array_at_mut::<4>(p) { + for byte in four { + *byte = decode_symbol_x1(bit_d, dt, dt_log); + } + } + p += 4; + } + } else { + bit_d.reload(); + } + for byte in out.get_mut(p..).unwrap_or_default() { + *byte = decode_symbol_x1(bit_d, dt, dt_log); + } +} + +/// `HUF_decompress1X1_usingDTable_internal_body` +fn decompress_1x1_using_dtable(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { + let mut bit_d = BitDStream::new(src)?; + decode_stream_x1(dst, &mut bit_d, &dtable.x1, dtable.table_log); + if !bit_d.is_at_end() { + return Err(ErrorCode::CorruptionDetected); + } + Ok(dst.len()) +} + +/// `HUF_decompress4X1_usingDTable_internal_body` +fn decompress_4x1_body(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { + if src.len() < 10 { + return Err(ErrorCode::CorruptionDetected); + } + if dst.len() < 6 { + return Err(ErrorCode::CorruptionDetected); + } + let dst_size = dst.len(); + let dt = &dtable.x1; + let dt_log = dtable.table_log; + let streams = Streams::new(src)?; + let [_, _, _, length4] = streams.length; + if length4 > src.len() { + return Err(ErrorCode::CorruptionDetected); + } + let segment_size = dst_size.div_ceil(4); + if 3 * segment_size > dst_size { + return Err(ErrorCode::CorruptionDetected); + } + let [mut bit_d1, mut bit_d2, mut bit_d3, mut bit_d4] = streams.open(src)?; + + let [out1, out2, out3, out4] = split_in_4(dst, segment_size)?; + // The four streams advance together, by `done`, and the last part is the shortest. + let mut done = 0usize; + if out4.len() >= 8 { + let mut end_signal = true; + while end_signal & (done + 3 < out4.len()) { + let four1 = out1.array_at_mut::<4>(done)?; + let four2 = out2.array_at_mut::<4>(done)?; + let four3 = out3.array_at_mut::<4>(done)?; + let four4 = out4.array_at_mut::<4>(done)?; + for (((byte1, byte2), byte3), byte4) in + four1.iter_mut().zip(four2).zip(four3).zip(four4) + { + *byte1 = decode_symbol_x1(&mut bit_d1, dt, dt_log); + *byte2 = decode_symbol_x1(&mut bit_d2, dt, dt_log); + *byte3 = decode_symbol_x1(&mut bit_d3, dt, dt_log); + *byte4 = decode_symbol_x1(&mut bit_d4, dt, dt_log); + } + done += 4; + end_signal &= bit_d1.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d2.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d3.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d4.reload_fast() == DStreamStatus::Unfinished; + } + } + + decode_stream_x1(out1.sub_mut(done..)?, &mut bit_d1, dt, dt_log); + decode_stream_x1(out2.sub_mut(done..)?, &mut bit_d2, dt, dt_log); + decode_stream_x1(out3.sub_mut(done..)?, &mut bit_d3, dt, dt_log); + decode_stream_x1(out4.sub_mut(done..)?, &mut bit_d4, dt, dt_log); + + if !(bit_d1.is_at_end() & bit_d2.is_at_end() & bit_d3.is_at_end() & bit_d4.is_at_end()) { + return Err(ErrorCode::CorruptionDetected); + } + Ok(dst_size) +} + +/// `HUF_decompress4X1_usingDTable_internal_fast_c_loop` +fn fast_loop_4x1( + args: &mut FastArgs, + dst: &mut [u8], + src: &[u8], + dt: &[u16; TABLE_SIZE], +) -> Result<()> { + let [_, segment_size, ..] = args.op; + let [part0, part1, part2, part3] = split_in_4(dst, segment_size)?; + // How much of each part is written. It is the same for all of them. + let mut done = 0usize; + + let [mut ip0, mut ip1, mut ip2, mut ip3] = args.ip; + let [mut bits0, mut bits1, mut bits2, mut bits3] = args.bits; + + #[inline(always)] + fn decode(bits: &mut u64, dt: &[u16; TABLE_SIZE]) -> u8 { + let entry = entry_x1(dt, *bits >> 53); + *bits <<= entry & 0x3F; + (entry >> 8) as u8 + } + + loop { + // Each round writes 5 bytes to each part, of which the last is the shortest, and reads at most 7 of each stream. + let rounds = ((part3.len() - done) / 5).min(ip0 / 7); + if rounds == 0 { + break; + } + if !(ip0 <= ip1 && ip1 <= ip2 && ip2 <= ip3) { + break; + } + let (out0, _) = part0.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + let (out1, _) = part1.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + let (out2, _) = part2.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + let (out3, _) = part3.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + for (((out0, out1), out2), out3) in out0.iter_mut().zip(out1).zip(out2).zip(out3) { + for (((byte0, byte1), byte2), byte3) in out0.iter_mut().zip(out1).zip(out2).zip(out3) { + *byte0 = decode(&mut bits0, dt); + *byte1 = decode(&mut bits1, dt); + *byte2 = decode(&mut bits2, dt); + *byte3 = decode(&mut bits3, dt); + } + reload_fast(src, &mut ip0, &mut bits0)?; + reload_fast(src, &mut ip1, &mut bits1)?; + reload_fast(src, &mut ip2, &mut bits2)?; + reload_fast(src, &mut ip3, &mut bits3)?; + } + done += rounds * 5; + } + + args.ip = [ip0, ip1, ip2, ip3]; + args.bits = [bits0, bits1, bits2, bits3]; + args.op = args.op.map(|start| start + done); + Ok(()) +} + +/// `HUF_decompress4X1_usingDTable_internal_fast`. `None` if the input has to be decoded by the other loop. +fn decompress_4x1_fast(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result> { + let Some(mut args) = FastArgs::new(dst.len(), src, dtable.table_log)? else { + return Ok(None); + }; + fast_loop_4x1(&mut args, dst, src, &dtable.x1)?; + + for (stream, segment_end) in segment_ends(dst.len()).into_iter().enumerate() { + let mut bit_d = args.remaining_dstream(src, stream, segment_end)?; + let out = dst.sub_mut(args.op.at(stream)?..segment_end)?; + decode_stream_x1(out, &mut bit_d, &dtable.x1, HUF_DECODER_FAST_TABLELOG); + } + Ok(Some(dst.len())) +} + +/// `HUF_decompress4X1_usingDTable_internal` +fn decompress_4x1_using_dtable(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { + if let Some(size) = decompress_4x1_fast(dst, src, dtable)? { + return Ok(size); + } + decompress_4x1_body(dst, src, dtable) +} + +// ── X2 ──────────────────────────────────────────────────────────────────── + +type RankValCol = [u32; NB_RANKS]; + +/// `HUF_buildDEltX2U32` +#[inline(always)] +fn build_delt_x2(symbol: u32, nb_bits: u32, base_seq: u32, level: u32) -> u32 { + let seq = if level == 1 { + symbol + } else { + base_seq + (symbol << 8) + }; + seq + (nb_bits << 16) + (level << 24) +} + +/// `HUF_fillDTableX2ForWeight` +fn fill_dtable_x2_for_weight( + dtable_rank: &mut [u32], + symbols: &[u8], + nb_bits: u32, + table_log: u32, + base_seq: u32, + level: u32, +) -> Result<()> { + let length = 1usize << (table_log.wrapping_sub(nb_bits) & 0x1F); + let entries = dtable_rank.sub_mut(..length * symbols.len())?; + for (entries, &symbol) in entries.chunks_exact_mut(length).zip(symbols) { + entries.fill(build_delt_x2(u32::from(symbol), nb_bits, base_seq, level)); + } + Ok(()) +} + +/// `HUF_fillDTableX2Level2` +fn fill_dtable_x2_level2( + dtable: &mut [u32], + target_log: u32, + consumed_bits: u32, + rank_val: &RankValCol, + min_weight: usize, + max_weight1: usize, + sorted_symbols: &[u8; NB_SYMBOLS], + rank_start: &[u32; NB_RANKS + 2], + nb_bits_baseline: u32, + base_seq: u32, +) -> Result<()> { + // The codes that are too long for a second symbol to fit hold the first symbol alone. + if min_weight > 1 { + let length = 1usize << (target_log.wrapping_sub(consumed_bits) & 0x1F); + let skip_size = rank_val.at(min_weight)? as usize; + let filled = match length { + 2 | 4 => length, + _ => skip_size.next_multiple_of(8), + }; + dtable + .sub_mut(..filled)? + .fill(build_delt_x2(base_seq, consumed_bits, 0, 1)); + } + + for w in min_weight..max_weight1 { + let begin = rank_start.at(w)? as usize; + let end = rank_start.at(w + 1)? as usize; + let nb_bits = nb_bits_baseline - w as u32; + let total_bits = nb_bits + consumed_bits; + fill_dtable_x2_for_weight( + dtable.sub_mut(rank_val.at(w)? as usize..)?, + sorted_symbols.sub(begin..end)?, + total_bits, + target_log, + base_seq, + 2, + )?; + } + Ok(()) +} + +/// `HUF_fillDTableX2` +fn fill_dtable_x2( + dtable: &mut [u32; TABLE_SIZE], + target_log: u32, + sorted_list: &[u8; NB_SYMBOLS], + rank_start: &[u32; NB_RANKS + 2], + rank_val_origin: &[RankValCol; HUF_TABLELOG_MAX as usize], + max_weight: u32, + nb_bits_baseline: u32, +) -> Result<()> { + let [rank_val, ..] = rank_val_origin; + let scale_log = nb_bits_baseline as i32 - target_log as i32; + let min_bits = nb_bits_baseline - max_weight; + let w_end = max_weight as usize + 1; + + for w in 1..w_end { + let begin = rank_start.at(w)? as usize; + let end = rank_start.at(w + 1)? as usize; + let nb_bits = nb_bits_baseline - w as u32; + + if target_log.wrapping_sub(nb_bits) >= min_bits { + // Enough room for a second symbol. + let mut start = rank_val.at(w)? as usize; + let length = 1usize << (target_log.wrapping_sub(nb_bits) & 0x1F); + let min_weight = (nb_bits as i32 + scale_log).max(1) as usize; + for &symbol in sorted_list.sub(begin..end)? { + fill_dtable_x2_level2( + dtable.sub_mut(start..)?, + target_log, + nb_bits, + rank_val_origin.get(nb_bits as usize).ok_or(BUG)?, + min_weight, + w_end, + sorted_list, + rank_start, + nb_bits_baseline, + u32::from(symbol), + )?; + start += length; + } + } else { + fill_dtable_x2_for_weight( + dtable.sub_mut(rank_val.at(w)? as usize..)?, + sorted_list.sub(begin..end)?, + nb_bits, + target_log, + 0, + 1, + )?; + } + } + Ok(()) +} + +/// `HUF_readDTableX2_wksp`. Returns the size of the table's description in `src`. +pub(crate) fn read_dtable_x2(dtable: &mut HufDTable, src: &[u8]) -> Result { + let mut rank_val = [[0u32; NB_RANKS]; HUF_TABLELOG_MAX as usize]; + let mut rank_stats = [0u32; NB_RANKS]; + // `rank_start0[w + 1]` is where the symbols of weight `w` start, until sorting moves it to where they end. + let mut rank_start0 = [0u32; NB_RANKS + 2]; + let mut sorted_symbol = [0u8; NB_SYMBOLS]; + let mut weight_list = [0u8; NB_SYMBOLS]; + let mut max_table_log = HUF_TABLELOG_MAX; + + let stats = huf_read_stats(&mut weight_list, &mut rank_stats, src)?; + let table_log = stats.table_log; + + if table_log > max_table_log { + return Err(ErrorCode::TableLogTooLarge); + } + if table_log <= HUF_DECODER_FAST_TABLELOG && max_table_log > HUF_DECODER_FAST_TABLELOG { + max_table_log = HUF_DECODER_FAST_TABLELOG; + } + + let mut max_w = table_log as usize; + while rank_stats.at(max_w)? == 0 { + max_w = max_w.checked_sub(1).ok_or(BUG)?; + } + + let mut next_rank_start = 0u32; + for w in 1..=max_w { + rank_start0.put(w + 1, next_rank_start)?; + next_rank_start += rank_stats.at(w)?; + } + // The symbols of weight 0 go to the end. + rank_start0.put(1, next_rank_start)?; + rank_start0.put(max_w + 2, next_rank_start)?; + + for (symbol, &weight) in weight_list + .iter() + .enumerate() + .take(stats.nb_symbols as usize) + { + let start = rank_start0.at_mut(usize::from(weight) + 1)?; + sorted_symbol.put(*start as usize, symbol as u8)?; + *start += 1; + } + rank_start0.put(1, 0)?; + + { + let rescale = (max_table_log - table_log) as i32 - 1; + let mut next_rank_val = 0u32; + let [rank_val0, ..] = &mut rank_val; + for w in 1..=max_w { + rank_val0.put(w, next_rank_val)?; + next_rank_val += rank_stats.at(w)? << (w as i32 + rescale); + } + let rank_val0 = *rank_val0; + let min_bits = table_log + 1 - max_w as u32; + for consumed in min_bits..(max_table_log - min_bits + 1) { + let rank_val_col = rank_val.get_mut(consumed as usize).ok_or(BUG)?; + for (shifted, &val) in rank_val_col.iter_mut().zip(&rank_val0).skip(1).take(max_w) { + *shifted = val >> consumed; + } + } + } + + fill_dtable_x2( + &mut dtable.x2, + max_table_log, + &sorted_symbol, + &rank_start0, + &rank_val, + max_w as u32, + table_log + 1, + )?; + + dtable.table_log = max_table_log; + dtable.table_type = TableType::X2; + Ok(stats.size) +} + +/// `HUF_decodeSymbolX2`: writes two bytes at `pos` and returns how many of them count. +#[inline(always)] +fn decode_symbol_x2( + out: &mut [u8], + pos: usize, + bit_d: &mut BitDStream<'_>, + dt: &[u32; TABLE_SIZE], + dt_log: u32, +) -> Result { + let entry = entry_x2(dt, bit_d.look_bits_fast(dt_log)); + *out.array_at_mut(pos)? = (entry as u16).to_le_bytes(); + bit_d.skip_bits((entry >> 16) & 0xFF); + Ok((entry >> 24) as usize) +} + +/// `HUF_decodeLastSymbolX2` +#[inline(always)] +fn decode_last_symbol_x2( + out: &mut [u8], + pos: usize, + bit_d: &mut BitDStream<'_>, + dt: &[u32; TABLE_SIZE], + dt_log: u32, +) -> Result<()> { + let entry = entry_x2(dt, bit_d.look_bits_fast(dt_log)); + out.put(pos, entry as u8)?; + let nb_bits = (entry >> 16) & 0xFF; + if entry >> 24 == 1 { + bit_d.skip_bits(nb_bits); + } else if bit_d.bits_consumed < u64::BITS { + bit_d.skip_bits(nb_bits); + // The code of the second symbol, which is not wanted, may be cut off. + if bit_d.bits_consumed > u64::BITS { + bit_d.bits_consumed = u64::BITS; + } + } + Ok(()) +} + +/// `HUF_decodeStreamX2`: fills `out` and returns its length. +#[inline] +fn decode_stream_x2( + out: &mut [u8], + bit_d: &mut BitDStream<'_>, + dt: &[u32; TABLE_SIZE], + dt_log: u32, +) -> Result { + let end = out.len(); + let mut p = 0usize; + + if end >= 8 { + if dt_log <= 11 { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 9 < end) { + for _ in 0..5 { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + } + } else { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 7 < end) { + for _ in 0..4 { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + } + } + } else { + bit_d.reload(); + } + + if end - p >= 2 { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 2 <= end) { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + while p + 2 <= end { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + } + + if p < end { + decode_last_symbol_x2(out, p, bit_d, dt, dt_log)?; + p += 1; + } + Ok(p) +} + +/// `HUF_decompress1X2_usingDTable_internal_body` +fn decompress_1x2_using_dtable(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { + let mut bit_d = BitDStream::new(src)?; + decode_stream_x2(dst, &mut bit_d, &dtable.x2, dtable.table_log)?; + if !bit_d.is_at_end() { + return Err(ErrorCode::CorruptionDetected); + } + Ok(dst.len()) +} + +/// `HUF_decompress4X2_usingDTable_internal_body` +fn decompress_4x2_body(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { + if src.len() < 10 { + return Err(ErrorCode::CorruptionDetected); + } + if dst.len() < 6 { + return Err(ErrorCode::CorruptionDetected); + } + let oend = dst.len(); + let dt = &dtable.x2; + let dt_log = dtable.table_log; + let streams = Streams::new(src)?; + let [_, _, _, length4] = streams.length; + if length4 > src.len() { + return Err(ErrorCode::CorruptionDetected); + } + let segment_size = oend.div_ceil(4); + let op_start2 = segment_size; + let op_start3 = 2 * segment_size; + let op_start4 = 3 * segment_size; + if op_start4 > oend { + return Err(ErrorCode::CorruptionDetected); + } + let [mut bit_d1, mut bit_d2, mut bit_d3, mut bit_d4] = streams.open(src)?; + let (mut op1, mut op2, mut op3, mut op4) = (0, op_start2, op_start3, op_start4); + + // A damaged stream can run over the start of the next part, so the parts are not split here. + if oend - op4 >= 8 { + let mut end_signal = true; + while end_signal & (op4 + 7 < oend) { + for _ in 0..4 { + op1 += decode_symbol_x2(dst, op1, &mut bit_d1, dt, dt_log)?; + } + for _ in 0..4 { + op2 += decode_symbol_x2(dst, op2, &mut bit_d2, dt, dt_log)?; + } + end_signal &= bit_d1.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d2.reload_fast() == DStreamStatus::Unfinished; + for _ in 0..4 { + op3 += decode_symbol_x2(dst, op3, &mut bit_d3, dt, dt_log)?; + } + for _ in 0..4 { + op4 += decode_symbol_x2(dst, op4, &mut bit_d4, dt, dt_log)?; + } + end_signal &= bit_d3.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d4.reload_fast() == DStreamStatus::Unfinished; + } + } + + if op1 > op_start2 || op2 > op_start3 || op3 > op_start4 { + return Err(ErrorCode::CorruptionDetected); + } + + decode_stream_x2(dst.sub_mut(op1..op_start2)?, &mut bit_d1, dt, dt_log)?; + decode_stream_x2(dst.sub_mut(op2..op_start3)?, &mut bit_d2, dt, dt_log)?; + decode_stream_x2(dst.sub_mut(op3..op_start4)?, &mut bit_d3, dt, dt_log)?; + decode_stream_x2(dst.sub_mut(op4..oend)?, &mut bit_d4, dt, dt_log)?; + + if !(bit_d1.is_at_end() & bit_d2.is_at_end() & bit_d3.is_at_end() & bit_d4.is_at_end()) { + return Err(ErrorCode::CorruptionDetected); + } + Ok(oend) +} + +/// `HUF_decompress4X2_usingDTable_internal_fast_c_loop` +fn fast_loop_4x2( + args: &mut FastArgs, + dst: &mut [u8], + src: &[u8], + dt: &[u32; TABLE_SIZE], +) -> Result<()> { + let [_, end0, end1, end2] = args.op; + let oend = [end0, end1, end2, dst.len()]; + + #[inline(always)] + fn decode( + dst: &mut [u8], + op: &mut usize, + bits: &mut u64, + dt: &[u32; TABLE_SIZE], + ) -> Result<()> { + let entry = entry_x2(dt, *bits >> 53); + *dst.array_at_mut(*op)? = (entry as u16).to_le_bytes(); + *bits <<= (entry >> 16) & 0x3F; + *op += (entry >> 24) as usize; + Ok(()) + } + + loop { + // Each round writes at most 10 bytes to each part and reads at most 7 of each stream. + let mut rounds = args.input_rounds(); + for (&end, &op) in oend.iter().zip(&args.op) { + rounds = rounds.min((end - op) / 10); + } + if rounds == 0 { + break; + } + if !args.streams_are_ordered() { + break; + } + let [mut op0, mut op1, mut op2, mut op3] = args.op; + let [mut ip0, mut ip1, mut ip2, mut ip3] = args.ip; + let [mut bits0, mut bits1, mut bits2, mut bits3] = args.bits; + let olimit = op3 + rounds * 5; + loop { + for _ in 0..5 { + decode(dst, &mut op0, &mut bits0, dt)?; + decode(dst, &mut op1, &mut bits1, dt)?; + decode(dst, &mut op2, &mut bits2, dt)?; + decode(dst, &mut op3, &mut bits3, dt)?; + } + reload_fast(src, &mut ip0, &mut bits0)?; + reload_fast(src, &mut ip1, &mut bits1)?; + reload_fast(src, &mut ip2, &mut bits2)?; + reload_fast(src, &mut ip3, &mut bits3)?; + if op3 >= olimit { + break; + } + } + args.op = [op0, op1, op2, op3]; + args.ip = [ip0, ip1, ip2, ip3]; + args.bits = [bits0, bits1, bits2, bits3]; + } + Ok(()) +} + +/// `HUF_decompress4X2_usingDTable_internal_fast`. `None` if the input has to be decoded by the other loop. +fn decompress_4x2_fast(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result> { + let Some(mut args) = FastArgs::new(dst.len(), src, dtable.table_log)? else { + return Ok(None); + }; + fast_loop_4x2(&mut args, dst, src, &dtable.x2)?; + + for (stream, segment_end) in segment_ends(dst.len()).into_iter().enumerate() { + let mut bit_d = args.remaining_dstream(src, stream, segment_end)?; + let out = dst.sub_mut(args.op.at(stream)?..segment_end)?; + decode_stream_x2(out, &mut bit_d, &dtable.x2, HUF_DECODER_FAST_TABLELOG)?; + } + Ok(Some(dst.len())) +} + +/// `HUF_decompress4X2_usingDTable_internal` +fn decompress_4x2_using_dtable(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { + if let Some(size) = decompress_4x2_fast(dst, src, dtable)? { + return Ok(size); + } + decompress_4x2_body(dst, src, dtable) +} + +// ── choosing between X1 and X2 ──────────────────────────────────────────── + +/// `algoTime`: for each sixteenth of compression ratio, the time to build a table and to decode 256 bytes, for X1 and X2. +#[rustfmt::skip] +const ALGO_TIME: [[(u32, u32); 2]; 16] = [ + [( 0, 0), ( 1, 1)], + [( 0, 0), ( 1, 1)], + [( 150, 216), ( 381, 119)], + [( 170, 205), ( 514, 112)], + [( 177, 199), ( 539, 110)], + [( 197, 194), ( 644, 107)], + [( 221, 192), ( 735, 107)], + [( 256, 189), ( 881, 106)], + [( 359, 188), (1167, 109)], + [( 582, 187), (1570, 114)], + [( 688, 187), (1712, 122)], + [( 825, 186), (1965, 136)], + [( 976, 185), (2131, 150)], + [(1180, 186), (2070, 175)], + [(1377, 185), (1731, 202)], + [(1412, 185), (1695, 202)], +]; + +/// `HUF_selectDecoder`: whether X2 is expected to be the faster one. `dst_size` is not 0. +fn select_decoder_x2(dst_size: usize, src_size: usize) -> Result { + let q = if src_size >= dst_size { + 15 + } else { + src_size * 16 / dst_size + }; + let d256 = (dst_size >> 8) as u32; + let [(table0, decode0), (table1, decode1)] = ALGO_TIME.at(q)?; + let dtime0 = table0 + decode0 * d256; + let mut dtime1 = table1 + decode1 * d256; + // A small advantage to the algorithm that uses less memory. + dtime1 += dtime1 >> 5; + Ok(dtime1 < dtime0) +} + +/// `HUF_decompress1X_usingDTable` +pub(crate) fn decompress_1x_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &HufDTable, +) -> Result { + match dtable.table_type { + TableType::X1 => decompress_1x1_using_dtable(dst, src, dtable), + TableType::X2 => decompress_1x2_using_dtable(dst, src, dtable), + } +} + +/// `HUF_decompress1X1_DCtx_wksp` +pub(crate) fn decompress_1x1(dtable: &mut HufDTable, dst: &mut [u8], src: &[u8]) -> Result { + let h_size = read_dtable_x1(dtable, src)?; + if h_size >= src.len() { + return Err(ErrorCode::SrcSizeWrong); + } + decompress_1x1_using_dtable(dst, src.sub(h_size..)?, dtable) +} + +/// `HUF_decompress4X_usingDTable` +pub(crate) fn decompress_4x_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &HufDTable, +) -> Result { + match dtable.table_type { + TableType::X1 => decompress_4x1_using_dtable(dst, src, dtable), + TableType::X2 => decompress_4x2_using_dtable(dst, src, dtable), + } +} + +/// `HUF_decompress4X_hufOnly_wksp` +pub(crate) fn decompress_4x_huf_only( + dtable: &mut HufDTable, + dst: &mut [u8], + src: &[u8], +) -> Result { + if dst.is_empty() { + return Err(ErrorCode::DstSizeTooSmall); + } + if src.is_empty() { + return Err(ErrorCode::CorruptionDetected); + } + let x2 = select_decoder_x2(dst.len(), src.len())?; + let h_size = if x2 { + read_dtable_x2(dtable, src)? + } else { + read_dtable_x1(dtable, src)? + }; + if h_size >= src.len() { + return Err(ErrorCode::SrcSizeWrong); + } + let body = src.sub(h_size..)?; + if x2 { + decompress_4x2_using_dtable(dst, body, dtable) + } else { + decompress_4x1_using_dtable(dst, body, dtable) + } +} diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs new file mode 100644 index 000000000000..68bc85b136f1 --- /dev/null +++ b/src/zstd/decompress/mod.rs @@ -0,0 +1,1285 @@ +//! `zstd_decompress.c` and `zstd_ddict.c`: frames, dictionaries and streaming. + +mod block; +mod huf; +mod seq_table; + +use std::borrow::Cow; + +use block::{ + BlockDecoder, BlockOutput, EntropyDTables, ExtDict, Streaming, TableSource, get_c_block_size, +}; +use seq_table::{LL_BASE, ML_BASE, OF_BASE, OF_BITS}; + +use crate::common::entropy::read_ncount; +use crate::common::xxhash::Xxh64; +use crate::common::{ + BlockType, LL_BITS, LL_FSE_LOG, MAX_LL, MAX_ML, MAX_OFF, ML_BITS, ML_FSE_LOG, OFF_FSE_LOG, + REP_START_VALUE, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_BLOCKSIZE_MAX, + ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_DID_FIELD_SIZE, ZSTD_FCS_FIELD_SIZE, + ZSTD_FRAMEIDSIZE, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGIC_SKIPPABLE_MASK, ZSTD_MAGIC_SKIPPABLE_START, + ZSTD_MAGICNUMBER, ZSTD_SKIPPABLEHEADERSIZE, ZSTD_WINDOWLOG_ABSOLUTEMIN, + ZSTD_WINDOWLOG_LIMIT_DEFAULT, ZSTD_WINDOWLOG_MAX, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, read_le16, read_le32, read_le64, try_copy, try_zeroed}; +use crate::{InBuffer, OutBuffer, ResetDirective}; + +/// `ZSTD_FRAMEHEADERSIZE_PREFIX`: what it takes to know the size of a frame header. +const FRAMEHEADERSIZE_PREFIX: usize = 5; +const FRAMEHEADERSIZE_MIN: usize = 6; +const FRAMEHEADERSIZE_MAX: usize = 18; + +const ZSTD_MAXWINDOWSIZE_DEFAULT: usize = (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1; +const ZSTD_NO_FORWARD_PROGRESS_MAX: u32 = 16; +const ZSTD_WORKSPACETOOLARGE_FACTOR: usize = 3; +const ZSTD_WORKSPACETOOLARGE_MAXDURATION: usize = 128; + +/// `ZSTD_d_windowLogMax`, the only parameter of a [`DCtx`] that `node:zlib` can name. +pub const ZSTD_D_WINDOW_LOG_MAX: i32 = 100; + +// ── frame headers ───────────────────────────────────────────────────────── + +#[derive(Clone, Copy, PartialEq, Eq)] +enum FrameType { + Frame, + Skippable, +} + +/// `ZSTD_FrameHeader` +#[derive(Clone, Copy)] +struct FrameHeader { + /// [`ZSTD_CONTENTSIZE_UNKNOWN`] if the header does not say. For a skippable frame, the size of its content. + frame_content_size: u64, + window_size: u64, + block_size_max: usize, + frame_type: FrameType, + header_size: usize, + dict_id: u32, + checksum_flag: bool, +} + +impl FrameHeader { + const EMPTY: Self = Self { + frame_content_size: 0, + window_size: 0, + block_size_max: 0, + frame_type: FrameType::Frame, + header_size: 0, + dict_id: 0, + checksum_flag: false, + }; +} + +enum HeaderStatus { + Complete(FrameHeader), + /// The header cannot be read from fewer bytes than this. + Needs(usize), +} + +#[inline] +fn is_skippable_magic(magic: u32) -> bool { + magic & ZSTD_MAGIC_SKIPPABLE_MASK == ZSTD_MAGIC_SKIPPABLE_START +} + +/// `ZSTD_frameHeaderSize_internal` +fn frame_header_size(src: &[u8]) -> Result { + if src.len() < FRAMEHEADERSIZE_PREFIX { + return Err(ErrorCode::SrcSizeWrong); + } + let fhd = src.at(FRAMEHEADERSIZE_PREFIX - 1)?; + let dict_id = usize::from(fhd & 3); + let single_segment = (fhd >> 5) & 1 != 0; + let fcs_id = usize::from(fhd >> 6); + Ok(FRAMEHEADERSIZE_PREFIX + + usize::from(!single_segment) + + ZSTD_DID_FIELD_SIZE.at(dict_id)? + + ZSTD_FCS_FIELD_SIZE.at(fcs_id)? + + usize::from(single_segment && fcs_id == 0)) +} + +/// `ZSTD_getFrameHeader` +fn get_frame_header(src: &[u8]) -> Result { + const MIN_INPUT_SIZE: usize = FRAMEHEADERSIZE_PREFIX; + + if src.len() < MIN_INPUT_SIZE { + if !src.is_empty() { + // Refuse what cannot become a magic number. + let known = src.sub(..src.len().min(4))?; + let mut hbuf = ZSTD_MAGICNUMBER.to_le_bytes(); + hbuf.write_at(0, known)?; + if u32::from_le_bytes(hbuf) != ZSTD_MAGICNUMBER { + hbuf = ZSTD_MAGIC_SKIPPABLE_START.to_le_bytes(); + hbuf.write_at(0, known)?; + if !is_skippable_magic(u32::from_le_bytes(hbuf)) { + return Err(ErrorCode::PrefixUnknown); + } + } + } + return Ok(HeaderStatus::Needs(MIN_INPUT_SIZE)); + } + + let magic = read_le32(src, 0)?; + if magic != ZSTD_MAGICNUMBER { + if is_skippable_magic(magic) { + if src.len() < ZSTD_SKIPPABLEHEADERSIZE { + return Ok(HeaderStatus::Needs(ZSTD_SKIPPABLEHEADERSIZE)); + } + return Ok(HeaderStatus::Complete(FrameHeader { + frame_type: FrameType::Skippable, + dict_id: magic - ZSTD_MAGIC_SKIPPABLE_START, + header_size: ZSTD_SKIPPABLEHEADERSIZE, + frame_content_size: u64::from(read_le32(src, ZSTD_FRAMEIDSIZE)?), + ..FrameHeader::EMPTY + })); + } + return Err(ErrorCode::PrefixUnknown); + } + + let fhsize = frame_header_size(src)?; + if src.len() < fhsize { + return Ok(HeaderStatus::Needs(fhsize)); + } + + let fhd_byte = src.at(MIN_INPUT_SIZE - 1)?; + let mut pos = MIN_INPUT_SIZE; + let dict_id_size_code = fhd_byte & 3; + let checksum_flag = (fhd_byte >> 2) & 1 != 0; + let single_segment = (fhd_byte >> 5) & 1 != 0; + let fcs_id = fhd_byte >> 6; + let mut window_size = 0u64; + let mut dict_id = 0u32; + let mut frame_content_size = ZSTD_CONTENTSIZE_UNKNOWN; + if fhd_byte & 0x08 != 0 { + return Err(ErrorCode::FrameParameterUnsupported); + } + + if !single_segment { + let wl_byte = src.at(pos)?; + pos += 1; + let window_log = u32::from(wl_byte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + if window_log > ZSTD_WINDOWLOG_MAX { + return Err(ErrorCode::FrameParameterWindowTooLarge); + } + window_size = 1u64 << window_log; + window_size += (window_size >> 3) * u64::from(wl_byte & 7); + } + match dict_id_size_code { + 1 => { + dict_id = u32::from(src.at(pos)?); + pos += 1; + } + 2 => { + dict_id = u32::from(read_le16(src, pos)?); + pos += 2; + } + 3 => { + dict_id = read_le32(src, pos)?; + pos += 4; + } + _ => {} + } + match fcs_id { + 1 => frame_content_size = u64::from(read_le16(src, pos)?) + 256, + 2 => frame_content_size = u64::from(read_le32(src, pos)?), + 3 => frame_content_size = read_le64(src, pos)?, + _ => { + if single_segment { + frame_content_size = u64::from(src.at(pos)?); + } + } + } + if single_segment { + window_size = frame_content_size; + } + + Ok(HeaderStatus::Complete(FrameHeader { + frame_type: FrameType::Frame, + frame_content_size, + window_size, + block_size_max: window_size.min(ZSTD_BLOCKSIZE_MAX as u64) as usize, + header_size: fhsize, + dict_id, + checksum_flag, + })) +} + +/// `ZSTD_getFrameContentSize`: [`ZSTD_CONTENTSIZE_UNKNOWN`] if the frame at the start of `src` does not say, and +/// [`ZSTD_CONTENTSIZE_ERROR`] if there is no whole frame header. +pub fn get_frame_content_size(src: &[u8]) -> u64 { + match get_frame_header(src) { + Ok(HeaderStatus::Complete(header)) => match header.frame_type { + FrameType::Skippable => 0, + FrameType::Frame => header.frame_content_size, + }, + _ => ZSTD_CONTENTSIZE_ERROR, + } +} + +/// `readSkippableFrameSize` +fn read_skippable_frame_size(src: &[u8]) -> Result { + if src.len() < ZSTD_SKIPPABLEHEADERSIZE { + return Err(ErrorCode::SrcSizeWrong); + } + let size_u32 = read_le32(src, ZSTD_FRAMEIDSIZE)?; + if size_u32 + .checked_add(ZSTD_SKIPPABLEHEADERSIZE as u32) + .is_none() + { + return Err(ErrorCode::FrameParameterUnsupported); + } + let skippable_size = ZSTD_SKIPPABLEHEADERSIZE + size_u32 as usize; + if skippable_size > src.len() { + return Err(ErrorCode::SrcSizeWrong); + } + Ok(skippable_size) +} + +/// `ZSTD_findFrameCompressedSize`: how many bytes the frame at the start of `src` takes. +pub fn find_frame_compressed_size(src: &[u8]) -> Result { + if src.len() >= ZSTD_SKIPPABLEHEADERSIZE && is_skippable_magic(read_le32(src, 0)?) { + return read_skippable_frame_size(src); + } + let header = match get_frame_header(src)? { + HeaderStatus::Complete(header) => header, + HeaderStatus::Needs(_) => return Err(ErrorCode::SrcSizeWrong), + }; + let mut ip = header.header_size; + loop { + let remaining = src.sub(ip..)?; + let (c_block_size, properties) = get_c_block_size(remaining)?; + if ZSTD_BLOCKHEADERSIZE + c_block_size > remaining.len() { + return Err(ErrorCode::SrcSizeWrong); + } + ip += ZSTD_BLOCKHEADERSIZE + c_block_size; + if properties.last_block { + break; + } + } + if header.checksum_flag { + if src.len() - ip < 4 { + return Err(ErrorCode::SrcSizeWrong); + } + ip += 4; + } + Ok(ip) +} + +/// `ZSTD_findDecompressedSize`: the size of all the frames of `src` together, once decompressed. +/// [`ZSTD_CONTENTSIZE_UNKNOWN`] if one of them does not say, [`ZSTD_CONTENTSIZE_ERROR`] if `src` is not whole frames. +pub fn find_decompressed_size(mut src: &[u8]) -> u64 { + let mut total_dst_size = 0u64; + while src.len() >= FRAMEHEADERSIZE_PREFIX { + let frame_src_size = if read_le32(src, 0).is_ok_and(is_skippable_magic) { + read_skippable_frame_size(src) + } else { + let fcs = get_frame_content_size(src); + if fcs >= ZSTD_CONTENTSIZE_ERROR { + return fcs; + } + match total_dst_size.checked_add(fcs) { + Some(total) => total_dst_size = total, + None => return ZSTD_CONTENTSIZE_ERROR, + } + find_frame_compressed_size(src) + }; + match frame_src_size.ok().and_then(|size| src.get(size..)) { + Some(rest) => src = rest, + None => return ZSTD_CONTENTSIZE_ERROR, + } + } + if !src.is_empty() { + return ZSTD_CONTENTSIZE_ERROR; + } + total_dst_size +} + +// ── dictionaries ────────────────────────────────────────────────────────── + +/// `ZSTD_loadDEntropy`: reads the tables at the start of a dictionary, and returns where its content starts. +fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result { + const CORRUPTED: ErrorCode = ErrorCode::DictionaryCorrupted; + if dict.len() <= 8 { + return Err(CORRUPTED); + } + // The magic number and the ID. + let mut pos = 8usize; + + pos += huf::read_dtable_x2(&mut entropy.huf_table, dict.sub(pos..)?).map_err(|_| CORRUPTED)?; + + { + let mut ncount = [0i16; MAX_OFF as usize + 1]; + let header = read_ncount(&mut ncount, MAX_OFF, dict.sub(pos..)?).map_err(|_| CORRUPTED)?; + if header.max_symbol_value > MAX_OFF || header.table_log > OFF_FSE_LOG { + return Err(CORRUPTED); + } + entropy.of_table.build( + ncount.sub(..header.max_symbol_value as usize + 1)?, + &OF_BASE, + &OF_BITS, + header.table_log, + )?; + pos += header.size; + } + { + let mut ncount = [0i16; MAX_ML as usize + 1]; + let header = read_ncount(&mut ncount, MAX_ML, dict.sub(pos..)?).map_err(|_| CORRUPTED)?; + if header.max_symbol_value > MAX_ML || header.table_log > ML_FSE_LOG { + return Err(CORRUPTED); + } + entropy.ml_table.build( + ncount.sub(..header.max_symbol_value as usize + 1)?, + &ML_BASE, + &ML_BITS, + header.table_log, + )?; + pos += header.size; + } + { + let mut ncount = [0i16; MAX_LL as usize + 1]; + let header = read_ncount(&mut ncount, MAX_LL, dict.sub(pos..)?).map_err(|_| CORRUPTED)?; + if header.max_symbol_value > MAX_LL || header.table_log > LL_FSE_LOG { + return Err(CORRUPTED); + } + entropy.ll_table.build( + ncount.sub(..header.max_symbol_value as usize + 1)?, + &LL_BASE, + &LL_BITS, + header.table_log, + )?; + pos += header.size; + } + + if pos + 12 > dict.len() { + return Err(CORRUPTED); + } + let dict_content_size = dict.len() - (pos + 12); + for rep in &mut entropy.rep { + *rep = read_le32(dict, pos)?; + pos += 4; + if *rep == 0 || *rep as usize > dict_content_size { + return Err(CORRUPTED); + } + } + Ok(pos) +} + +/// `ZSTD_dictContentType_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum DictContentType { + /// A dictionary with tables if it starts with the magic number of one, else content to refer to. + Auto, + RawContent, +} + +/// `ZSTD_DDict`: a dictionary that is ready to decompress with. +pub struct DDict<'a> { + /// All of it. Matches can reach into the tables too. + content: Cow<'a, [u8]>, + /// The tables, if the dictionary has some. + entropy: Option>, + dict_id: u32, +} + +impl<'a> DDict<'a> { + /// `ZSTD_initDDict_internal` + fn new(content: Cow<'a, [u8]>, content_type: DictContentType) -> Result { + let mut ddict = Self { + content, + entropy: None, + dict_id: 0, + }; + let dict = &*ddict.content; + if content_type == DictContentType::RawContent + || dict.len() < 8 + || read_le32(dict, 0)? != ZSTD_MAGIC_DICTIONARY + { + return Ok(ddict); + } + ddict.dict_id = read_le32(dict, ZSTD_FRAMEIDSIZE)?; + let mut entropy = Box::new(EntropyDTables::new()); + load_d_entropy(&mut entropy, dict).map_err(|_| ErrorCode::DictionaryCorrupted)?; + ddict.entropy = Some(entropy); + Ok(ddict) + } + + /// `ZSTD_createDDict_byReference` + pub fn by_reference(dict: &'a [u8]) -> Result { + Self::new(Cow::Borrowed(dict), DictContentType::Auto) + } + + /// What `ZSTD_DCtx_refPrefix` makes of `prefix`: content to refer to, whatever it starts with. + pub fn prefix(prefix: &'a [u8]) -> Result { + Self::new(Cow::Borrowed(prefix), DictContentType::RawContent) + } + + /// The address one past the content, which stands for the dictionary where the C code compares pointers. + fn end_address(&self) -> usize { + self.content.as_ptr_range().end.addr() + } +} + +// ── the context ─────────────────────────────────────────────────────────── + +/// `ZSTD_dStage` +#[derive(Clone, Copy, PartialEq, Eq)] +enum DStage { + GetFrameHeaderSize, + DecodeBlockHeader, + DecompressBlock, + DecompressLastBlock, + CheckChecksum, + SkipFrame, +} + +/// `ZSTD_dStreamStage` +#[derive(Clone, Copy, PartialEq, Eq)] +enum DStreamStage { + Init, + LoadHeader, + Read, + Load, + Flush, +} + +/// What precedes `prefix_start`. +#[derive(Clone, Copy)] +enum ExtKind { + /// The dictionary, or nothing. + Dict, + /// See [`ExtDict::Within`]. + Within { end: usize, len: usize }, +} + +/// All of a `ZSTD_DCtx` but the dictionary it owns. +struct DState { + block: BlockDecoder, + + // `prefixStart`, `previousDstEnd`, `virtualStart` and `dictEnd`, as positions in the buffer that is written to. + prefix_start: usize, + /// `None`: nothing of this frame is written yet. + previous_dst_end: Option, + ext: ExtKind, + /// The address `dictEnd` would hold if it points to the end of a dictionary. 0 for `NULL`, 1 for anything else. + dict_end_address: usize, + + expected: usize, + f_params: FrameHeader, + decoded_size: u64, + b_type: BlockType, + stage: DStage, + xxh_state: Xxh64, + validate_checksum: bool, + rle_size: usize, + dict_id: u32, + + stream_stage: DStreamStage, + in_buff: Vec, + in_pos: usize, + max_window_size: usize, + out_buff: Vec, + out_start: usize, + out_end: usize, + lh_size: usize, + /// The last byte of a frame's input is held back until its output is flushed. + hostage_byte: bool, + no_forward_progress: u32, + header_buffer: [u8; FRAMEHEADERSIZE_MAX], + oversized_duration: usize, +} + +/// `ZSTD_DCtx`, which is also `ZSTD_DStream`. +pub struct DCtx { + state: DState, + /// `ddictLocal` + ddict: Option>>, +} + +impl Default for DCtx { + fn default() -> Self { + Self::new() + } +} + +impl DCtx { + /// `ZSTD_createDCtx` + pub fn new() -> Self { + Self { + state: DState { + block: BlockDecoder::new(), + prefix_start: 0, + previous_dst_end: None, + ext: ExtKind::Dict, + dict_end_address: 0, + expected: 0, + f_params: FrameHeader::EMPTY, + decoded_size: 0, + b_type: BlockType::Reserved, + stage: DStage::GetFrameHeaderSize, + xxh_state: Xxh64::new(), + validate_checksum: false, + rle_size: 0, + dict_id: 0, + stream_stage: DStreamStage::Init, + in_buff: Vec::new(), + in_pos: 0, + max_window_size: ZSTD_MAXWINDOWSIZE_DEFAULT, + out_buff: Vec::new(), + out_start: 0, + out_end: 0, + lh_size: 0, + hostage_byte: false, + no_forward_progress: 0, + header_buffer: [0; FRAMEHEADERSIZE_MAX], + oversized_duration: 0, + }, + ddict: None, + } + } + + /// `ZSTD_decompressDCtx`: decompresses all the frames of `src`, which has to end with one, to the start of `dst`. + /// Returns how many bytes that is. + pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + self.state + .decompress_multi_frame(dst, 0, src, self.ddict.as_deref()) + } + + /// `ZSTD_decompress_usingDDict` + pub fn decompress_using_ddict( + &mut self, + dst: &mut [u8], + src: &[u8], + ddict: &DDict<'_>, + ) -> Result { + self.state.decompress_multi_frame(dst, 0, src, Some(ddict)) + } + + /// `ZSTD_DCtx_loadDictionary`: `dict` is copied, and used for every frame from now on. None if it is empty. + pub fn load_dictionary(&mut self, dict: &[u8]) -> Result<()> { + if self.state.stream_stage != DStreamStage::Init { + return Err(ErrorCode::StageWrong); + } + self.ddict = None; + if !dict.is_empty() { + // The C function has nothing else to report a dictionary that it cannot make sense of as. + let ddict = DDict::new(Cow::Owned(try_copy(dict)?), DictContentType::Auto) + .map_err(|_| ErrorCode::MemoryAllocation)?; + self.ddict = Some(Box::new(ddict)); + } + Ok(()) + } + + /// `ZSTD_DCtx_setParameter` + pub fn set_parameter(&mut self, param: i32, mut value: i32) -> Result<()> { + if self.state.stream_stage != DStreamStage::Init { + return Err(ErrorCode::StageWrong); + } + if param != ZSTD_D_WINDOW_LOG_MAX { + return Err(ErrorCode::ParameterUnsupported); + } + if value == 0 { + value = ZSTD_WINDOWLOG_LIMIT_DEFAULT as i32; + } + if value < ZSTD_WINDOWLOG_ABSOLUTEMIN as i32 || value > ZSTD_WINDOWLOG_MAX as i32 { + return Err(ErrorCode::ParameterOutOfBound); + } + self.state.max_window_size = 1usize << value; + Ok(()) + } + + /// `ZSTD_DCtx_reset` + pub fn reset(&mut self, reset: ResetDirective) -> Result<()> { + if matches!( + reset, + ResetDirective::SessionOnly | ResetDirective::SessionAndParameters + ) { + self.state.stream_stage = DStreamStage::Init; + self.state.no_forward_progress = 0; + } + if matches!( + reset, + ResetDirective::Parameters | ResetDirective::SessionAndParameters + ) { + if self.state.stream_stage != DStreamStage::Init { + return Err(ErrorCode::StageWrong); + } + self.ddict = None; + self.state.max_window_size = ZSTD_MAXWINDOWSIZE_DEFAULT; + } + Ok(()) + } + + /// `ZSTD_initDStream`: forgets the frame in progress and the dictionary. + pub fn init_stream(&mut self) { + self.state.stream_stage = DStreamStage::Init; + self.state.no_forward_progress = 0; + self.ddict = None; + } + + /// `ZSTD_decompressStream`: reads from `input` and writes to `output` as much as it can, and moves their positions. + /// Returns 0 when a frame is complete and all of it is written, else a hint of how much input to give next. + pub fn decompress_stream( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + ) -> Result { + self.state + .decompress_stream(output, input, self.ddict.as_deref()) + } +} + +/// `ZSTD_decompress` +pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { + DCtx::new().decompress(dst, src) +} + +/// `ZSTD_copyRawBlock` +fn copy_raw_block(dst: &mut [u8], src: &[u8]) -> Result { + let Some(dst) = dst.get_mut(..src.len()) else { + return Err(ErrorCode::DstSizeTooSmall); + }; + dst.copy_from_slice(src); + Ok(src.len()) +} + +/// `ZSTD_setRleBlock` +fn set_rle_block(dst: &mut [u8], byte: u8, regen_size: usize) -> Result { + let Some(dst) = dst.get_mut(..regen_size) else { + return Err(ErrorCode::DstSizeTooSmall); + }; + dst.fill(byte); + Ok(regen_size) +} + +/// `ZSTD_decodingBufferSize_internal` +fn decoding_buffer_size(window_size: u64, frame_content_size: u64, block_size_max: usize) -> usize { + let block_size = (window_size.min(ZSTD_BLOCKSIZE_MAX as u64) as usize).min(block_size_max); + let needed_rb_size = window_size + (block_size * 2) as u64 + (WILDCOPY_OVERLENGTH * 2) as u64; + frame_content_size.min(needed_rb_size) as usize +} + +impl DState { + /// `ZSTD_decompressBegin_usingDDict` + fn begin_using_ddict(&mut self, ddict: Option<&DDict<'_>>) { + if let Some(ddict) = ddict { + self.block.ddict_is_cold = self.dict_end_address != ddict.end_address(); + } + + self.expected = FRAMEHEADERSIZE_PREFIX; + self.stage = DStage::GetFrameHeaderSize; + self.decoded_size = 0; + self.previous_dst_end = None; + self.prefix_start = 0; + self.ext = ExtKind::Dict; + self.dict_end_address = 0; + self.block.lit_entropy = false; + self.block.fse_entropy = false; + self.dict_id = 0; + self.b_type = BlockType::Reserved; + self.block.entropy.rep = REP_START_VALUE; + self.block.ll_source = TableSource::Local; + self.block.ml_source = TableSource::Local; + self.block.of_source = TableSource::Local; + self.block.huf_source = TableSource::Local; + + // `ZSTD_copyDDictParameters` + if let Some(ddict) = ddict { + self.dict_id = ddict.dict_id; + self.dict_end_address = ddict.end_address(); + if let Some(entropy) = &ddict.entropy { + self.block.lit_entropy = true; + self.block.fse_entropy = true; + self.block.ll_source = TableSource::Dict; + self.block.ml_source = TableSource::Dict; + self.block.of_source = TableSource::Dict; + self.block.huf_source = TableSource::Dict; + self.block.entropy.rep = entropy.rep; + } + } + } + + /// `ZSTD_checkContinuity`: `dst` is where the output continues, with room for `dst_size` bytes. + fn check_continuity(&mut self, dst: usize, dst_size: usize) { + if self.previous_dst_end != Some(dst) && dst_size > 0 { + if let Some(end) = self.previous_dst_end { + self.ext = ExtKind::Within { + end, + len: end - self.prefix_start, + }; + self.dict_end_address = 1; + } + self.prefix_start = dst; + self.previous_dst_end = Some(dst); + } + } + + /// `ZSTD_decodeFrameHeader`: `src` is all of a frame header. + fn decode_frame_header(&mut self, src: &[u8]) -> Result<()> { + match get_frame_header(src)? { + HeaderStatus::Complete(header) => self.f_params = header, + HeaderStatus::Needs(_) => return Err(ErrorCode::SrcSizeWrong), + } + if self.f_params.dict_id != 0 && self.dict_id != self.f_params.dict_id { + return Err(ErrorCode::DictionaryWrong); + } + self.validate_checksum = self.f_params.checksum_flag; + if self.validate_checksum { + self.xxh_state = Xxh64::new(); + } + Ok(()) + } + + /// One compressed block, to `buf` from `start` on. + fn decompress_block( + &mut self, + buf: &mut [u8], + start: usize, + capacity: usize, + src: &[u8], + ddict: Option<&DDict<'_>>, + streaming: Streaming, + ) -> Result { + let ext_dict = match self.ext { + ExtKind::Dict => ExtDict::Dict(ddict.map_or(&[], |ddict| &*ddict.content)), + ExtKind::Within { end, len } => ExtDict::Within { end, len }, + }; + self.block.decompress_block( + BlockOutput { + buf, + prefix_start: self.prefix_start, + start, + capacity, + ext_dict, + }, + src, + ddict.and_then(|ddict| ddict.entropy.as_deref()), + self.f_params.block_size_max, + streaming, + ) + } + + /// `ZSTD_decompressFrame`: one frame, from the start of `src` to `dst` from `ostart` on. + /// Returns how much was written, and how much was read. + fn decompress_frame( + &mut self, + dst: &mut [u8], + ostart: usize, + src: &[u8], + ddict: Option<&DDict<'_>>, + ) -> Result<(usize, usize)> { + let mut ip = 0usize; + let mut op = ostart; + + if src.len() < FRAMEHEADERSIZE_MIN + ZSTD_BLOCKHEADERSIZE { + return Err(ErrorCode::SrcSizeWrong); + } + { + let frame_header_size = frame_header_size(src.sub(..FRAMEHEADERSIZE_PREFIX)?)?; + if src.len() < frame_header_size + ZSTD_BLOCKHEADERSIZE { + return Err(ErrorCode::SrcSizeWrong); + } + self.decode_frame_header(src.sub(..frame_header_size)?)?; + ip += frame_header_size; + } + + loop { + let (c_block_size, properties) = get_c_block_size(src.sub(ip..)?)?; + ip += ZSTD_BLOCKHEADERSIZE; + if c_block_size > src.len() - ip { + return Err(ErrorCode::SrcSizeWrong); + } + let block = src.span(ip, c_block_size)?; + + let decoded_size = match properties.block_type { + BlockType::Compressed => { + let capacity = dst.len() - op; + self.decompress_block(dst, op, capacity, block, ddict, Streaming::No)? + } + BlockType::Raw => copy_raw_block(dst.sub_mut(op..)?, block)?, + BlockType::Rle => set_rle_block( + dst.sub_mut(op..)?, + block.at(0)?, + properties.orig_size as usize, + )?, + BlockType::Reserved => return Err(ErrorCode::CorruptionDetected), + }; + if self.validate_checksum { + self.xxh_state.update(dst.span(op, decoded_size)?); + } + op += decoded_size; + ip += c_block_size; + if properties.last_block { + break; + } + } + + if self.f_params.frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN + && (op - ostart) as u64 != self.f_params.frame_content_size + { + return Err(ErrorCode::CorruptionDetected); + } + if self.f_params.checksum_flag { + if src.len() - ip < 4 { + return Err(ErrorCode::ChecksumWrong); + } + if read_le32(src, ip)? != self.xxh_state.digest() as u32 { + return Err(ErrorCode::ChecksumWrong); + } + ip += 4; + } + Ok((op - ostart, ip)) + } + + /// `ZSTD_decompressMultiFrame`: all the frames of `src`, to `dst` from `dst_start` on. + fn decompress_multi_frame( + &mut self, + dst: &mut [u8], + dst_start: usize, + mut src: &[u8], + ddict: Option<&DDict<'_>>, + ) -> Result { + let mut op = dst_start; + let mut more_than_1_frame = false; + + while src.len() >= FRAMEHEADERSIZE_PREFIX { + if is_skippable_magic(read_le32(src, 0)?) { + src = src.sub(read_skippable_frame_size(src)?..)?; + continue; + } + + self.begin_using_ddict(ddict); + self.check_continuity(op, dst.len() - op); + + match self.decompress_frame(dst, op, src, ddict) { + // More likely `src` is longer than its frames than damaged where a magic number is expected. + Err(ErrorCode::PrefixUnknown) if more_than_1_frame => { + return Err(ErrorCode::SrcSizeWrong); + } + Err(err) => return Err(err), + Ok((written, read)) => { + op += written; + src = src.sub(read..)?; + } + } + more_than_1_frame = true; + } + + if !src.is_empty() { + return Err(ErrorCode::SrcSizeWrong); + } + Ok(op - dst_start) + } + + /// `ZSTD_nextSrcSizeToDecompressWithInputSize` + fn next_src_size_to_decompress_with_input_size(&self, input_size: usize) -> usize { + if !matches!( + self.stage, + DStage::DecompressBlock | DStage::DecompressLastBlock + ) { + return self.expected; + } + if self.b_type != BlockType::Raw { + return self.expected; + } + input_size.min(self.expected).max(1) + } + + /// `ZSTD_decompressContinue`, for the stages that `ZSTD_decompressStream` leaves to it. The output goes to + /// `out_buff` from `out_start` on, where there is room for `dst_capacity` bytes. + fn decompress_continue( + &mut self, + dst_capacity: usize, + src: &[u8], + ddict: Option<&DDict<'_>>, + ) -> Result { + if src.len() != self.next_src_size_to_decompress_with_input_size(src.len()) { + return Err(ErrorCode::SrcSizeWrong); + } + let dst = self.out_start; + self.check_continuity(dst, dst_capacity); + + match self.stage { + DStage::DecodeBlockHeader => { + let (c_block_size, bp) = get_c_block_size(src)?; + if c_block_size > self.f_params.block_size_max { + return Err(ErrorCode::CorruptionDetected); + } + self.expected = c_block_size; + self.b_type = bp.block_type; + self.rle_size = bp.orig_size as usize; + if c_block_size != 0 { + self.stage = if bp.last_block { + DStage::DecompressLastBlock + } else { + DStage::DecompressBlock + }; + return Ok(0); + } + // An empty block. + if bp.last_block { + self.end_of_blocks(); + } else { + self.expected = ZSTD_BLOCKHEADERSIZE; + self.stage = DStage::DecodeBlockHeader; + } + Ok(0) + } + + DStage::DecompressLastBlock | DStage::DecompressBlock => { + let mut out_buff = core::mem::take(&mut self.out_buff); + let r_size = match self.b_type { + BlockType::Compressed => { + self.expected = 0; + self.decompress_block( + &mut out_buff, + dst, + dst_capacity, + src, + ddict, + Streaming::Yes, + ) + } + BlockType::Raw => out_buff + .span_mut(dst, dst_capacity) + .and_then(|dst| copy_raw_block(dst, src)) + .inspect(|&r_size| self.expected -= r_size), + BlockType::Rle => { + self.expected = 0; + out_buff + .span_mut(dst, dst_capacity) + .and_then(|dst| set_rle_block(dst, src.at(0)?, self.rle_size)) + } + BlockType::Reserved => Err(ErrorCode::CorruptionDetected), + }; + self.out_buff = out_buff; + let r_size = r_size?; + if r_size > self.f_params.block_size_max { + return Err(ErrorCode::CorruptionDetected); + } + self.decoded_size += r_size as u64; + if self.validate_checksum { + self.xxh_state.update(self.out_buff.span(dst, r_size)?); + } + self.previous_dst_end = Some(dst + r_size); + + // Stay on the same stage until all of a raw block is in. + if self.expected > 0 { + return Ok(r_size); + } + + if self.stage == DStage::DecompressLastBlock { + if self.f_params.frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN + && self.decoded_size != self.f_params.frame_content_size + { + return Err(ErrorCode::CorruptionDetected); + } + self.end_of_blocks(); + } else { + self.stage = DStage::DecodeBlockHeader; + self.expected = ZSTD_BLOCKHEADERSIZE; + } + Ok(r_size) + } + + DStage::CheckChecksum => { + if self.validate_checksum && read_le32(src, 0)? != self.xxh_state.digest() as u32 { + return Err(ErrorCode::ChecksumWrong); + } + self.expected = 0; + self.stage = DStage::GetFrameHeaderSize; + Ok(0) + } + + DStage::SkipFrame => { + self.expected = 0; + self.stage = DStage::GetFrameHeaderSize; + Ok(0) + } + + // `ZSTD_decompressStream` reads frame headers itself, and is done when it gets back here. + DStage::GetFrameHeaderSize => Err(ErrorCode::Generic), + } + } + + /// After the last block comes the checksum, if there is one. + fn end_of_blocks(&mut self) { + if self.f_params.checksum_flag { + self.expected = 4; + self.stage = DStage::CheckChecksum; + } else { + self.expected = 0; + self.stage = DStage::GetFrameHeaderSize; + } + } + + /// `ZSTD_decompressContinueStream` + fn decompress_continue_stream(&mut self, src: &[u8], ddict: Option<&DDict<'_>>) -> Result<()> { + let is_skip_frame = self.stage == DStage::SkipFrame; + let dst_size = if is_skip_frame { + 0 + } else { + self.out_buff.len() - self.out_start + }; + let decoded_size = self.decompress_continue(dst_size, src, ddict)?; + if decoded_size == 0 && !is_skip_frame { + self.stream_stage = DStreamStage::Read; + } else { + self.out_end = self.out_start + decoded_size; + self.stream_stage = DStreamStage::Flush; + } + Ok(()) + } + + /// `ZSTD_decompressStream` + fn decompress_stream( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + ddict: Option<&DDict<'_>>, + ) -> Result { + if input.pos > input.src.len() { + return Err(ErrorCode::SrcSizeWrong); + } + if output.pos > output.dst.len() { + return Err(ErrorCode::DstSizeTooSmall); + } + let src = input.src; + let istart = input.pos; + let iend = src.len(); + let mut ip = istart; + let ostart = output.pos; + let oend = output.dst.len(); + let mut op = ostart; + + 'work: loop { + if self.stream_stage == DStreamStage::Init { + self.stream_stage = DStreamStage::LoadHeader; + self.lh_size = 0; + self.in_pos = 0; + self.out_start = 0; + self.out_end = 0; + self.hostage_byte = false; + } + + if self.stream_stage == DStreamStage::LoadHeader { + match get_frame_header(self.header_buffer.sub(..self.lh_size)?)? { + HeaderStatus::Complete(header) => self.f_params = header, + HeaderStatus::Needs(h_size) => { + let to_load = h_size - self.lh_size; + let remaining_input = iend - ip; + if to_load > remaining_input { + self.header_buffer.write_at(self.lh_size, src.sub(ip..)?)?; + self.lh_size += remaining_input; + input.pos = iend; + // Refuse what cannot become a frame header. + get_frame_header(self.header_buffer.sub(..self.lh_size)?)?; + return Ok(FRAMEHEADERSIZE_MIN.max(h_size) - self.lh_size + + ZSTD_BLOCKHEADERSIZE); + } + self.header_buffer + .write_at(self.lh_size, src.span(ip, to_load)?)?; + self.lh_size = h_size; + ip += to_load; + continue 'work; + } + } + + // A frame that is all there, with room for all of its output, needs no buffer. + if self.f_params.frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN + && self.f_params.frame_type != FrameType::Skippable + && (oend - op) as u64 >= self.f_params.frame_content_size + { + if let Ok(c_size) = find_frame_compressed_size(src.sub(istart..)?) { + let decompressed_size = self.decompress_multi_frame( + output.dst, + op, + src.span(istart, c_size)?, + ddict, + )?; + ip = istart + c_size; + op += decompressed_size; + self.expected = 0; + self.stream_stage = DStreamStage::Init; + break 'work; + } + } + + self.begin_using_ddict(ddict); + + if is_skippable_magic(read_le32(&self.header_buffer, 0)?) { + self.expected = read_le32(&self.header_buffer, ZSTD_FRAMEIDSIZE)? as usize; + self.stage = DStage::SkipFrame; + } else { + let header_buffer = self.header_buffer; + self.decode_frame_header(header_buffer.sub(..self.lh_size)?)?; + self.expected = ZSTD_BLOCKHEADERSIZE; + self.stage = DStage::DecodeBlockHeader; + } + + self.f_params.window_size = self + .f_params + .window_size + .max(1 << ZSTD_WINDOWLOG_ABSOLUTEMIN); + if self.f_params.window_size > self.max_window_size as u64 { + return Err(ErrorCode::FrameParameterWindowTooLarge); + } + + { + let needed_in_buff_size = self.f_params.block_size_max.max(4); + let needed_out_buff_size = decoding_buffer_size( + self.f_params.window_size, + self.f_params.frame_content_size, + self.f_params.block_size_max, + ); + + // `ZSTD_DCtx_updateOversizedDuration` + if self.in_buff.len() + self.out_buff.len() + >= (needed_in_buff_size + needed_out_buff_size) + * ZSTD_WORKSPACETOOLARGE_FACTOR + { + self.oversized_duration += 1; + } else { + self.oversized_duration = 0; + } + + let too_small = self.in_buff.len() < needed_in_buff_size + || self.out_buff.len() < needed_out_buff_size; + let too_large = self.oversized_duration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; + if too_small || too_large { + self.in_buff = Vec::new(); + self.out_buff = Vec::new(); + self.in_buff = try_zeroed(needed_in_buff_size)?; + self.out_buff = try_zeroed(needed_out_buff_size)?; + } + } + self.stream_stage = DStreamStage::Read; + } + + if self.stream_stage == DStreamStage::Read { + let needed_in_size = self.next_src_size_to_decompress_with_input_size(iend - ip); + if needed_in_size == 0 { + // The end of the frame. + self.stream_stage = DStreamStage::Init; + break 'work; + } + if iend - ip >= needed_in_size { + // Decode straight from the input. + self.decompress_continue_stream(src.span(ip, needed_in_size)?, ddict)?; + ip += needed_in_size; + continue 'work; + } + if ip == iend { + break 'work; + } + self.stream_stage = DStreamStage::Load; + } + + if self.stream_stage == DStreamStage::Load { + let needed_in_size = self.expected; + let to_load = needed_in_size - self.in_pos; + let loaded_size = to_load.min(iend - ip); + if self.stage != DStage::SkipFrame { + if to_load > self.in_buff.len() - self.in_pos { + return Err(ErrorCode::CorruptionDetected); + } + self.in_buff + .write_at(self.in_pos, src.span(ip, loaded_size)?)?; + } + ip += loaded_size; + self.in_pos += loaded_size; + if loaded_size < to_load { + break 'work; + } + + self.in_pos = 0; + let in_buff = core::mem::take(&mut self.in_buff); + // Nothing of a skippable frame is read, or kept. + let loaded = in_buff.get(..needed_in_size).unwrap_or_default(); + let result = if self.stage == DStage::SkipFrame { + self.skip_frame_loaded() + } else { + self.decompress_continue_stream(loaded, ddict) + }; + self.in_buff = in_buff; + result?; + continue 'work; + } + + // `zdss_flush` + let to_flush_size = self.out_end - self.out_start; + let flushed_size = to_flush_size.min(oend - op); + output + .dst + .write_at(op, self.out_buff.span(self.out_start, flushed_size)?)?; + op += flushed_size; + self.out_start += flushed_size; + if flushed_size == to_flush_size { + self.stream_stage = DStreamStage::Read; + if (self.out_buff.len() as u64) < self.f_params.frame_content_size + && self.out_start + self.f_params.block_size_max > self.out_buff.len() + { + self.out_start = 0; + self.out_end = 0; + } + continue 'work; + } + // The output is full. + break 'work; + } + + input.pos = ip; + output.pos = op; + + if ip == istart && op == ostart { + self.no_forward_progress += 1; + if self.no_forward_progress >= ZSTD_NO_FORWARD_PROGRESS_MAX { + if op == oend { + return Err(ErrorCode::NoForwardProgressDestFull); + } + if ip == iend { + return Err(ErrorCode::NoForwardProgressInputEmpty); + } + } + } else { + self.no_forward_progress = 0; + } + + let mut next_src_size_hint = self.expected; + if next_src_size_hint == 0 { + // The frame is decoded. + if self.out_end == self.out_start { + // And flushed. + if self.hostage_byte { + if input.pos >= input.src.len() { + // The caller has to present the byte again for it to be released. + self.stream_stage = DStreamStage::Read; + return Ok(1); + } + input.pos += 1; + } + return Ok(0); + } + if !self.hostage_byte { + // So that the caller comes back to flush, even one that stops when its input is consumed. + input.pos = input.pos.wrapping_sub(1); + self.hostage_byte = true; + } + return Ok(1); + } + if self.stage == DStage::DecompressBlock { + // The header of the next block too. + next_src_size_hint += ZSTD_BLOCKHEADERSIZE; + } + next_src_size_hint -= self.in_pos; + Ok(next_src_size_hint) + } + + /// `ZSTD_decompressContinueStream` for the content of a skippable frame that came in several pieces. + fn skip_frame_loaded(&mut self) -> Result<()> { + self.expected = 0; + self.stage = DStage::GetFrameHeaderSize; + self.out_end = self.out_start; + self.stream_stage = DStreamStage::Flush; + Ok(()) + } +} diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs new file mode 100644 index 000000000000..54c99f97f3a6 --- /dev/null +++ b/src/zstd/decompress/seq_table.rs @@ -0,0 +1,241 @@ +//! The FSE tables that sequences are decoded with: `ZSTD_seqSymbol` and `ZSTD_buildFSETable`. + +use crate::common::{ + LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, ML_DEFAULT_NORM_LOG, + OF_DEFAULT_NORM_LOG, fse_table_step, +}; +use crate::error::Result; +use crate::util::{SliceExt, highbit32, write_le64}; + +#[rustfmt::skip] +pub(crate) const LL_BASE: [u32; MAX_LL as usize + 1] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000, +]; + +#[rustfmt::skip] +pub(crate) const OF_BASE: [u32; MAX_OFF as usize + 1] = [ + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD, +]; + +#[rustfmt::skip] +pub(crate) const OF_BITS: [u8; MAX_OFF as usize + 1] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, +]; + +#[rustfmt::skip] +pub(crate) const ML_BASE: [u32; MAX_ML as usize + 1] = [ + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003, +]; + +/// `ZSTD_seqSymbol` +#[derive(Clone, Copy)] +pub(crate) struct SeqSymbol { + pub(crate) next_state: u16, + pub(crate) nb_additional_bits: u8, + pub(crate) nb_bits: u8, + pub(crate) base_value: u32, +} + +const fn s(next_state: u16, nb_additional_bits: u8, nb_bits: u8, base_value: u32) -> SeqSymbol { + SeqSymbol { + next_state, + nb_additional_bits, + nb_bits, + base_value, + } +} + +/// A table of at most `N` states, which is a power of 2. +#[derive(Clone)] +pub(crate) struct SeqTable { + pub(crate) table_log: u32, + table: [SeqSymbol; N], +} + +impl SeqTable { + pub(crate) const fn new() -> Self { + Self { + table_log: 0, + table: [s(0, 0, 0, 0); N], + } + } + + const fn predefined(table_log: u32, symbols: [SeqSymbol; M]) -> Self { + let mut table = [s(0, 0, 0, 0); N]; + let (head, _) = table.split_at_mut(M); + head.copy_from_slice(&symbols); + Self { table_log, table } + } + + #[inline(always)] + pub(crate) fn get(&self, state: usize) -> SeqSymbol { + self.table + .get(state & (N - 1)) + .copied() + .unwrap_or(s(0, 0, 0, 0)) + } + + /// The states in use. + pub(crate) fn symbols(&self) -> &[SeqSymbol] { + self.table.get(..1 << self.table_log).unwrap_or_default() + } + + /// `ZSTD_buildSeqTable_rle` + pub(crate) fn build_rle(&mut self, base_value: u32, nb_additional_bits: u8) { + self.table_log = 0; + if let Some(cell) = self.table.first_mut() { + *cell = s(0, nb_additional_bits, 0, base_value); + } + } + + /// `ZSTD_buildFSETable`: `table_log` is such that the table fits, and `normalized_counter` adds up to its size. + pub(crate) fn build( + &mut self, + normalized_counter: &[i16], + base_value: &[u32], + nb_additional_bits: &[u8], + table_log: u32, + ) -> Result<()> { + let table_size = 1usize << table_log; + let table_decode = self.table.sub_mut(..table_size)?; + let mut symbol_next = [0u16; MAX_SEQ as usize + 1]; + let mut high_threshold = table_size - 1; + + // Until the last pass, `base_value` holds the symbol. + self.table_log = table_log; + for (symbol, (&count, next)) in normalized_counter.iter().zip(&mut symbol_next).enumerate() + { + if count == -1 { + table_decode.at_mut(high_threshold)?.base_value = symbol as u32; + high_threshold = high_threshold.wrapping_sub(1); + *next = 1; + } else { + *next = count as u16; + } + } + + let table_mask = table_size - 1; + let step = fse_table_step(table_size); + if high_threshold == table_size - 1 { + // No symbol of low probability: lay the symbols down in order, then spread them in a second pass that + // has no branch to mispredict. + let mut spread = [0u8; (1 << 9) + 8]; + let mut pos = 0usize; + let mut sv = 0u64; + for &count in normalized_counter { + let n = count as usize; + for i in (0..n.max(1)).step_by(8) { + write_le64(&mut spread, pos + i, sv)?; + } + pos += n; + sv += 0x0101_0101_0101_0101; + } + let mut position = 0usize; + for &symbol in spread.sub(..table_size)? { + table_decode.at_mut(position)?.base_value = u32::from(symbol); + position = (position + step) & table_mask; + } + } else { + let mut position = 0usize; + for (symbol, &count) in normalized_counter.iter().enumerate() { + for _ in 0..count { + table_decode.at_mut(position)?.base_value = symbol as u32; + position = (position + step) & table_mask; + while position > high_threshold { + position = (position + step) & table_mask; + } + } + } + } + + for cell in table_decode { + let symbol = cell.base_value as usize; + let next = symbol_next.at_mut(symbol)?; + let next_state = u32::from(*next); + *next = next.wrapping_add(1); + let nb_bits = table_log.wrapping_sub(highbit32(next_state)); + *cell = SeqSymbol { + next_state: (next_state.wrapping_shl(nb_bits) as usize).wrapping_sub(table_size) + as u16, + nb_additional_bits: nb_additional_bits.at(symbol)?, + nb_bits: nb_bits as u8, + base_value: base_value.at(symbol)?, + }; + } + Ok(()) + } +} + +#[rustfmt::skip] +const LL_DEFAULT: [SeqSymbol; 64] = [ + s( 0, 0, 4, 0), s(16, 0, 4, 0), s(32, 0, 5, 1), s( 0, 0, 5, 3), + s( 0, 0, 5, 4), s( 0, 0, 5, 6), s( 0, 0, 5, 7), s( 0, 0, 5, 9), + s( 0, 0, 5, 10), s( 0, 0, 5, 12), s( 0, 0, 6, 14), s( 0, 1, 5, 16), + s( 0, 1, 5, 20), s( 0, 1, 5, 22), s( 0, 2, 5, 28), s( 0, 3, 5, 32), + s( 0, 4, 5, 48), s(32, 6, 5, 64), s( 0, 7, 5, 128), s( 0, 8, 6, 256), + s( 0, 10, 6, 1024), s( 0, 12, 6, 4096), s(32, 0, 4, 0), s( 0, 0, 4, 1), + s( 0, 0, 5, 2), s(32, 0, 5, 4), s( 0, 0, 5, 5), s(32, 0, 5, 7), + s( 0, 0, 5, 8), s(32, 0, 5, 10), s( 0, 0, 5, 11), s( 0, 0, 6, 13), + s(32, 1, 5, 16), s( 0, 1, 5, 18), s(32, 1, 5, 22), s( 0, 2, 5, 24), + s(32, 3, 5, 32), s( 0, 3, 5, 40), s( 0, 6, 4, 64), s(16, 6, 4, 64), + s(32, 7, 5, 128), s( 0, 9, 6, 512), s( 0, 11, 6, 2048), s(48, 0, 4, 0), + s(16, 0, 4, 1), s(32, 0, 5, 2), s(32, 0, 5, 3), s(32, 0, 5, 5), + s(32, 0, 5, 6), s(32, 0, 5, 8), s(32, 0, 5, 9), s(32, 0, 5, 11), + s(32, 0, 5, 12), s( 0, 0, 6, 15), s(32, 1, 5, 18), s(32, 1, 5, 20), + s(32, 2, 5, 24), s(32, 2, 5, 28), s(32, 3, 5, 40), s(32, 4, 5, 48), + s( 0, 16, 6, 65536), s( 0, 15, 6, 32768), s( 0, 14, 6, 16384), s( 0, 13, 6, 8192), +]; + +#[rustfmt::skip] +const OF_DEFAULT: [SeqSymbol; 32] = [ + s( 0, 0, 5, 0), s( 0, 6, 4, 61), s( 0, 9, 5, 509), s( 0, 15, 5, 32765), + s( 0, 21, 5, 2097149), s( 0, 3, 5, 5), s( 0, 7, 4, 125), s( 0, 12, 5, 4093), + s( 0, 18, 5, 262141), s( 0, 23, 5, 8388605), s( 0, 5, 5, 29), s( 0, 8, 4, 253), + s( 0, 14, 5, 16381), s( 0, 20, 5, 1048573), s( 0, 2, 5, 1), s(16, 7, 4, 125), + s( 0, 11, 5, 2045), s( 0, 17, 5, 131069), s( 0, 22, 5, 4194301), s( 0, 4, 5, 13), + s(16, 8, 4, 253), s( 0, 13, 5, 8189), s( 0, 19, 5, 524285), s( 0, 1, 5, 1), + s(16, 6, 4, 61), s( 0, 10, 5, 1021), s( 0, 16, 5, 65533), s( 0, 28, 5, 268435453), + s( 0, 27, 5, 134217725), s( 0, 26, 5, 67108861), s( 0, 25, 5, 33554429), s( 0, 24, 5, 16777213), +]; + +#[rustfmt::skip] +const ML_DEFAULT: [SeqSymbol; 64] = [ + s( 0, 0, 6, 3), s( 0, 0, 4, 4), s(32, 0, 5, 5), s( 0, 0, 5, 6), + s( 0, 0, 5, 8), s( 0, 0, 5, 9), s( 0, 0, 5, 11), s( 0, 0, 6, 13), + s( 0, 0, 6, 16), s( 0, 0, 6, 19), s( 0, 0, 6, 22), s( 0, 0, 6, 25), + s( 0, 0, 6, 28), s( 0, 0, 6, 31), s( 0, 0, 6, 34), s( 0, 1, 6, 37), + s( 0, 1, 6, 41), s( 0, 2, 6, 47), s( 0, 3, 6, 59), s( 0, 4, 6, 83), + s( 0, 7, 6, 131), s( 0, 9, 6, 515), s(16, 0, 4, 4), s( 0, 0, 4, 5), + s(32, 0, 5, 6), s( 0, 0, 5, 7), s(32, 0, 5, 9), s( 0, 0, 5, 10), + s( 0, 0, 6, 12), s( 0, 0, 6, 15), s( 0, 0, 6, 18), s( 0, 0, 6, 21), + s( 0, 0, 6, 24), s( 0, 0, 6, 27), s( 0, 0, 6, 30), s( 0, 0, 6, 33), + s( 0, 1, 6, 35), s( 0, 1, 6, 39), s( 0, 2, 6, 43), s( 0, 3, 6, 51), + s( 0, 4, 6, 67), s( 0, 5, 6, 99), s( 0, 8, 6, 259), s(32, 0, 4, 4), + s(48, 0, 4, 4), s(16, 0, 4, 5), s(32, 0, 5, 7), s(32, 0, 5, 8), + s(32, 0, 5, 10), s(32, 0, 5, 11), s( 0, 0, 6, 14), s( 0, 0, 6, 17), + s( 0, 0, 6, 20), s( 0, 0, 6, 23), s( 0, 0, 6, 26), s( 0, 0, 6, 29), + s( 0, 0, 6, 32), s( 0, 16, 6, 65539), s( 0, 15, 6, 32771), s( 0, 14, 6, 16387), + s( 0, 13, 6, 8195), s( 0, 12, 6, 4099), s( 0, 11, 6, 2051), s( 0, 10, 6, 1027), +]; +pub(crate) static LL_DEFAULT_DTABLE: SeqTable<512> = + SeqTable::predefined(LL_DEFAULT_NORM_LOG, LL_DEFAULT); +pub(crate) static OF_DEFAULT_DTABLE: SeqTable<256> = + SeqTable::predefined(OF_DEFAULT_NORM_LOG, OF_DEFAULT); +pub(crate) static ML_DEFAULT_DTABLE: SeqTable<512> = + SeqTable::predefined(ML_DEFAULT_NORM_LOG, ML_DEFAULT); diff --git a/src/zstd/error.rs b/src/zstd/error.rs new file mode 100644 index 000000000000..2992bd86808b --- /dev/null +++ b/src/zstd/error.rs @@ -0,0 +1,122 @@ +//! `zstd_errors.h` and `error_private.c`. + +use core::ffi::CStr; + +/// `ZSTD_ErrorCode`, without `ZSTD_error_no_error`. The values are part of `node:zlib`'s constants. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ErrorCode { + Generic = 1, + PrefixUnknown = 10, + VersionUnsupported = 12, + FrameParameterUnsupported = 14, + FrameParameterWindowTooLarge = 16, + CorruptionDetected = 20, + ChecksumWrong = 22, + LiteralsHeaderWrong = 24, + DictionaryCorrupted = 30, + DictionaryWrong = 32, + DictionaryCreationFailed = 34, + ParameterUnsupported = 40, + ParameterCombinationUnsupported = 41, + ParameterOutOfBound = 42, + TableLogTooLarge = 44, + MaxSymbolValueTooLarge = 46, + MaxSymbolValueTooSmall = 48, + CannotProduceUncompressedBlock = 49, + StabilityConditionNotRespected = 50, + StageWrong = 60, + InitMissing = 62, + MemoryAllocation = 64, + WorkSpaceTooSmall = 66, + DstSizeTooSmall = 70, + SrcSizeWrong = 72, + DstBufferNull = 74, + NoForwardProgressDestFull = 80, + NoForwardProgressInputEmpty = 82, +} + +pub(crate) type Result = core::result::Result; + +impl ErrorCode { + /// `ZSTD_getErrorString`. + pub const fn message(self) -> &'static CStr { + match self { + Self::Generic => c"Error (generic)", + Self::PrefixUnknown => c"Unknown frame descriptor", + Self::VersionUnsupported => c"Version not supported", + Self::FrameParameterUnsupported => c"Unsupported frame parameter", + Self::FrameParameterWindowTooLarge => c"Frame requires too much memory for decoding", + Self::CorruptionDetected => c"Data corruption detected", + Self::ChecksumWrong => c"Restored data doesn't match checksum", + Self::LiteralsHeaderWrong => { + c"Header of Literals' block doesn't respect format specification" + } + Self::ParameterUnsupported => c"Unsupported parameter", + Self::ParameterCombinationUnsupported => c"Unsupported combination of parameters", + Self::ParameterOutOfBound => c"Parameter is out of bound", + Self::InitMissing => c"Context should be init first", + Self::MemoryAllocation => c"Allocation error : not enough memory", + Self::WorkSpaceTooSmall => c"workSpace buffer is not large enough", + Self::StageWrong => c"Operation not authorized at current processing stage", + Self::TableLogTooLarge => c"tableLog requires too much memory : unsupported", + Self::MaxSymbolValueTooLarge => c"Unsupported max Symbol Value : too large", + Self::MaxSymbolValueTooSmall => c"Specified maxSymbolValue is too small", + Self::CannotProduceUncompressedBlock => { + c"This mode cannot generate an uncompressed block" + } + Self::StabilityConditionNotRespected => { + c"pledged buffer stability condition is not respected" + } + Self::DictionaryCorrupted => c"Dictionary is corrupted", + Self::DictionaryWrong => c"Dictionary mismatch", + Self::DictionaryCreationFailed => c"Cannot create Dictionary from provided samples", + Self::DstSizeTooSmall => c"Destination buffer is too small", + Self::SrcSizeWrong => c"Src size is incorrect", + Self::DstBufferNull => c"Operation on NULL destination buffer", + Self::NoForwardProgressDestFull => { + c"Operation made no progress over multiple calls, due to output buffer being full" + } + Self::NoForwardProgressInputEmpty => { + c"Operation made no progress over multiple calls, due to input being empty" + } + } + } + + /// The name of the constant in `zstd_errors.h`, which `node:zlib` reports as the error's `code`. + pub const fn name(self) -> &'static CStr { + match self { + Self::Generic => c"ZSTD_error_GENERIC", + Self::PrefixUnknown => c"ZSTD_error_prefix_unknown", + Self::VersionUnsupported => c"ZSTD_error_version_unsupported", + Self::FrameParameterUnsupported => c"ZSTD_error_frameParameter_unsupported", + Self::FrameParameterWindowTooLarge => c"ZSTD_error_frameParameter_windowTooLarge", + Self::CorruptionDetected => c"ZSTD_error_corruption_detected", + Self::ChecksumWrong => c"ZSTD_error_checksum_wrong", + Self::LiteralsHeaderWrong => c"ZSTD_error_literals_headerWrong", + Self::DictionaryCorrupted => c"ZSTD_error_dictionary_corrupted", + Self::DictionaryWrong => c"ZSTD_error_dictionary_wrong", + Self::DictionaryCreationFailed => c"ZSTD_error_dictionaryCreation_failed", + Self::ParameterUnsupported => c"ZSTD_error_parameter_unsupported", + Self::ParameterCombinationUnsupported => { + c"ZSTD_error_parameter_combination_unsupported" + } + Self::ParameterOutOfBound => c"ZSTD_error_parameter_outOfBound", + Self::TableLogTooLarge => c"ZSTD_error_tableLog_tooLarge", + Self::MaxSymbolValueTooLarge => c"ZSTD_error_maxSymbolValue_tooLarge", + Self::MaxSymbolValueTooSmall => c"ZSTD_error_maxSymbolValue_tooSmall", + // `node:zlib` has no constant for it. + Self::CannotProduceUncompressedBlock => c"ZSTD_error_GENERIC", + Self::StabilityConditionNotRespected => c"ZSTD_error_stabilityCondition_notRespected", + Self::StageWrong => c"ZSTD_error_stage_wrong", + Self::InitMissing => c"ZSTD_error_init_missing", + Self::MemoryAllocation => c"ZSTD_error_memory_allocation", + Self::WorkSpaceTooSmall => c"ZSTD_error_workSpace_tooSmall", + Self::DstSizeTooSmall => c"ZSTD_error_dstSize_tooSmall", + Self::SrcSizeWrong => c"ZSTD_error_srcSize_wrong", + Self::DstBufferNull => c"ZSTD_error_dstBuffer_null", + Self::NoForwardProgressDestFull => c"ZSTD_error_noForwardProgress_destFull", + Self::NoForwardProgressInputEmpty => c"ZSTD_error_noForwardProgress_inputEmpty", + } + } +} diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index b9fd810c9449..9d373f48f62f 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -1,224 +1,73 @@ +//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the +//! same bytes in, in both directions, and the same errors. +#![forbid(unsafe_code)] +// Frames come from anywhere and a panic aborts the process. +#![deny( + clippy::indexing_slicing, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unreachable +)] #![warn(unused_must_use)] -use core::ffi::{c_ulonglong, c_void}; - -use bun_core::ZStr; - -// ─── FFI bindings ───────────────────────────────────────────────────────── -// Externs stay in this crate per PORTING.md §FFI: "If your file has externs -// and isn't already *_sys, leave them in place". -#[allow(non_camel_case_types, non_upper_case_globals)] -pub mod c { - use core::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; - - // `ZSTD_DStream` — opaque streaming-decompression context (Nomicon FFI pattern). - // - // `UnsafeCell` makes the type `!Freeze` so a `&ZSTD_DStream` does not assert - // immutability of the C-owned state (zstd mutates internally on every call). - bun_opaque::opaque_ffi! { - pub struct ZSTD_DStream; - /// `ZSTD_CCtx` — opaque streaming-compression context. - pub struct ZSTD_CCtx; - } - /// `typedef ZSTD_DCtx ZSTD_DStream;` — same opaque object. - pub(crate) type ZSTD_DCtx = ZSTD_DStream; - - // C enums passed by value across FFI — model as `c_uint` (their declared - // underlying type) so callers can pass raw values without transmute. - pub(crate) type ZSTD_ErrorCode = c_uint; - pub(crate) type ZSTD_EndDirective = c_uint; - pub(crate) type ZSTD_ResetDirective = c_uint; - pub(crate) type ZSTD_cParameter = c_uint; - pub(crate) type ZSTD_dParameter = c_uint; - - // ZSTD_EndDirective - pub const ZSTD_e_continue: ZSTD_EndDirective = 0; - - // ZSTD_cParameter - pub const ZSTD_c_compressionLevel: ZSTD_cParameter = 100; - - pub const ZSTD_reset_session_and_parameters: ZSTD_ResetDirective = 3; - - // ZSTD_ErrorCode (zstd_errors.h) — only the public stable subset. - pub const ZSTD_error_no_error: ZSTD_ErrorCode = 0; - pub const ZSTD_error_GENERIC: ZSTD_ErrorCode = 1; - pub const ZSTD_error_prefix_unknown: ZSTD_ErrorCode = 10; - pub const ZSTD_error_version_unsupported: ZSTD_ErrorCode = 12; - pub const ZSTD_error_frameParameter_unsupported: ZSTD_ErrorCode = 14; - pub const ZSTD_error_frameParameter_windowTooLarge: ZSTD_ErrorCode = 16; - pub const ZSTD_error_corruption_detected: ZSTD_ErrorCode = 20; - pub const ZSTD_error_checksum_wrong: ZSTD_ErrorCode = 22; - pub const ZSTD_error_literals_headerWrong: ZSTD_ErrorCode = 24; - pub const ZSTD_error_dictionary_corrupted: ZSTD_ErrorCode = 30; - pub const ZSTD_error_dictionary_wrong: ZSTD_ErrorCode = 32; - pub const ZSTD_error_dictionaryCreation_failed: ZSTD_ErrorCode = 34; - pub const ZSTD_error_parameter_unsupported: ZSTD_ErrorCode = 40; - pub const ZSTD_error_parameter_combination_unsupported: ZSTD_ErrorCode = 41; - pub const ZSTD_error_parameter_outOfBound: ZSTD_ErrorCode = 42; - pub const ZSTD_error_tableLog_tooLarge: ZSTD_ErrorCode = 44; - pub const ZSTD_error_maxSymbolValue_tooLarge: ZSTD_ErrorCode = 46; - pub const ZSTD_error_maxSymbolValue_tooSmall: ZSTD_ErrorCode = 48; - pub const ZSTD_error_stabilityCondition_notRespected: ZSTD_ErrorCode = 50; - pub const ZSTD_error_stage_wrong: ZSTD_ErrorCode = 60; - pub const ZSTD_error_init_missing: ZSTD_ErrorCode = 62; - pub const ZSTD_error_memory_allocation: ZSTD_ErrorCode = 64; - pub const ZSTD_error_workSpace_tooSmall: ZSTD_ErrorCode = 66; - pub const ZSTD_error_dstSize_tooSmall: ZSTD_ErrorCode = 70; - pub const ZSTD_error_srcSize_wrong: ZSTD_ErrorCode = 72; - pub const ZSTD_error_dstBuffer_null: ZSTD_ErrorCode = 74; - pub const ZSTD_error_noForwardProgress_destFull: ZSTD_ErrorCode = 80; - pub const ZSTD_error_noForwardProgress_inputEmpty: ZSTD_ErrorCode = 82; - - #[repr(C)] - pub struct ZSTD_inBuffer { - pub src: *const c_void, - pub size: usize, - pub pos: usize, - } +mod common; +mod decompress; +mod error; +mod util; + +pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; +pub use decompress::{ + DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, + find_frame_compressed_size, get_frame_content_size, +}; +pub use error::ErrorCode; + +/// `ZSTD_inBuffer` +pub struct InBuffer<'a> { + pub src: &'a [u8], + /// How much of `src` is read. + pub pos: usize, +} - #[repr(C)] - pub struct ZSTD_outBuffer { - pub dst: *mut c_void, - pub size: usize, - pub pos: usize, - } +/// `ZSTD_outBuffer` +pub struct OutBuffer<'a> { + pub dst: &'a mut [u8], + /// How much of `dst` is written. + pub pos: usize, +} - unsafe extern "C" { - pub(crate) fn ZSTD_compress( - dst: *mut c_void, - dst_capacity: usize, - src: *const c_void, - src_size: usize, - compression_level: c_int, - ) -> usize; - pub(crate) safe fn ZSTD_compressBound(src_size: usize) -> usize; - pub(crate) fn ZSTD_decompress( - dst: *mut c_void, - dst_capacity: usize, - src: *const c_void, - compressed_size: usize, - ) -> usize; - // Pure scalar fns — no preconditions. - pub safe fn ZSTD_isError(code: usize) -> c_uint; - pub(crate) safe fn ZSTD_getErrorName(code: usize) -> *const c_char; - pub(crate) safe fn ZSTD_defaultCLevel() -> c_int; - - pub(crate) safe fn ZSTD_createDStream() -> *mut ZSTD_DStream; - pub(crate) fn ZSTD_freeDStream(zds: *mut ZSTD_DStream) -> usize; - pub(crate) fn ZSTD_initDStream(zds: *mut ZSTD_DStream) -> usize; - pub fn ZSTD_decompressStream( - zds: *mut ZSTD_DStream, - output: *mut ZSTD_outBuffer, - input: *mut ZSTD_inBuffer, - ) -> usize; - - // ZSTD_findDecompressedSize() : - // `src` should point to the start of a series of ZSTD encoded and/or skippable frames - // `srcSize` must be the _exact_ size of this series - // (i.e. there should be a frame boundary at `src + srcSize`) - // @return : - decompressed size of all data in all successive frames - // - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN - // - if an error occurred: ZSTD_CONTENTSIZE_ERROR - // - // note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. - // When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - // In which case, it's necessary to use streaming mode to decompress data. - // note 2 : decompressed size is always present when compression is done with ZSTD_compress() - // note 3 : decompressed size can be very large (64-bits value), - // potentially larger than what local system can handle as a single memory segment. - // In which case, it's necessary to use streaming mode to decompress data. - // note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. - // Always ensure result fits within application's authorized limits. - // Each application can set its own limits. - // note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to - // read each contained frame header. This is fast as most of the data is skipped, - // however it does mean that all frame data must be present and valid. */ - pub(crate) fn ZSTD_findDecompressedSize(src: *const c_void, src_size: usize) - -> c_ulonglong; - - // ── streaming-compress / advanced API (used by NativeZstd) ─────── - pub safe fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; - pub fn ZSTD_freeCCtx(cctx: *mut ZSTD_CCtx) -> usize; - pub safe fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; - pub fn ZSTD_freeDCtx(dctx: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_CCtx_setPledgedSrcSize( - cctx: *mut ZSTD_CCtx, - pledged_src_size: c_ulonglong, - ) -> usize; - /// Copies the dictionary into the context, so `dict` need not outlive - /// the call (unlike the `_byReference` variant). - pub fn ZSTD_CCtx_loadDictionary( - cctx: *mut ZSTD_CCtx, - dict: *const c_void, - dict_size: usize, - ) -> usize; - /// Copies the dictionary into the context; see `ZSTD_CCtx_loadDictionary`. - pub fn ZSTD_DCtx_loadDictionary( - dctx: *mut ZSTD_DCtx, - dict: *const c_void, - dict_size: usize, - ) -> usize; - /// References `prefix` as the raw content dictionary of the next frame only. It is not - /// copied, so it has to outlive the decompression of that frame. - pub(crate) fn ZSTD_DCtx_refPrefix( - dctx: *mut ZSTD_DCtx, - prefix: *const c_void, - prefix_size: usize, - ) -> usize; - pub(crate) fn ZSTD_decompressDCtx( - dctx: *mut ZSTD_DCtx, - dst: *mut c_void, - dst_capacity: usize, - src: *const c_void, - src_size: usize, - ) -> usize; - pub fn ZSTD_CCtx_setParameter( - cctx: *mut ZSTD_CCtx, - param: ZSTD_cParameter, - value: c_int, - ) -> usize; - pub fn ZSTD_DCtx_setParameter( - dctx: *mut ZSTD_DCtx, - param: ZSTD_dParameter, - value: c_int, - ) -> usize; - pub fn ZSTD_DCtx_reset(dctx: *mut ZSTD_DCtx, reset: ZSTD_ResetDirective) -> usize; - pub fn ZSTD_compressStream2( - cctx: *mut ZSTD_CCtx, - output: *mut ZSTD_outBuffer, - input: *mut ZSTD_inBuffer, - end_op: ZSTD_EndDirective, - ) -> usize; - pub safe fn ZSTD_getErrorCode(function_result: usize) -> ZSTD_ErrorCode; - pub safe fn ZSTD_getErrorString(code: ZSTD_ErrorCode) -> *const c_char; - } +/// `ZSTD_ResetDirective` +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ResetDirective { + SessionOnly = 1, + Parameters = 2, + SessionAndParameters = 3, } // ----------------------------------- -pub enum Result { - Success(usize), - // Always assigned from ZSTD_getErrorName (static C string). - Err(&'static ZStr), -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] pub enum ZstdError { /// The output, or the decoder state the frame's window size dictates, could not be allocated. OutOfMemory, InvalidZstdData, DecompressionFailed, - ZstdFailedToCreateInstance, ZstdDecompressionError, ShortRead, } -bun_core::impl_tag_error!(ZstdError); +impl core::fmt::Display for ZstdError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(<&'static str>::from(self)) + } +} +impl core::error::Error for ZstdError {} impl ZstdError { - /// The error for a failed (`ZSTD_isError`) decompression call; `other` is the non-allocation failure. - fn for_decompression(rc: usize, other: ZstdError) -> ZstdError { - if c::ZSTD_getErrorCode(rc) == c::ZSTD_error_memory_allocation { + /// The error for a failed decompression call; `other` is the non-allocation failure. + fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { + if code == ErrorCode::MemoryAllocation { ZstdError::OutOfMemory } else { other @@ -226,108 +75,18 @@ impl ZstdError { } } -/// ZSTD_compress() : -/// Compresses `src` content as a single zstd compressed frame into already allocated `dst`. -/// NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have -/// enough space to successfully compress the data. -/// @return : compressed size written into `dst` (<= `dstCapacity), -/// or an error code if it fails (which can be tested using ZSTD_isError()). */ -// ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, -// const void* src, size_t srcSize, -// int compressionLevel); -pub fn compress(dest: &mut [u8], src: &[u8], level: Option) -> Result { - // SAFETY: dest/src are valid for their lengths; ZSTD_compress reads src and writes dest. - let result = unsafe { - c::ZSTD_compress( - dest.as_mut_ptr().cast::(), - dest.len(), - src.as_ptr().cast::(), - src.len(), - // Not redundant_closure: extern "C" fn items don't implement FnOnce - // (Fn* traits are only blanket-impl'd for the Rust ABI). - level.unwrap_or_else(|| c::ZSTD_defaultCLevel()), - ) - }; - if c::ZSTD_isError(result) != 0 { - // SAFETY: ZSTD_getErrorName returns a static NUL-terminated string. - return Result::Err(unsafe { ZStr::from_c_ptr(c::ZSTD_getErrorName(result)) }); - } - Result::Success(result) -} - -pub fn compress_bound(src_size: usize) -> usize { - c::ZSTD_compressBound(src_size) -} - -/// [`compress`] into `out`'s spare capacity (append mode). On success -/// `out.len()` is advanced by the compressed size. Callers must ensure -/// `out.spare_capacity_mut().len() >= compress_bound(src.len())` to -/// guarantee success. -pub fn compress_append(out: &mut Vec, src: &[u8], level: Option) -> Result { - let spare = out.spare_capacity_mut(); - // SAFETY: spare/src are valid for their lengths; ZSTD_compress reads src - // and writes into spare. - let rc = unsafe { - c::ZSTD_compress( - spare.as_mut_ptr().cast::(), - spare.len(), - src.as_ptr().cast::(), - src.len(), - level.unwrap_or_else(|| c::ZSTD_defaultCLevel()), - ) - }; - if c::ZSTD_isError(rc) != 0 { - // SAFETY: ZSTD_getErrorName returns a static NUL-terminated string. - return Result::Err(unsafe { ZStr::from_c_ptr(c::ZSTD_getErrorName(rc)) }); - } - // SAFETY: zstd has initialized `rc` bytes at the start of spare. - unsafe { bun_core::vec::commit_spare(out, rc) }; - Result::Success(rc) -} - -/// `ZSTD_isError` — true when a `size_t` return value (including from -/// [`compress_bound`], see zstd.h: "ZSTD_compressBound() itself can fail, if -/// `srcSize >= ZSTD_MAX_INPUT_SIZE`") is an error code rather than a size. -/// Error codes are near `usize::MAX`, so treating one as a size requests an -/// absurd allocation. -pub fn is_error(code: usize) -> bool { - c::ZSTD_isError(code) != 0 -} - -/// `ZSTD_decompress` into `out`'s spare capacity, which is the output bound; commits the bytes written. -fn decompress_append(out: &mut Vec, src: &[u8]) -> core::result::Result<(), ZstdError> { - let spare = out.spare_capacity_mut(); - // SAFETY: spare/src are valid for their lengths; ZSTD_decompress reads src - // and writes at most `spare.len()` bytes into spare. - let rc = unsafe { - c::ZSTD_decompress( - spare.as_mut_ptr().cast::(), - spare.len(), - src.as_ptr().cast::(), - src.len(), - ) - }; - if c::ZSTD_isError(rc) != 0 { - return Err(ZstdError::for_decompression( - rc, - ZstdError::DecompressionFailed, - )); - } - // SAFETY: zstd has initialized `rc` bytes at the start of spare. - unsafe { bun_core::vec::commit_spare(out, rc) }; - Ok(()) +/// `len` zeros for a decoder to write to. +fn output_buffer(len: usize) -> Result, ZstdError> { + util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) } /// Decompress data, automatically allocating the output buffer. -/// Returns owned slice that must be freed by the caller. /// Handles both frames with known and unknown content sizes. /// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. /// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. -pub fn decompress_alloc(src: &[u8]) -> core::result::Result, ZstdError> { - let size = get_decompressed_size(src); +pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { + let size = find_decompressed_size(src); - const ZSTD_CONTENTSIZE_UNKNOWN: usize = c_ulonglong::MAX as usize; // 0ULL - 1 - const ZSTD_CONTENTSIZE_ERROR: usize = (c_ulonglong::MAX - 1) as usize; // 0ULL - 2 const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit if size == ZSTD_CONTENTSIZE_ERROR { @@ -337,7 +96,7 @@ pub fn decompress_alloc(src: &[u8]) -> core::result::Result, ZstdError> // Use streaming decompression if: // 1. Content size is unknown, OR // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) - if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE { + if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { // A frame's output is rarely smaller than its input. src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) @@ -348,288 +107,72 @@ pub fn decompress_alloc(src: &[u8]) -> core::result::Result, ZstdError> let mut list: Vec = Vec::new(); list.try_reserve_exact(initial_capacity) .map_err(|_| ZstdError::OutOfMemory)?; - let mut reader = ZstdReaderArrayList::init(src, &mut list)?; - - reader.read_all(true)?; - drop(reader); + StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; return Ok(list); } // Fast path: size is known and within reasonable limits - let mut output: Vec = Vec::new(); - output - .try_reserve_exact(size) - .map_err(|_| ZstdError::OutOfMemory)?; - - decompress_append(&mut output, src)?; + let mut output = output_buffer(size as usize)?; + let written = decompress(&mut output, src) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); Ok(output) } /// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is /// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. -pub fn decompress_alloc_with_prefix( - src: &[u8], - prefix: &[u8], -) -> core::result::Result, ZstdError> { - const MAX_SIZE: usize = 16 * 1024 * 1024; - let size = get_decompressed_size(src); +pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { + const MAX_SIZE: u64 = 16 * 1024 * 1024; + let size = find_decompressed_size(src); // Also if it is unknown or invalid, which are the two largest values. if size > MAX_SIZE { return Err(ZstdError::InvalidZstdData); } - let mut output: Vec = Vec::new(); - output - .try_reserve_exact(size) - .map_err(|_| ZstdError::OutOfMemory)?; - let dctx = c::ZSTD_createDCtx(); - if dctx.is_null() { - return Err(ZstdError::OutOfMemory); - } - let spare = output.spare_capacity_mut(); - // SAFETY: `dctx` is live until it is freed below. `prefix`, `src` and `spare` are valid for - // their lengths and outlive the calls, and at most `spare.len()` bytes are written to `spare`. - let rc = unsafe { - let rc = c::ZSTD_DCtx_refPrefix(dctx, prefix.as_ptr().cast::(), prefix.len()); - let rc = if c::ZSTD_isError(rc) != 0 { - rc - } else { - c::ZSTD_decompressDCtx( - dctx, - spare.as_mut_ptr().cast::(), - spare.len(), - src.as_ptr().cast::(), - src.len(), - ) - }; - c::ZSTD_freeDCtx(dctx); - rc - }; - if c::ZSTD_isError(rc) != 0 { - return Err(ZstdError::for_decompression( - rc, - ZstdError::DecompressionFailed, - )); - } - // SAFETY: zstd has initialized `rc` bytes at the start of spare. - unsafe { bun_core::vec::commit_spare(&mut output, rc) }; + let mut output = output_buffer(size as usize)?; + let written = DDict::prefix(prefix) + .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); Ok(output) } -pub fn get_decompressed_size(src: &[u8]) -> usize { - // SAFETY: src is valid for src.len() bytes. - unsafe { c::ZSTD_findDecompressedSize(src.as_ptr().cast::(), src.len()) as usize } +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + Uninitialized, + Inflating, + End, + Error, } -pub use bun_core::compress::State; - -/// Minimum spare output capacity offered to `ZSTD_decompressStream` per call. +/// Minimum spare output capacity offered to the decoder per call. const STREAMING_OUTPUT_STEP: usize = 4096; -struct ZstdReaderArrayList<'a> { - pub(crate) input: &'a [u8], - // We operate on the caller's Vec directly via the `&mut` borrow. - pub(crate) list_ptr: &'a mut Vec, - // `list_allocator` / `allocator` params deleted — global mimalloc. - pub(crate) zstd: *mut c::ZSTD_DStream, - pub(crate) state: State, - pub(crate) total_in: usize, - /// Decompression-bomb guard: `read_all` errors instead of growing the - /// output past this many bytes. Defaults to unbounded. - pub(crate) max_output_size: usize, -} - -impl<'a> ZstdReaderArrayList<'a> { - // `pub const new = bun.TrivialNew(...)` → Box::new; no associated const needed. - - pub(crate) fn init( - input: &'a [u8], - list: &'a mut Vec, - ) -> core::result::Result>, ZstdError> { - Self::init_with_list_allocator(input, list) - } - - pub(crate) fn init_with_list_allocator( - input: &'a [u8], - list: &'a mut Vec, - // list_allocator / allocator params deleted (global mimalloc). - ) -> core::result::Result>, ZstdError> { - let zstd = c::ZSTD_createDStream(); - if zstd.is_null() { - return Err(ZstdError::ZstdFailedToCreateInstance); - } - // SAFETY: zstd is a freshly created non-null DStream. - let _ = unsafe { c::ZSTD_initDStream(zstd) }; - - Ok(Box::new(ZstdReaderArrayList { - input, - list_ptr: list, - zstd, - state: State::Uninitialized, - total_in: 0, - max_output_size: usize::MAX, - })) - } - - pub(crate) fn end(&mut self) { - if self.state != State::End { - // SAFETY: self.zstd was created by ZSTD_createDStream and has not been freed - // (guarded by state != End). - let _ = unsafe { c::ZSTD_freeDStream(self.zstd) }; - self.state = State::End; - } - } - - pub(crate) fn read_all(&mut self, is_done: bool) -> core::result::Result<(), ZstdError> { - if self.state == State::End || self.state == State::Error { - return Ok(()); - } - - while self.state == State::Uninitialized || self.state == State::Inflating { - let next_in = &self.input[self.total_in..]; - - // If we have no input to process - if next_in.is_empty() { - if is_done { - // If we're in the middle of inflating and stream is done, it's truncated - if self.state == State::Inflating { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - // No more input and stream is done, we can end - self.end(); - } - return Ok(()); - } - - // Decompression-bomb guard: clamp the output space handed to a single - // ZSTD_decompressStream call so one call can never write past the cap. - let remaining_output = self.max_output_size.saturating_sub(self.list_ptr.len()); - if remaining_output == 0 { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - - if self.list_ptr.try_reserve(STREAMING_OUTPUT_STEP).is_err() { - self.state = State::Error; - return Err(ZstdError::OutOfMemory); - } - let spare = self.list_ptr.spare_capacity_mut(); - let mut in_buf = c::ZSTD_inBuffer { - src: next_in.as_ptr().cast::(), - size: next_in.len(), - pos: 0, - }; - let mut out_buf = c::ZSTD_outBuffer { - dst: spare.as_mut_ptr().cast::(), - size: spare.len().min(remaining_output), - pos: 0, - }; - - // SAFETY: self.zstd is a valid DStream (state != End); in_buf/out_buf point - // into live slices with correct sizes. - let rc = - unsafe { c::ZSTD_decompressStream(self.zstd, &raw mut out_buf, &raw mut in_buf) }; - if c::ZSTD_isError(rc) != 0 { - self.state = State::Error; - return Err(ZstdError::for_decompression( - rc, - ZstdError::ZstdDecompressionError, - )); - } - - let bytes_written = out_buf.pos; - let bytes_read = in_buf.pos; - // SAFETY: ZSTD_decompressStream wrote exactly `bytes_written` initialized bytes - // into the spare capacity starting at the previous len. - unsafe { bun_core::vec::commit_spare(self.list_ptr, bytes_written) }; - self.total_in += bytes_read; - - if rc == 0 { - // Frame is complete - self.state = State::Uninitialized; // Reset state since frame is complete - - // Check if there's more input (multiple frames) - if self.total_in >= self.input.len() { - // We've consumed all available input - if is_done { - // No more data coming, we can end the stream - self.end(); - return Ok(()); - } - // Frame is complete and no more input available right now. - // Just return normally - the caller can provide more data later if they have it. - return Ok(()); - } - // More input available, reset for the next frame - // ZSTD_initDStream() safely resets the stream state without needing cleanup - // It's designed to be called multiple times on the same DStream object - // SAFETY: self.zstd is a valid DStream. - let _ = unsafe { c::ZSTD_initDStream(self.zstd) }; - continue; - } - - // If rc > 0, decompressor needs more data - if rc > 0 { - self.state = State::Inflating; - } - - if bytes_read == next_in.len() { - // We've consumed all available input - if bytes_written > 0 { - // We wrote some output, continue to see if we need more output space - continue; - } - - if is_done { - // Stream is truncated - we're at EOF but need more data - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - // Not at EOF - we can retry with more data - return Err(ZstdError::ShortRead); - } - } - Ok(()) - } -} - -impl Drop for ZstdReaderArrayList<'_> { - fn drop(&mut self) { - self.end(); - } -} - // ────────────────────────────────────────────────────────────────────────── // StreamingDecoder // ────────────────────────────────────────────────────────────────────────── -/// Streaming zstd decoder that owns only the `ZSTD_DStream`. Unlike -/// [`ZstdReaderArrayList`] it stores no `&'a [u8]` / `&'a mut Vec` -/// borrows — input and output are passed to [`decompress`](Self::decompress) -/// per call, so callers can hold the decoder across multiple body chunks -/// without lifetime erasure. +/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) +/// per call, so callers can hold the decoder across multiple body chunks. pub struct StreamingDecoder { - stream: core::ptr::NonNull, - pub(crate) state: State, - /// Decompression-bomb guard: `decompress` errors instead of growing the - /// output past this many bytes. Defaults to unbounded. - pub(crate) max_output_size: usize, - /// zstd filled its last window and may hold more. `max_output` can end a call there. + stream: DCtx, + state: State, + /// The decoder filled its last window and may hold more. `max_output` can end a call there. output_full: bool, } +impl Default for StreamingDecoder { + fn default() -> Self { + Self::new() + } +} + impl StreamingDecoder { - pub fn new() -> core::result::Result { - let stream = core::ptr::NonNull::new(c::ZSTD_createDStream()) - .ok_or(ZstdError::ZstdFailedToCreateInstance)?; - // SAFETY: stream is a freshly created non-null DStream. - let _ = unsafe { c::ZSTD_initDStream(stream.as_ptr()) }; - Ok(Self { - stream, + pub fn new() -> Self { + Self { + stream: DCtx::new(), state: State::Uninitialized, - max_output_size: usize::MAX, output_full: false, - }) + } } #[inline] @@ -646,17 +189,17 @@ impl StreamingDecoder { out: &mut Vec, max_output: usize, is_done: bool, - ) -> core::result::Result { + ) -> Result { if matches!(self.state, State::End | State::Error) { return Ok(input.len()); } - let mut total_in = 0usize; + let mut in_buf = InBuffer { src: input, pos: 0 }; while matches!(self.state, State::Uninitialized | State::Inflating) { - let next_in = &input[total_in..]; + let next_in_len = input.len() - in_buf.pos; - // Call zstd again with no input until it leaves the window short. - if next_in.is_empty() && !self.output_full { + // Call the decoder again with no input until it leaves the window short. + if next_in_len == 0 && !self.output_full { if is_done { if self.state == State::Inflating { self.state = State::Error; @@ -664,76 +207,61 @@ impl StreamingDecoder { } self.state = State::End; } - return Ok(total_in); + return Ok(in_buf.pos); } if out.len() >= max_output { - return Ok(total_in); - } - - let remaining_output = self.max_output_size.saturating_sub(out.len()); - if remaining_output == 0 { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); + return Ok(in_buf.pos); } if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { self.state = State::Error; return Err(ZstdError::OutOfMemory); } - let budget = max_output - out.len(); - let spare = out.spare_capacity_mut(); - let mut in_buf = c::ZSTD_inBuffer { - src: next_in.as_ptr().cast::(), - size: next_in.len(), - pos: 0, - }; - let mut out_buf = c::ZSTD_outBuffer { - dst: spare.as_mut_ptr().cast::(), - size: spare.len().min(remaining_output).min(budget), + // The decoder writes to initialized memory only. + let written_before = out.len(); + let window = (out.capacity() - written_before).min(max_output - written_before); + out.resize(written_before + window, 0); + let mut out_buf = OutBuffer { + dst: out.get_mut(written_before..).unwrap_or_default(), pos: 0, }; + let read_before = in_buf.pos; - // SAFETY: stream is a valid DStream (not freed); in_buf/out_buf - // point into live slices with correct sizes. - let rc = unsafe { - c::ZSTD_decompressStream(self.stream.as_ptr(), &raw mut out_buf, &raw mut in_buf) + let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); + let bytes_written = out_buf.pos; + out.truncate(written_before + bytes_written); + let remaining = match result { + Ok(remaining) => remaining, + Err(code) => { + self.state = State::Error; + return Err(ZstdError::for_decompression( + code, + ZstdError::ZstdDecompressionError, + )); + } }; - if c::ZSTD_isError(rc) != 0 { - self.state = State::Error; - return Err(ZstdError::for_decompression( - rc, - ZstdError::ZstdDecompressionError, - )); - } - let bytes_written = out_buf.pos; - let bytes_read = in_buf.pos; - self.output_full = bytes_written == out_buf.size; - // SAFETY: zstd wrote exactly `bytes_written` initialized bytes into - // the spare capacity starting at the previous len. - unsafe { bun_core::vec::commit_spare(out, bytes_written) }; - total_in += bytes_read; - - if rc == 0 { + self.output_full = bytes_written == window; + + if remaining == 0 { // Frame complete, and fully flushed. self.state = State::Uninitialized; self.output_full = false; - if total_in >= input.len() { + if in_buf.pos >= input.len() { if is_done { self.state = State::End; } - return Ok(total_in); + return Ok(in_buf.pos); } // More input available — reinitialize for the next frame. - // SAFETY: stream is a valid DStream. - let _ = unsafe { c::ZSTD_initDStream(self.stream.as_ptr()) }; + self.stream.init_stream(); continue; } self.state = State::Inflating; - if bytes_read == next_in.len() { + if in_buf.pos - read_before == next_in_len { if self.output_full { continue; } @@ -744,14 +272,7 @@ impl StreamingDecoder { return Err(ZstdError::ShortRead); } } - Ok(total_in) - } -} - -impl Drop for StreamingDecoder { - fn drop(&mut self) { - // SAFETY: stream was created by ZSTD_createDStream; freed once here. - let _ = unsafe { c::ZSTD_freeDStream(self.stream.as_ptr()) }; + Ok(in_buf.pos) } } @@ -843,6 +364,7 @@ macro_rules! embed_compressed { /// Cold, shared body of [`embed_compressed!`]'s release arm. #[cold] #[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { decompress_alloc(compressed) .expect("embedded asset: invalid zstd frame") @@ -852,6 +374,7 @@ pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { /// [`inflate_embedded`] plus a trailing NUL byte. #[cold] #[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); inflated.reserve_exact(1); diff --git a/src/zstd/util.rs b/src/zstd/util.rs new file mode 100644 index 000000000000..85b00622f155 --- /dev/null +++ b/src/zstd/util.rs @@ -0,0 +1,176 @@ +//! Slice access that reports a broken invariant instead of panicking, and `mem.h` / `bits.h`. + +use core::slice::SliceIndex; + +use crate::error::{ErrorCode, Result}; + +/// An index or a length that the algorithm keeps in range was out of range. No input should be able to cause it. +pub(crate) const BUG: ErrorCode = ErrorCode::Generic; + +pub(crate) trait SliceExt { + fn at(&self, index: usize) -> Result + where + T: Copy; + fn at_mut(&mut self, index: usize) -> Result<&mut T>; + fn put(&mut self, index: usize, value: T) -> Result<()>; + fn sub>(&self, range: I) -> Result<&[T]>; + fn sub_mut>(&mut self, range: I) -> Result<&mut [T]>; + /// The `len` elements from `start` on. + fn span(&self, start: usize, len: usize) -> Result<&[T]>; + fn span_mut(&mut self, start: usize, len: usize) -> Result<&mut [T]>; + /// The `N` elements from `start` on. + fn array_at(&self, start: usize) -> Result<&[T; N]>; + fn array_at_mut(&mut self, start: usize) -> Result<&mut [T; N]>; + /// `memmove(&self[dst], &self[src], len)`. + fn move_within(&mut self, src: usize, dst: usize, len: usize) -> Result<()> + where + T: Copy; + /// `memcpy(&self[start], from, from.len())`. + fn write_at(&mut self, start: usize, from: &[T]) -> Result<()> + where + T: Copy; +} + +impl SliceExt for [T] { + #[inline(always)] + fn at(&self, index: usize) -> Result + where + T: Copy, + { + self.get(index).copied().ok_or(BUG) + } + + #[inline(always)] + fn at_mut(&mut self, index: usize) -> Result<&mut T> { + self.get_mut(index).ok_or(BUG) + } + + #[inline(always)] + fn put(&mut self, index: usize, value: T) -> Result<()> { + *self.get_mut(index).ok_or(BUG)? = value; + Ok(()) + } + + #[inline(always)] + fn sub>(&self, range: I) -> Result<&[T]> { + self.get(range).ok_or(BUG) + } + + #[inline(always)] + fn sub_mut>(&mut self, range: I) -> Result<&mut [T]> { + self.get_mut(range).ok_or(BUG) + } + + #[inline(always)] + fn span(&self, start: usize, len: usize) -> Result<&[T]> { + self.get(start..).and_then(|s| s.get(..len)).ok_or(BUG) + } + + #[inline(always)] + fn span_mut(&mut self, start: usize, len: usize) -> Result<&mut [T]> { + self.get_mut(start..) + .and_then(|s| s.get_mut(..len)) + .ok_or(BUG) + } + + #[inline(always)] + fn array_at(&self, start: usize) -> Result<&[T; N]> { + self.get(start..).and_then(<[T]>::first_chunk).ok_or(BUG) + } + + #[inline(always)] + fn array_at_mut(&mut self, start: usize) -> Result<&mut [T; N]> { + self.get_mut(start..) + .and_then(<[T]>::first_chunk_mut) + .ok_or(BUG) + } + + #[inline(always)] + fn move_within(&mut self, src: usize, dst: usize, len: usize) -> Result<()> + where + T: Copy, + { + let src_end = src.checked_add(len).ok_or(BUG)?; + let dst_end = dst.checked_add(len).ok_or(BUG)?; + if src_end > self.len() || dst_end > self.len() { + return Err(BUG); + } + self.copy_within(src..src_end, dst); + Ok(()) + } + + #[inline(always)] + fn write_at(&mut self, start: usize, from: &[T]) -> Result<()> + where + T: Copy, + { + self.span_mut(start, from.len())?.copy_from_slice(from); + Ok(()) + } +} + +#[inline(always)] +pub(crate) fn read_le16(data: &[u8], pos: usize) -> Result { + Ok(u16::from_le_bytes(*data.array_at(pos)?)) +} + +#[inline(always)] +pub(crate) fn read_le24(data: &[u8], pos: usize) -> Result { + let [a, b, c] = *data.array_at(pos)?; + Ok(u32::from_le_bytes([a, b, c, 0])) +} + +#[inline(always)] +pub(crate) fn read_le32(data: &[u8], pos: usize) -> Result { + Ok(u32::from_le_bytes(*data.array_at(pos)?)) +} + +#[inline(always)] +pub(crate) fn read_le64(data: &[u8], pos: usize) -> Result { + Ok(u64::from_le_bytes(*data.array_at(pos)?)) +} + +#[inline(always)] +pub(crate) fn write_le16(data: &mut [u8], pos: usize, value: u16) -> Result<()> { + *data.array_at_mut(pos)? = value.to_le_bytes(); + Ok(()) +} + +#[inline(always)] +pub(crate) fn write_le24(data: &mut [u8], pos: usize, value: u32) -> Result<()> { + let [a, b, c, _] = value.to_le_bytes(); + *data.array_at_mut(pos)? = [a, b, c]; + Ok(()) +} + +#[inline(always)] +pub(crate) fn write_le32(data: &mut [u8], pos: usize, value: u32) -> Result<()> { + *data.array_at_mut(pos)? = value.to_le_bytes(); + Ok(()) +} + +#[inline(always)] +pub(crate) fn write_le64(data: &mut [u8], pos: usize, value: u64) -> Result<()> { + *data.array_at_mut(pos)? = value.to_le_bytes(); + Ok(()) +} + +/// `ZSTD_highbit32`: the position of the highest bit set. `value` is not 0. +#[inline(always)] +pub(crate) const fn highbit32(value: u32) -> u32 { + 31 - (value | 1).leading_zeros() +} + +/// `len` zeros, or [`ErrorCode::MemoryAllocation`]. A large one costs nothing until it is written to. +pub(crate) fn try_zeroed(len: usize) -> Result> { + bun_pod::try_zeroed_vec(len).ok_or(ErrorCode::MemoryAllocation) +} + +/// A copy of `bytes`, or [`ErrorCode::MemoryAllocation`]. +pub(crate) fn try_copy(bytes: &[u8]) -> Result> { + let mut vec = Vec::new(); + vec.try_reserve_exact(bytes.len()) + .map_err(|_| ErrorCode::MemoryAllocation)?; + vec.extend_from_slice(bytes); + Ok(vec) +} From 93bb106b975534e2213745e76ea721dc37bd707b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 12:24:21 +0000 Subject: [PATCH 003/199] zstd: add the compressor in safe Rust (work in progress) Parameters, frames, dictionaries, block splitting, streaming and the multithreaded mode, on top of the match finders and the entropy coders. The users of the C API move to the new one. src/zstd/wip holds the differential test harness and scratch copies, to be removed. --- src/http/Decompressor.rs | 2 +- src/http/compress_body.rs | 12 +- src/jsc/bindings/BunProcess.cpp | 4 +- src/jsc/bindings/ProcessBindingConstants.cpp | 123 +- src/jsc/bindings/bun_icu_decompress.cpp | 44 +- src/runtime/api/BunObject.rs | 21 +- src/runtime/icu_decompress.rs | 77 + src/runtime/lib.rs | 2 + src/runtime/node/zlib/NativeZstd.rs | 383 +--- src/runtime/webcore/CompressionStreamCoder.rs | 141 +- src/standalone_graph/StandaloneModuleGraph.rs | 16 +- src/zstd/common/mod.rs | 3 + src/zstd/compress/block_entropy.rs | 690 +++++++ src/zstd/compress/cctx.rs | 1727 +++++++++++++++++ src/zstd/compress/double_fast.rs | 785 ++++++++ src/zstd/compress/fast.rs | 829 ++++++++ src/zstd/compress/fse.rs | 672 +++++++ src/zstd/compress/hist.rs | 130 ++ src/zstd/compress/huf.rs | 1142 +++++++++++ src/zstd/compress/internal.rs | 1267 ++++++++++++ src/zstd/compress/lazy.rs | 1384 +++++++++++++ src/zstd/compress/ldm.rs | 779 ++++++++ src/zstd/compress/literals.rs | 147 ++ src/zstd/compress/mod.rs | 76 + src/zstd/compress/mt.rs | 829 ++++++++ src/zstd/compress/opt.rs | 1721 ++++++++++++++++ src/zstd/compress/params.rs | 911 +++++++++ src/zstd/compress/presplit.rs | 184 ++ src/zstd/compress/sequences.rs | 308 +++ src/zstd/compress/stream.rs | 522 +++++ src/zstd/compress/superblock.rs | 601 ++++++ src/zstd/decompress/mod.rs | 41 +- src/zstd/lib.rs | 29 +- src/zstd/wip/build-ref.sh | 15 + src/zstd/wip/clevels.rs | 100 + src/zstd/wip/default_tables.rs | 51 + src/zstd/wip/harness/Cargo.toml | 20 + src/zstd/wip/harness/build.rs | 6 + src/zstd/wip/harness/rust-toolchain.toml | 35 + src/zstd/wip/harness/src/c.rs | 104 + src/zstd/wip/harness/src/comp.rs | 296 +++ src/zstd/wip/harness/src/data.rs | 65 + src/zstd/wip/harness/src/dec.rs | 184 ++ src/zstd/wip/harness/src/main.rs | 18 + src/zstd/wip/par.sh | 11 + src/zstd/wip/strip.py | 7 + src/zstd/wip/template/mkwork.sh | 61 + src/zstd/wip/template/seqs.rs | 73 + src/zstd/wip/template/testing.rs | 140 ++ src/zstd/wip/work/entropy/harness/Cargo.toml | 20 + src/zstd/wip/work/entropy/harness/build.rs | 20 + .../wip/work/entropy/harness/entropy_shim.c | 43 + .../work/entropy/harness/rust-toolchain.toml | 35 + src/zstd/wip/work/entropy/harness/src/c.rs | 104 + src/zstd/wip/work/entropy/harness/src/data.rs | 65 + .../wip/work/entropy/harness/src/entropy.rs | 774 ++++++++ src/zstd/wip/work/entropy/harness/src/main.rs | 22 + src/zstd/wip/work/entropy/harness/src/seqs.rs | 73 + .../work/entropy/zstd/compress/double_fast.rs | 48 + .../wip/work/entropy/zstd/compress/fast.rs | 48 + .../wip/work/entropy/zstd/compress/fse.rs | 678 +++++++ .../wip/work/entropy/zstd/compress/hist.rs | 132 ++ .../wip/work/entropy/zstd/compress/huf.rs | 1162 +++++++++++ .../work/entropy/zstd/compress/internal.rs | 1212 ++++++++++++ .../wip/work/entropy/zstd/compress/lazy.rs | 34 + .../wip/work/entropy/zstd/compress/ldm.rs | 1 + .../wip/work/entropy/zstd/compress/mod.rs | 68 + .../wip/work/entropy/zstd/compress/opt.rs | 29 + .../work/entropy/zstd/compress/presplit.rs | 1 + src/zstd/wip/work/entropy/zstd/lib.rs | 385 ++++ src/zstd/wip/work/entropy/zstd/testing.rs | 457 +++++ src/zstd/wip/work/fast/harness/Cargo.toml | 20 + src/zstd/wip/work/fast/harness/build.rs | 21 + src/zstd/wip/work/fast/harness/c/shim.c | 169 ++ .../wip/work/fast/harness/c/shim_cctx.inc | 30 + .../wip/work/fast/harness/rust-toolchain.toml | 35 + src/zstd/wip/work/fast/harness/src/c.rs | 104 + src/zstd/wip/work/fast/harness/src/data.rs | 65 + src/zstd/wip/work/fast/harness/src/frames.rs | 178 ++ src/zstd/wip/work/fast/harness/src/main.rs | 44 + src/zstd/wip/work/fast/harness/src/seqs.rs | 78 + src/zstd/wip/work/fast/harness/src/stream.rs | 301 +++ .../work/fast/zstd/compress/double_fast.rs | 785 ++++++++ src/zstd/wip/work/fast/zstd/compress/fast.rs | 829 ++++++++ src/zstd/wip/work/fast/zstd/compress/fse.rs | 70 + src/zstd/wip/work/fast/zstd/compress/hist.rs | 33 + src/zstd/wip/work/fast/zstd/compress/huf.rs | 38 + .../wip/work/fast/zstd/compress/internal.rs | 1212 ++++++++++++ src/zstd/wip/work/fast/zstd/compress/lazy.rs | 34 + src/zstd/wip/work/fast/zstd/compress/ldm.rs | 1 + src/zstd/wip/work/fast/zstd/compress/mod.rs | 68 + src/zstd/wip/work/fast/zstd/compress/opt.rs | 29 + .../wip/work/fast/zstd/compress/presplit.rs | 1 + src/zstd/wip/work/fast/zstd/lib.rs | 386 ++++ src/zstd/wip/work/fast/zstd/testing.rs | 364 ++++ src/zstd/wip/work/lazy/harness/Cargo.toml | 20 + src/zstd/wip/work/lazy/harness/build.rs | 18 + .../wip/work/lazy/harness/rust-toolchain.toml | 35 + src/zstd/wip/work/lazy/harness/shim.c | 41 + src/zstd/wip/work/lazy/harness/src/c.rs | 104 + src/zstd/wip/work/lazy/harness/src/data.rs | 65 + src/zstd/wip/work/lazy/harness/src/main.rs | 27 + src/zstd/wip/work/lazy/harness/src/seqs.rs | 193 ++ src/zstd/wip/work/lazy/lazy.exp-filter.rs | 1360 +++++++++++++ src/zstd/wip/work/lazy/lazy.exp-prefetch.rs | 1384 +++++++++++++ src/zstd/wip/work/lazy/lazy.v1.rs | 1353 +++++++++++++ src/zstd/wip/work/lazy/lazy.v2.rs | 1419 ++++++++++++++ src/zstd/wip/work/lazy/lib.orig.rs | 385 ++++ .../work/lazy/zstd/compress/double_fast.rs | 48 + src/zstd/wip/work/lazy/zstd/compress/fast.rs | 48 + src/zstd/wip/work/lazy/zstd/compress/fse.rs | 70 + src/zstd/wip/work/lazy/zstd/compress/hist.rs | 33 + src/zstd/wip/work/lazy/zstd/compress/huf.rs | 38 + .../wip/work/lazy/zstd/compress/internal.rs | 1212 ++++++++++++ src/zstd/wip/work/lazy/zstd/compress/lazy.rs | 1420 ++++++++++++++ src/zstd/wip/work/lazy/zstd/compress/ldm.rs | 1 + src/zstd/wip/work/lazy/zstd/compress/mod.rs | 68 + src/zstd/wip/work/lazy/zstd/compress/opt.rs | 96 + .../wip/work/lazy/zstd/compress/presplit.rs | 1 + src/zstd/wip/work/lazy/zstd/lib.rs | 386 ++++ src/zstd/wip/work/lazy/zstd/testing.rs | 372 ++++ src/zstd/wip/work/ldm/harness/Cargo.toml | 20 + src/zstd/wip/work/ldm/harness/build.rs | 19 + .../wip/work/ldm/harness/rust-toolchain.toml | 35 + src/zstd/wip/work/ldm/harness/shim.c | 180 ++ src/zstd/wip/work/ldm/harness/src/c.rs | 104 + src/zstd/wip/work/ldm/harness/src/data.rs | 65 + src/zstd/wip/work/ldm/harness/src/ldm.rs | 452 +++++ src/zstd/wip/work/ldm/harness/src/main.rs | 30 + src/zstd/wip/work/ldm/harness/src/seqs.rs | 73 + src/zstd/wip/work/ldm/snap/double_fast.rs | 785 ++++++++ src/zstd/wip/work/ldm/snap/fast.rs | 829 ++++++++ src/zstd/wip/work/ldm/snap/lazy.rs | 1353 +++++++++++++ src/zstd/wip/work/ldm/snap/opt.rs | 1721 ++++++++++++++++ src/zstd/wip/work/ldm/stubs/double_fast.rs | 48 + src/zstd/wip/work/ldm/stubs/fast.rs | 48 + src/zstd/wip/work/ldm/stubs/lazy.rs | 34 + src/zstd/wip/work/ldm/stubs/opt.rs | 29 + .../wip/work/ldm/zstd/compress/double_fast.rs | 785 ++++++++ src/zstd/wip/work/ldm/zstd/compress/fast.rs | 829 ++++++++ src/zstd/wip/work/ldm/zstd/compress/fse.rs | 70 + src/zstd/wip/work/ldm/zstd/compress/hist.rs | 33 + src/zstd/wip/work/ldm/zstd/compress/huf.rs | 38 + .../wip/work/ldm/zstd/compress/internal.rs | 1212 ++++++++++++ src/zstd/wip/work/ldm/zstd/compress/lazy.rs | 1353 +++++++++++++ src/zstd/wip/work/ldm/zstd/compress/ldm.rs | 779 ++++++++ src/zstd/wip/work/ldm/zstd/compress/mod.rs | 68 + src/zstd/wip/work/ldm/zstd/compress/opt.rs | 1721 ++++++++++++++++ .../wip/work/ldm/zstd/compress/presplit.rs | 184 ++ src/zstd/wip/work/ldm/zstd/lib.rs | 385 ++++ src/zstd/wip/work/ldm/zstd/testing.rs | 656 +++++++ src/zstd/wip/work/opt/harness/Cargo.toml | 20 + src/zstd/wip/work/opt/harness/build.rs | 20 + .../wip/work/opt/harness/rust-toolchain.toml | 35 + src/zstd/wip/work/opt/harness/shim.c | 98 + src/zstd/wip/work/opt/harness/src/c.rs | 104 + src/zstd/wip/work/opt/harness/src/data.rs | 65 + src/zstd/wip/work/opt/harness/src/main.rs | 25 + src/zstd/wip/work/opt/harness/src/seqs.rs | 235 +++ .../wip/work/opt/zstd/compress/double_fast.rs | 48 + src/zstd/wip/work/opt/zstd/compress/fast.rs | 48 + src/zstd/wip/work/opt/zstd/compress/fse.rs | 70 + src/zstd/wip/work/opt/zstd/compress/hist.rs | 33 + src/zstd/wip/work/opt/zstd/compress/huf.rs | 38 + .../wip/work/opt/zstd/compress/internal.rs | 1212 ++++++++++++ src/zstd/wip/work/opt/zstd/compress/lazy.rs | 34 + src/zstd/wip/work/opt/zstd/compress/ldm.rs | 1 + src/zstd/wip/work/opt/zstd/compress/mod.rs | 68 + src/zstd/wip/work/opt/zstd/compress/opt.rs | 1721 ++++++++++++++++ .../wip/work/opt/zstd/compress/presplit.rs | 1 + src/zstd/wip/work/opt/zstd/lib.rs | 385 ++++ src/zstd/wip/work/opt/zstd/testing.rs | 288 +++ test/js/node/process/process.test.js | 4 +- 173 files changed, 55275 insertions(+), 507 deletions(-) create mode 100644 src/runtime/icu_decompress.rs create mode 100644 src/zstd/compress/block_entropy.rs create mode 100644 src/zstd/compress/cctx.rs create mode 100644 src/zstd/compress/double_fast.rs create mode 100644 src/zstd/compress/fast.rs create mode 100644 src/zstd/compress/fse.rs create mode 100644 src/zstd/compress/hist.rs create mode 100644 src/zstd/compress/huf.rs create mode 100644 src/zstd/compress/internal.rs create mode 100644 src/zstd/compress/lazy.rs create mode 100644 src/zstd/compress/ldm.rs create mode 100644 src/zstd/compress/literals.rs create mode 100644 src/zstd/compress/mod.rs create mode 100644 src/zstd/compress/mt.rs create mode 100644 src/zstd/compress/opt.rs create mode 100644 src/zstd/compress/params.rs create mode 100644 src/zstd/compress/presplit.rs create mode 100644 src/zstd/compress/sequences.rs create mode 100644 src/zstd/compress/stream.rs create mode 100644 src/zstd/compress/superblock.rs create mode 100755 src/zstd/wip/build-ref.sh create mode 100644 src/zstd/wip/clevels.rs create mode 100644 src/zstd/wip/default_tables.rs create mode 100644 src/zstd/wip/harness/Cargo.toml create mode 100644 src/zstd/wip/harness/build.rs create mode 100644 src/zstd/wip/harness/rust-toolchain.toml create mode 100644 src/zstd/wip/harness/src/c.rs create mode 100644 src/zstd/wip/harness/src/comp.rs create mode 100644 src/zstd/wip/harness/src/data.rs create mode 100644 src/zstd/wip/harness/src/dec.rs create mode 100644 src/zstd/wip/harness/src/main.rs create mode 100755 src/zstd/wip/par.sh create mode 100644 src/zstd/wip/strip.py create mode 100755 src/zstd/wip/template/mkwork.sh create mode 100644 src/zstd/wip/template/seqs.rs create mode 100644 src/zstd/wip/template/testing.rs create mode 100644 src/zstd/wip/work/entropy/harness/Cargo.toml create mode 100644 src/zstd/wip/work/entropy/harness/build.rs create mode 100644 src/zstd/wip/work/entropy/harness/entropy_shim.c create mode 100644 src/zstd/wip/work/entropy/harness/rust-toolchain.toml create mode 100644 src/zstd/wip/work/entropy/harness/src/c.rs create mode 100644 src/zstd/wip/work/entropy/harness/src/data.rs create mode 100644 src/zstd/wip/work/entropy/harness/src/entropy.rs create mode 100644 src/zstd/wip/work/entropy/harness/src/main.rs create mode 100644 src/zstd/wip/work/entropy/harness/src/seqs.rs create mode 100644 src/zstd/wip/work/entropy/zstd/compress/double_fast.rs create mode 100644 src/zstd/wip/work/entropy/zstd/compress/fast.rs create mode 100644 src/zstd/wip/work/entropy/zstd/compress/fse.rs create mode 100644 src/zstd/wip/work/entropy/zstd/compress/hist.rs create mode 100644 src/zstd/wip/work/entropy/zstd/compress/huf.rs create mode 100644 src/zstd/wip/work/entropy/zstd/compress/internal.rs create mode 100644 src/zstd/wip/work/entropy/zstd/compress/lazy.rs create mode 100644 src/zstd/wip/work/entropy/zstd/compress/ldm.rs create mode 100644 src/zstd/wip/work/entropy/zstd/compress/mod.rs create mode 100644 src/zstd/wip/work/entropy/zstd/compress/opt.rs create mode 100644 src/zstd/wip/work/entropy/zstd/compress/presplit.rs create mode 100644 src/zstd/wip/work/entropy/zstd/lib.rs create mode 100644 src/zstd/wip/work/entropy/zstd/testing.rs create mode 100644 src/zstd/wip/work/fast/harness/Cargo.toml create mode 100644 src/zstd/wip/work/fast/harness/build.rs create mode 100644 src/zstd/wip/work/fast/harness/c/shim.c create mode 100644 src/zstd/wip/work/fast/harness/c/shim_cctx.inc create mode 100644 src/zstd/wip/work/fast/harness/rust-toolchain.toml create mode 100644 src/zstd/wip/work/fast/harness/src/c.rs create mode 100644 src/zstd/wip/work/fast/harness/src/data.rs create mode 100644 src/zstd/wip/work/fast/harness/src/frames.rs create mode 100644 src/zstd/wip/work/fast/harness/src/main.rs create mode 100644 src/zstd/wip/work/fast/harness/src/seqs.rs create mode 100644 src/zstd/wip/work/fast/harness/src/stream.rs create mode 100644 src/zstd/wip/work/fast/zstd/compress/double_fast.rs create mode 100644 src/zstd/wip/work/fast/zstd/compress/fast.rs create mode 100644 src/zstd/wip/work/fast/zstd/compress/fse.rs create mode 100644 src/zstd/wip/work/fast/zstd/compress/hist.rs create mode 100644 src/zstd/wip/work/fast/zstd/compress/huf.rs create mode 100644 src/zstd/wip/work/fast/zstd/compress/internal.rs create mode 100644 src/zstd/wip/work/fast/zstd/compress/lazy.rs create mode 100644 src/zstd/wip/work/fast/zstd/compress/ldm.rs create mode 100644 src/zstd/wip/work/fast/zstd/compress/mod.rs create mode 100644 src/zstd/wip/work/fast/zstd/compress/opt.rs create mode 100644 src/zstd/wip/work/fast/zstd/compress/presplit.rs create mode 100644 src/zstd/wip/work/fast/zstd/lib.rs create mode 100644 src/zstd/wip/work/fast/zstd/testing.rs create mode 100644 src/zstd/wip/work/lazy/harness/Cargo.toml create mode 100644 src/zstd/wip/work/lazy/harness/build.rs create mode 100644 src/zstd/wip/work/lazy/harness/rust-toolchain.toml create mode 100644 src/zstd/wip/work/lazy/harness/shim.c create mode 100644 src/zstd/wip/work/lazy/harness/src/c.rs create mode 100644 src/zstd/wip/work/lazy/harness/src/data.rs create mode 100644 src/zstd/wip/work/lazy/harness/src/main.rs create mode 100644 src/zstd/wip/work/lazy/harness/src/seqs.rs create mode 100644 src/zstd/wip/work/lazy/lazy.exp-filter.rs create mode 100644 src/zstd/wip/work/lazy/lazy.exp-prefetch.rs create mode 100644 src/zstd/wip/work/lazy/lazy.v1.rs create mode 100644 src/zstd/wip/work/lazy/lazy.v2.rs create mode 100644 src/zstd/wip/work/lazy/lib.orig.rs create mode 100644 src/zstd/wip/work/lazy/zstd/compress/double_fast.rs create mode 100644 src/zstd/wip/work/lazy/zstd/compress/fast.rs create mode 100644 src/zstd/wip/work/lazy/zstd/compress/fse.rs create mode 100644 src/zstd/wip/work/lazy/zstd/compress/hist.rs create mode 100644 src/zstd/wip/work/lazy/zstd/compress/huf.rs create mode 100644 src/zstd/wip/work/lazy/zstd/compress/internal.rs create mode 100644 src/zstd/wip/work/lazy/zstd/compress/lazy.rs create mode 100644 src/zstd/wip/work/lazy/zstd/compress/ldm.rs create mode 100644 src/zstd/wip/work/lazy/zstd/compress/mod.rs create mode 100644 src/zstd/wip/work/lazy/zstd/compress/opt.rs create mode 100644 src/zstd/wip/work/lazy/zstd/compress/presplit.rs create mode 100644 src/zstd/wip/work/lazy/zstd/lib.rs create mode 100644 src/zstd/wip/work/lazy/zstd/testing.rs create mode 100644 src/zstd/wip/work/ldm/harness/Cargo.toml create mode 100644 src/zstd/wip/work/ldm/harness/build.rs create mode 100644 src/zstd/wip/work/ldm/harness/rust-toolchain.toml create mode 100644 src/zstd/wip/work/ldm/harness/shim.c create mode 100644 src/zstd/wip/work/ldm/harness/src/c.rs create mode 100644 src/zstd/wip/work/ldm/harness/src/data.rs create mode 100644 src/zstd/wip/work/ldm/harness/src/ldm.rs create mode 100644 src/zstd/wip/work/ldm/harness/src/main.rs create mode 100644 src/zstd/wip/work/ldm/harness/src/seqs.rs create mode 100644 src/zstd/wip/work/ldm/snap/double_fast.rs create mode 100644 src/zstd/wip/work/ldm/snap/fast.rs create mode 100644 src/zstd/wip/work/ldm/snap/lazy.rs create mode 100644 src/zstd/wip/work/ldm/snap/opt.rs create mode 100644 src/zstd/wip/work/ldm/stubs/double_fast.rs create mode 100644 src/zstd/wip/work/ldm/stubs/fast.rs create mode 100644 src/zstd/wip/work/ldm/stubs/lazy.rs create mode 100644 src/zstd/wip/work/ldm/stubs/opt.rs create mode 100644 src/zstd/wip/work/ldm/zstd/compress/double_fast.rs create mode 100644 src/zstd/wip/work/ldm/zstd/compress/fast.rs create mode 100644 src/zstd/wip/work/ldm/zstd/compress/fse.rs create mode 100644 src/zstd/wip/work/ldm/zstd/compress/hist.rs create mode 100644 src/zstd/wip/work/ldm/zstd/compress/huf.rs create mode 100644 src/zstd/wip/work/ldm/zstd/compress/internal.rs create mode 100644 src/zstd/wip/work/ldm/zstd/compress/lazy.rs create mode 100644 src/zstd/wip/work/ldm/zstd/compress/ldm.rs create mode 100644 src/zstd/wip/work/ldm/zstd/compress/mod.rs create mode 100644 src/zstd/wip/work/ldm/zstd/compress/opt.rs create mode 100644 src/zstd/wip/work/ldm/zstd/compress/presplit.rs create mode 100644 src/zstd/wip/work/ldm/zstd/lib.rs create mode 100644 src/zstd/wip/work/ldm/zstd/testing.rs create mode 100644 src/zstd/wip/work/opt/harness/Cargo.toml create mode 100644 src/zstd/wip/work/opt/harness/build.rs create mode 100644 src/zstd/wip/work/opt/harness/rust-toolchain.toml create mode 100644 src/zstd/wip/work/opt/harness/shim.c create mode 100644 src/zstd/wip/work/opt/harness/src/c.rs create mode 100644 src/zstd/wip/work/opt/harness/src/data.rs create mode 100644 src/zstd/wip/work/opt/harness/src/main.rs create mode 100644 src/zstd/wip/work/opt/harness/src/seqs.rs create mode 100644 src/zstd/wip/work/opt/zstd/compress/double_fast.rs create mode 100644 src/zstd/wip/work/opt/zstd/compress/fast.rs create mode 100644 src/zstd/wip/work/opt/zstd/compress/fse.rs create mode 100644 src/zstd/wip/work/opt/zstd/compress/hist.rs create mode 100644 src/zstd/wip/work/opt/zstd/compress/huf.rs create mode 100644 src/zstd/wip/work/opt/zstd/compress/internal.rs create mode 100644 src/zstd/wip/work/opt/zstd/compress/lazy.rs create mode 100644 src/zstd/wip/work/opt/zstd/compress/ldm.rs create mode 100644 src/zstd/wip/work/opt/zstd/compress/mod.rs create mode 100644 src/zstd/wip/work/opt/zstd/compress/opt.rs create mode 100644 src/zstd/wip/work/opt/zstd/compress/presplit.rs create mode 100644 src/zstd/wip/work/opt/zstd/lib.rs create mode 100644 src/zstd/wip/work/opt/zstd/testing.rs diff --git a/src/http/Decompressor.rs b/src/http/Decompressor.rs index cf27f19981c2..6c32378ad9d8 100644 --- a/src/http/Decompressor.rs +++ b/src/http/Decompressor.rs @@ -61,7 +61,7 @@ impl Decompressor { )?)); } Encoding::Zstd => { - *self = Decompressor::Zstd(Box::new(bun_zstd::StreamingDecoder::new()?)); + *self = Decompressor::Zstd(Box::new(bun_zstd::StreamingDecoder::new())); } _ => unreachable!("Invalid encoding. This code should not be reachable"), } diff --git a/src/http/compress_body.rs b/src/http/compress_body.rs index 2d4cc829bd86..6f0f6a1745e2 100644 --- a/src/http/compress_body.rs +++ b/src/http/compress_body.rs @@ -219,14 +219,12 @@ fn compress_zstd( level: Option, spill: &mut Vec, ) -> crate::Result { + let level = level.unwrap_or(bun_zstd::ZSTD_CLEVEL_DEFAULT); let bound = bun_zstd::compress_bound(input.len()); - if bun_zstd::is_error(bound) { - return Err(crate::Error::CompressionFailed); - } if bound <= state.shared_buffer.len() { return match bun_zstd::compress(&mut state.shared_buffer, input, level) { - bun_zstd::Result::Success(n) => Ok(CompressOutput::Shared(n)), - bun_zstd::Result::Err(_) => Err(crate::Error::CompressionFailed), + Ok(n) => Ok(CompressOutput::Shared(n)), + Err(_) => Err(crate::Error::CompressionFailed), }; } @@ -234,7 +232,7 @@ fn compress_zstd( .try_reserve_exact(bound) .map_err(|_| bun_alloc::AllocError)?; match bun_zstd::compress_append(spill, input, level) { - bun_zstd::Result::Success(_) => Ok(CompressOutput::Spilled), - bun_zstd::Result::Err(_) => Err(crate::Error::CompressionFailed), + Ok(_) => Ok(CompressOutput::Spilled), + Err(_) => Err(crate::Error::CompressionFailed), } } diff --git a/src/jsc/bindings/BunProcess.cpp b/src/jsc/bindings/BunProcess.cpp index 655c1049c077..1552cae06635 100644 --- a/src/jsc/bindings/BunProcess.cpp +++ b/src/jsc/bindings/BunProcess.cpp @@ -290,8 +290,8 @@ static JSValue constructVersions(VM& vm, JSObject* processObject) { "libdeflate", BUN_VERSION_LIBDEFLATE_HASH }, { "usockets", BUN_VERSION_USOCKETS }, { "lshpack", BUN_VERSION_LSHPACK }, - // Use commit hash for zstd (semantic version extraction not working yet) - { "zstd", BUN_VERSION_ZSTD_HASH }, + // The version of zstd that src/zstd gives the same output as. + { "zstd", "1.5.7" }, { "v8", REPORTED_NODEJS_V8_VERSION }, #if !OS(WINDOWS) { "uv", "1.48.0" }, diff --git a/src/jsc/bindings/ProcessBindingConstants.cpp b/src/jsc/bindings/ProcessBindingConstants.cpp index 96630a15bc64..c86f0b7b1d25 100644 --- a/src/jsc/bindings/ProcessBindingConstants.cpp +++ b/src/jsc/bindings/ProcessBindingConstants.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -1154,67 +1153,67 @@ static JSValue processBindingConstantsGetZlib(VM& vm, JSObject* bindingObject) { "BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2", static_cast(BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2) }, { "BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES", static_cast(BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES) }, { "BROTLI_DECODER_ERROR_UNREACHABLE", static_cast(BROTLI_DECODER_ERROR_UNREACHABLE) }, - { "ZSTD_e_continue", static_cast(ZSTD_e_continue) }, - { "ZSTD_e_flush", static_cast(ZSTD_e_flush) }, - { "ZSTD_e_end", static_cast(ZSTD_e_end) }, - { "ZSTD_fast", static_cast(ZSTD_fast) }, - { "ZSTD_dfast", static_cast(ZSTD_dfast) }, - { "ZSTD_greedy", static_cast(ZSTD_greedy) }, - { "ZSTD_lazy", static_cast(ZSTD_lazy) }, - { "ZSTD_lazy2", static_cast(ZSTD_lazy2) }, - { "ZSTD_btlazy2", static_cast(ZSTD_btlazy2) }, - { "ZSTD_btopt", static_cast(ZSTD_btopt) }, - { "ZSTD_btultra", static_cast(ZSTD_btultra) }, - { "ZSTD_btultra2", static_cast(ZSTD_btultra2) }, - { "ZSTD_c_compressionLevel", static_cast(ZSTD_c_compressionLevel) }, - { "ZSTD_c_windowLog", static_cast(ZSTD_c_windowLog) }, - { "ZSTD_c_hashLog", static_cast(ZSTD_c_hashLog) }, - { "ZSTD_c_chainLog", static_cast(ZSTD_c_chainLog) }, - { "ZSTD_c_searchLog", static_cast(ZSTD_c_searchLog) }, - { "ZSTD_c_minMatch", static_cast(ZSTD_c_minMatch) }, - { "ZSTD_c_targetLength", static_cast(ZSTD_c_targetLength) }, - { "ZSTD_c_strategy", static_cast(ZSTD_c_strategy) }, - { "ZSTD_c_enableLongDistanceMatching", static_cast(ZSTD_c_enableLongDistanceMatching) }, - { "ZSTD_c_ldmHashLog", static_cast(ZSTD_c_ldmHashLog) }, - { "ZSTD_c_ldmMinMatch", static_cast(ZSTD_c_ldmMinMatch) }, - { "ZSTD_c_ldmBucketSizeLog", static_cast(ZSTD_c_ldmBucketSizeLog) }, - { "ZSTD_c_ldmHashRateLog", static_cast(ZSTD_c_ldmHashRateLog) }, - { "ZSTD_c_contentSizeFlag", static_cast(ZSTD_c_contentSizeFlag) }, - { "ZSTD_c_checksumFlag", static_cast(ZSTD_c_checksumFlag) }, - { "ZSTD_c_dictIDFlag", static_cast(ZSTD_c_dictIDFlag) }, - { "ZSTD_c_nbWorkers", static_cast(ZSTD_c_nbWorkers) }, - { "ZSTD_c_jobSize", static_cast(ZSTD_c_jobSize) }, - { "ZSTD_c_overlapLog", static_cast(ZSTD_c_overlapLog) }, - { "ZSTD_d_windowLogMax", static_cast(ZSTD_d_windowLogMax) }, - { "ZSTD_CLEVEL_DEFAULT", static_cast(ZSTD_CLEVEL_DEFAULT) }, - { "ZSTD_error_no_error", static_cast(ZSTD_error_no_error) }, - { "ZSTD_error_GENERIC", static_cast(ZSTD_error_GENERIC) }, - { "ZSTD_error_prefix_unknown", static_cast(ZSTD_error_prefix_unknown) }, - { "ZSTD_error_version_unsupported", static_cast(ZSTD_error_version_unsupported) }, - { "ZSTD_error_frameParameter_unsupported", static_cast(ZSTD_error_frameParameter_unsupported) }, - { "ZSTD_error_frameParameter_windowTooLarge", static_cast(ZSTD_error_frameParameter_windowTooLarge) }, - { "ZSTD_error_corruption_detected", static_cast(ZSTD_error_corruption_detected) }, - { "ZSTD_error_checksum_wrong", static_cast(ZSTD_error_checksum_wrong) }, - { "ZSTD_error_literals_headerWrong", static_cast(ZSTD_error_literals_headerWrong) }, - { "ZSTD_error_dictionary_corrupted", static_cast(ZSTD_error_dictionary_corrupted) }, - { "ZSTD_error_dictionary_wrong", static_cast(ZSTD_error_dictionary_wrong) }, - { "ZSTD_error_dictionaryCreation_failed", static_cast(ZSTD_error_dictionaryCreation_failed) }, - { "ZSTD_error_parameter_unsupported", static_cast(ZSTD_error_parameter_unsupported) }, - { "ZSTD_error_parameter_combination_unsupported", static_cast(ZSTD_error_parameter_combination_unsupported) }, - { "ZSTD_error_parameter_outOfBound", static_cast(ZSTD_error_parameter_outOfBound) }, - { "ZSTD_error_tableLog_tooLarge", static_cast(ZSTD_error_tableLog_tooLarge) }, - { "ZSTD_error_maxSymbolValue_tooLarge", static_cast(ZSTD_error_maxSymbolValue_tooLarge) }, - { "ZSTD_error_maxSymbolValue_tooSmall", static_cast(ZSTD_error_maxSymbolValue_tooSmall) }, - { "ZSTD_error_stabilityCondition_notRespected", static_cast(ZSTD_error_stabilityCondition_notRespected) }, - { "ZSTD_error_stage_wrong", static_cast(ZSTD_error_stage_wrong) }, - { "ZSTD_error_init_missing", static_cast(ZSTD_error_init_missing) }, - { "ZSTD_error_memory_allocation", static_cast(ZSTD_error_memory_allocation) }, - { "ZSTD_error_workSpace_tooSmall", static_cast(ZSTD_error_workSpace_tooSmall) }, - { "ZSTD_error_dstSize_tooSmall", static_cast(ZSTD_error_dstSize_tooSmall) }, - { "ZSTD_error_srcSize_wrong", static_cast(ZSTD_error_srcSize_wrong) }, - { "ZSTD_error_dstBuffer_null", static_cast(ZSTD_error_dstBuffer_null) }, - { "ZSTD_error_noForwardProgress_destFull", static_cast(ZSTD_error_noForwardProgress_destFull) }, - { "ZSTD_error_noForwardProgress_inputEmpty", static_cast(ZSTD_error_noForwardProgress_inputEmpty) }, + { "ZSTD_e_continue", static_cast(0) }, + { "ZSTD_e_flush", static_cast(1) }, + { "ZSTD_e_end", static_cast(2) }, + { "ZSTD_fast", static_cast(1) }, + { "ZSTD_dfast", static_cast(2) }, + { "ZSTD_greedy", static_cast(3) }, + { "ZSTD_lazy", static_cast(4) }, + { "ZSTD_lazy2", static_cast(5) }, + { "ZSTD_btlazy2", static_cast(6) }, + { "ZSTD_btopt", static_cast(7) }, + { "ZSTD_btultra", static_cast(8) }, + { "ZSTD_btultra2", static_cast(9) }, + { "ZSTD_c_compressionLevel", static_cast(100) }, + { "ZSTD_c_windowLog", static_cast(101) }, + { "ZSTD_c_hashLog", static_cast(102) }, + { "ZSTD_c_chainLog", static_cast(103) }, + { "ZSTD_c_searchLog", static_cast(104) }, + { "ZSTD_c_minMatch", static_cast(105) }, + { "ZSTD_c_targetLength", static_cast(106) }, + { "ZSTD_c_strategy", static_cast(107) }, + { "ZSTD_c_enableLongDistanceMatching", static_cast(160) }, + { "ZSTD_c_ldmHashLog", static_cast(161) }, + { "ZSTD_c_ldmMinMatch", static_cast(162) }, + { "ZSTD_c_ldmBucketSizeLog", static_cast(163) }, + { "ZSTD_c_ldmHashRateLog", static_cast(164) }, + { "ZSTD_c_contentSizeFlag", static_cast(200) }, + { "ZSTD_c_checksumFlag", static_cast(201) }, + { "ZSTD_c_dictIDFlag", static_cast(202) }, + { "ZSTD_c_nbWorkers", static_cast(400) }, + { "ZSTD_c_jobSize", static_cast(401) }, + { "ZSTD_c_overlapLog", static_cast(402) }, + { "ZSTD_d_windowLogMax", static_cast(100) }, + { "ZSTD_CLEVEL_DEFAULT", static_cast(3) }, + { "ZSTD_error_no_error", static_cast(0) }, + { "ZSTD_error_GENERIC", static_cast(1) }, + { "ZSTD_error_prefix_unknown", static_cast(10) }, + { "ZSTD_error_version_unsupported", static_cast(12) }, + { "ZSTD_error_frameParameter_unsupported", static_cast(14) }, + { "ZSTD_error_frameParameter_windowTooLarge", static_cast(16) }, + { "ZSTD_error_corruption_detected", static_cast(20) }, + { "ZSTD_error_checksum_wrong", static_cast(22) }, + { "ZSTD_error_literals_headerWrong", static_cast(24) }, + { "ZSTD_error_dictionary_corrupted", static_cast(30) }, + { "ZSTD_error_dictionary_wrong", static_cast(32) }, + { "ZSTD_error_dictionaryCreation_failed", static_cast(34) }, + { "ZSTD_error_parameter_unsupported", static_cast(40) }, + { "ZSTD_error_parameter_combination_unsupported", static_cast(41) }, + { "ZSTD_error_parameter_outOfBound", static_cast(42) }, + { "ZSTD_error_tableLog_tooLarge", static_cast(44) }, + { "ZSTD_error_maxSymbolValue_tooLarge", static_cast(46) }, + { "ZSTD_error_maxSymbolValue_tooSmall", static_cast(48) }, + { "ZSTD_error_stabilityCondition_notRespected", static_cast(50) }, + { "ZSTD_error_stage_wrong", static_cast(60) }, + { "ZSTD_error_init_missing", static_cast(62) }, + { "ZSTD_error_memory_allocation", static_cast(64) }, + { "ZSTD_error_workSpace_tooSmall", static_cast(66) }, + { "ZSTD_error_dstSize_tooSmall", static_cast(70) }, + { "ZSTD_error_srcSize_wrong", static_cast(72) }, + { "ZSTD_error_dstBuffer_null", static_cast(74) }, + { "ZSTD_error_noForwardProgress_destFull", static_cast(80) }, + { "ZSTD_error_noForwardProgress_inputEmpty", static_cast(82) }, { nullptr, 0 }, }; putNumericConstants(vm, object, kConstants12); diff --git a/src/jsc/bindings/bun_icu_decompress.cpp b/src/jsc/bindings/bun_icu_decompress.cpp index d17798d3023f..8a8c09c5a5ee 100644 --- a/src/jsc/bindings/bun_icu_decompress.cpp +++ b/src/jsc/bindings/bun_icu_decompress.cpp @@ -34,16 +34,19 @@ #include #include -#define ZSTD_STATIC_LINKING_ONLY -#include - -static_assert(ZSTD_MAGICNUMBER == 0xFD2FB528); +static constexpr uint32_t ZSTD_MAGICNUMBER = 0xFD2FB528; // Raw ICU items have bytes[2..3] == {0xda, 0x27} (ucmndata.h MAGIC1/MAGIC2), // so their first u32 is 0x27da'hhhh — cannot collide with zstd's magic. extern "C" __attribute__((weak)) const unsigned char bun_icu_zstd_dict[]; extern "C" __attribute__((weak)) const unsigned int bun_icu_zstd_dict_size; +// src/runtime/icu_decompress.rs +struct IcuDecompressor; +extern "C" IcuDecompressor* Bun__IcuDecompressor__create(const unsigned char* dict, size_t dictSize); +extern "C" bool Bun__IcuDecompressor__frameSizes(const void* src, size_t bound, size_t* compressedSize, uint64_t* contentSize); +extern "C" bool Bun__IcuDecompressor__decompress(IcuDecompressor*, void* dst, size_t dstSize, const void* src, size_t srcSize); + namespace Bun { class ICUDecompressor { @@ -61,15 +64,13 @@ class ICUDecompressor { Locker locker { m_lock }; if (auto it = m_cache.find(p); it != m_cache.end()) { - *length = static_cast(ZSTD_getFrameContentSize(p, frameBound(*length))); - return it->value; + *length = it->value.second; + return it->value.first; } - size_t clen = ZSTD_findFrameCompressedSize(p, frameBound(*length)); - if (ZSTD_isError(clen)) - return p; - auto dlen = ZSTD_getFrameContentSize(p, clen); - if (dlen == ZSTD_CONTENTSIZE_UNKNOWN || dlen == ZSTD_CONTENTSIZE_ERROR) + size_t clen; + uint64_t dlen; + if (!Bun__IcuDecompressor__frameSizes(p, frameBound(*length), &clen, &dlen)) return p; // tryAlignedMalloc asserts size is a multiple of alignment in debug @@ -78,25 +79,23 @@ class ICUDecompressor { void* buf = MimallocMalloc::tryAlignedMalloc(alloc, 16); if (!buf) return p; - size_t r = m_ddict - ? ZSTD_decompress_usingDDict(m_dctx, buf, static_cast(dlen), p, clen, m_ddict) - : ZSTD_decompressDCtx(m_dctx, buf, static_cast(dlen), p, clen); - if (ZSTD_isError(r)) { + // The decoder takes initialized memory. + std::memset(buf, 0, alloc); + if (!Bun__IcuDecompressor__decompress(m_decompressor, buf, static_cast(dlen), p, clen)) { MimallocMalloc::free(buf); return p; } - m_cache.add(p, buf); *length = static_cast(dlen); + m_cache.add(p, std::pair { buf, *length }); return buf; } private: ICUDecompressor() - : m_dctx(ZSTD_createDCtx()) - , m_ddict(&bun_icu_zstd_dict_size && bun_icu_zstd_dict_size - ? ZSTD_createDDict_byReference(bun_icu_zstd_dict, bun_icu_zstd_dict_size) - : nullptr) + : m_decompressor(&bun_icu_zstd_dict_size && bun_icu_zstd_dict_size + ? Bun__IcuDecompressor__create(bun_icu_zstd_dict, bun_icu_zstd_dict_size) + : Bun__IcuDecompressor__create(nullptr, 0)) { } @@ -105,9 +104,8 @@ class ICUDecompressor { friend class WTF::LazyNeverDestroyed; WTF::Lock m_lock; - WTF::HashMap m_cache WTF_GUARDED_BY_LOCK(m_lock); - ZSTD_DCtx* const m_dctx; - ZSTD_DDict* const m_ddict; + WTF::HashMap> m_cache WTF_GUARDED_BY_LOCK(m_lock); + IcuDecompressor* const m_decompressor; }; } // namespace Bun diff --git a/src/runtime/api/BunObject.rs b/src/runtime/api/BunObject.rs index d14211cb1072..2704fcde9463 100644 --- a/src/runtime/api/BunObject.rs +++ b/src/runtime/api/BunObject.rs @@ -1952,25 +1952,10 @@ pub(crate) mod JSZstd { /// Boxed (trimmed to the bytes produced) so an empty result owns no memory, as `create_buffer_from_box` requires. fn compress_to_box(input: &[u8], level: i32) -> Result, Failure> { - let max_size = bun_zstd::compress_bound(input.len()); - // `ZSTD_compressBound` returns an error code for inputs over `ZSTD_MAX_INPUT_SIZE`. - if bun_zstd::is_error(max_size) { - return Err(Failure::Compression(b"Input is too large to compress")); + match bun_zstd::compress_alloc(input, level) { + Ok(output) => Ok(output.into_boxed_slice()), + Err(err) => Err(Failure::Compression(err.message().to_bytes())), } - - // Reserved, not zero-filled: zstd initializes exactly the bytes it reports. - let mut output: Vec = Vec::new(); - output - .try_reserve_exact(max_size) - .map_err(|_| Failure::OutOfMemory)?; - - if let bun_zstd::Result::Err(err) = - bun_zstd::compress_append(&mut output, input, Some(level)) - { - return Err(Failure::Compression(err.as_bytes())); - } - - Ok(output.into_boxed_slice()) } fn decompress_to_box(input: &[u8]) -> Result, Failure> { diff --git a/src/runtime/icu_decompress.rs b/src/runtime/icu_decompress.rs new file mode 100644 index 000000000000..1abababddfa3 --- /dev/null +++ b/src/runtime/icu_decompress.rs @@ -0,0 +1,77 @@ +//! The zstd side of `bun_icu_decompress.cpp`. + +use core::ffi::c_void; + +use bun_zstd::{DCtx, DDict, FrameExtent}; + +pub(crate) struct IcuDecompressor { + dctx: DCtx, + ddict: Option>, +} + +/// `dict` is the `dict_size` bytes of a dictionary that lives as long as the process, or null. +#[unsafe(no_mangle)] +pub(crate) extern "C" fn Bun__IcuDecompressor__create( + dict: *const u8, + dict_size: usize, +) -> *mut IcuDecompressor { + let ddict = if dict.is_null() || dict_size == 0 { + None + } else { + // SAFETY: the caller's contract. + DDict::by_reference(unsafe { core::slice::from_raw_parts(dict, dict_size) }).ok() + }; + bun_core::heap::into_raw(Box::new(IcuDecompressor { + dctx: DCtx::new(), + ddict, + })) +} + +/// The sizes of the frame that starts at `src`, compressed and decompressed. False if it is not a frame that says its +/// size, or if it is more than `bound` bytes. +#[unsafe(no_mangle)] +pub(crate) extern "C" fn Bun__IcuDecompressor__frameSizes( + src: *const u8, + bound: usize, + compressed_size: &mut usize, + content_size: &mut u64, +) -> bool { + // `bound` can be past the end of the data, so this only looks at the bytes that the frame says it has. + let mut known = 0; + loop { + // SAFETY: `src` starts a frame, which is at least `known` bytes. + let frame = unsafe { core::slice::from_raw_parts(src, known) }; + match bun_zstd::frame_extent(frame) { + Ok(FrameExtent::AtLeast(size)) if size > known && size <= bound => known = size, + Ok(FrameExtent::Whole(size)) => { + *compressed_size = size; + *content_size = bun_zstd::get_frame_content_size(frame); + return *content_size < bun_zstd::ZSTD_CONTENTSIZE_ERROR; + } + _ => return false, + } + } +} + +/// Decompresses the frame `src` to `dst`, which is initialized and of the size that the frame says. +#[unsafe(no_mangle)] +pub(crate) extern "C" fn Bun__IcuDecompressor__decompress( + this: &mut IcuDecompressor, + dst: *mut c_void, + dst_size: usize, + src: *const u8, + src_size: usize, +) -> bool { + // SAFETY: the caller's contract. + let (dst, src) = unsafe { + ( + core::slice::from_raw_parts_mut(dst.cast::(), dst_size), + core::slice::from_raw_parts(src, src_size), + ) + }; + match &this.ddict { + Some(ddict) => this.dctx.decompress_using_ddict(dst, src, ddict), + None => this.dctx.decompress(dst, src), + } + .is_ok() +} diff --git a/src/runtime/lib.rs b/src/runtime/lib.rs index c80555447ea1..c78e815e2f9a 100644 --- a/src/runtime/lib.rs +++ b/src/runtime/lib.rs @@ -53,6 +53,8 @@ pub(crate) mod shell; pub(crate) mod api; pub(crate) mod dispatch; pub(crate) mod hw_exports; +#[cfg(any(target_os = "linux", target_os = "android", windows))] +pub(crate) mod icu_decompress; #[cfg(debug_assertions)] pub(crate) mod imported_class_test; pub(crate) mod ipc; diff --git a/src/runtime/node/zlib/NativeZstd.rs b/src/runtime/node/zlib/NativeZstd.rs index 186e1b4648f7..924578bd52a7 100644 --- a/src/runtime/node/zlib/NativeZstd.rs +++ b/src/runtime/node/zlib/NativeZstd.rs @@ -2,13 +2,13 @@ pub(crate) use _impl::NativeZstd; mod _impl { use core::cell::Cell; - use core::ffi::{c_int, c_uint, c_void}; + use core::ffi::{c_int, c_uint}; use core::ptr; use bun_jsc::bindgen::{JsLocal, JsScope, JsWrapper}; use bun_jsc::{self as jsc, JSGlobalObject, JsCell, JsResult, StrongOptional, WorkPoolTask}; use bun_ptr::RefPtr; - use bun_zstd::c; // `bun.c` translated-c-headers (ZSTD_* fns/consts live here) + use bun_zstd::{CCtx, DCtx, EndDirective, InBuffer, OutBuffer}; use crate::node::node_zlib_binding::{CompressionStreamImpl as _, CountedKeepAlive, Error}; use crate::node::util::validators; @@ -243,25 +243,22 @@ mod _impl { } } - // `poll_ref` and `this_value` (Strong) clean up via their own Drop impls. - impl Drop for NativeZstd { - fn drop(&mut self) { - self.stream.with_mut(|s| match s.mode { - NodeMode::ZSTD_COMPRESS | NodeMode::ZSTD_DECOMPRESS => s.close(), - _ => {} - }); - } + enum State { + Compress(Box), + Decompress(Box), } pub(crate) struct Context { pub(crate) mode: NodeMode, - // LIFETIMES.tsv: FFI → Option<*mut c_void> (ZSTD_createCCtx/DCtx; freed in deinit_state) - pub(crate) state: Option<*mut c_void>, - pub flush: c_int, - pub(crate) input: c::ZSTD_inBuffer, - pub(crate) output: c::ZSTD_outBuffer, - pub(crate) pledged_src_size: u64, - pub(crate) remaining: u64, + state: Option, + flush: EndDirective, + // What `set_buffers` was given, which the caller keeps alive until the write is over. + input: *const [u8], + input_pos: usize, + output: *mut [u8], + output_pos: usize, + pledged_src_size: u64, + error: Option, } impl Default for Context { @@ -269,108 +266,55 @@ mod _impl { Self { mode: NodeMode::NONE, state: None, - flush: c::ZSTD_e_continue as c_int, - input: c::ZSTD_inBuffer { - src: ptr::null(), - size: 0, - pos: 0, - }, - output: c::ZSTD_outBuffer { - dst: ptr::null_mut(), - size: 0, - pos: 0, - }, + flush: EndDirective::Continue, + input: ptr::slice_from_raw_parts(ptr::dangling(), 0), + input_pos: 0, + output: ptr::slice_from_raw_parts_mut(ptr::dangling_mut(), 0), + output_pos: 0, pledged_src_size: u64::MAX, - remaining: 0, + error: None, } } } impl Context { - /// zstd copies the dictionary into the context, so `dictionary` does not - /// need to outlive this call (unlike brotli's attach APIs). - fn load_dictionary( - dictionary: Option<&[u8]>, - load: impl FnOnce(*const c_void, usize) -> usize, - ) -> Error { - let Some(dict) = dictionary.filter(|d| !d.is_empty()) else { - return Error::OK; - }; - let result = load(dict.as_ptr().cast(), dict.len()); - if c::ZSTD_isError(result) > 0 { - return Error::init( - c"Failed to load zstd dictionary".as_ptr(), - -1, - c"ERR_ZLIB_DICTIONARY_LOAD_FAILED".as_ptr(), - ); - } - Error::OK - } + const DICTIONARY_LOAD_FAILED: Error = Error::init( + c"Failed to load zstd dictionary".as_ptr(), + -1, + c"ERR_ZLIB_DICTIONARY_LOAD_FAILED".as_ptr(), + ); + /// The dictionary is copied into the context, so `dictionary` does not + /// need to outlive this call (unlike brotli's attach APIs). pub(crate) fn init(&mut self, pledged_src_size: u64, dictionary: Option<&[u8]>) -> Error { // Mirrors node's `state_.reset(ZSTD_createCCtx())`: free the previous // context first. `init` is JS-reachable twice on one handle. - if self.state.is_some() { - self.deinit_state(); - } + self.state = None; + let dictionary = dictionary.unwrap_or_default(); match self.mode { NodeMode::ZSTD_COMPRESS => { self.pledged_src_size = pledged_src_size; - let state = c::ZSTD_createCCtx(); - if state.is_null() { - return Error::init( - c"Could not initialize zstd instance".as_ptr(), - -1, - c"ERR_ZLIB_INITIALIZATION_FAILED".as_ptr(), - ); - } - self.state = Some(state.cast()); + let mut state = Box::new(CCtx::new()); // Node loads the dictionary before setting the pledged size. - // SAFETY: FFI — state is a valid CCtx allocated above. - let err = Self::load_dictionary(dictionary, |d, n| unsafe { - c::ZSTD_CCtx_loadDictionary(state, d, n) - }); - if err.is_error() { - // SAFETY: state is a valid CCtx allocated above. - let _ = unsafe { c::ZSTD_freeCCtx(state) }; - self.state = None; - return err; + if state.load_dictionary(dictionary).is_err() { + return Self::DICTIONARY_LOAD_FAILED; } - // SAFETY: state is non-null (checked above). - let result = - unsafe { c::ZSTD_CCtx_setPledgedSrcSize(state, pledged_src_size as _) }; - if c::ZSTD_isError(result) > 0 { - // SAFETY: state is a valid CCtx allocated above. - let _ = unsafe { c::ZSTD_freeCCtx(state) }; - self.state = None; + if state.set_pledged_src_size(pledged_src_size).is_err() { return Error::init( c"Could not set pledged src size".as_ptr(), -1, c"ERR_ZLIB_INITIALIZATION_FAILED".as_ptr(), ); } + self.state = Some(State::Compress(state)); Error::OK } NodeMode::ZSTD_DECOMPRESS => { - let state = c::ZSTD_createDCtx(); - if state.is_null() { - return Error::init( - c"Could not initialize zstd instance".as_ptr(), - -1, - c"ERR_ZLIB_INITIALIZATION_FAILED".as_ptr(), - ); - } - self.state = Some(state.cast()); - // SAFETY: FFI — state is a valid DCtx allocated above. - let err = Self::load_dictionary(dictionary, |d, n| unsafe { - c::ZSTD_DCtx_loadDictionary(state, d, n) - }); - if err.is_error() { - // SAFETY: state is a valid DCtx allocated above. - let _ = unsafe { c::ZSTD_freeDCtx(state) }; - self.state = None; - return err; + let mut state = Box::new(DCtx::new()); + if state.load_dictionary(dictionary).is_err() { + return Self::DICTIONARY_LOAD_FAILED; } + self.state = Some(State::Decompress(state)); Error::OK } _ => unreachable!(), @@ -378,37 +322,20 @@ mod _impl { } pub(crate) fn set_params(&mut self, key: c_uint, value: u32) -> Error { - match self.mode { - NodeMode::ZSTD_COMPRESS => { - // SAFETY: state is a valid CCtx set by init(); @bitCast u32→c_int is a same-size reinterpret. - let result = unsafe { - c::ZSTD_CCtx_setParameter(self.state_ptr().cast(), key, value as c_int) - }; - if c::ZSTD_isError(result) > 0 { - return Error::init( - c"Setting parameter failed".as_ptr(), - -1, - c"ERR_ZSTD_PARAM_SET_FAILED".as_ptr(), - ); - } - Error::OK - } - NodeMode::ZSTD_DECOMPRESS => { - // SAFETY: state is a valid DCtx set by init(). - let result = unsafe { - c::ZSTD_DCtx_setParameter(self.state_ptr().cast(), key, value as c_int) - }; - if c::ZSTD_isError(result) > 0 { - return Error::init( - c"Setting parameter failed".as_ptr(), - -1, - c"ERR_ZSTD_PARAM_SET_FAILED".as_ptr(), - ); - } - Error::OK - } - _ => unreachable!(), + let (key, value) = (key as i32, value as i32); + let result = match &mut self.state { + Some(State::Compress(state)) => state.set_parameter(key, value).map(drop), + Some(State::Decompress(state)) => state.set_parameter(key, value), + None => unreachable!(), + }; + if result.is_err() { + return Error::init( + c"Setting parameter failed".as_ptr(), + -1, + c"ERR_ZSTD_PARAM_SET_FAILED".as_ptr(), + ); } + Error::OK } pub(crate) fn reset(&mut self) -> Error { @@ -418,34 +345,11 @@ mod _impl { self.init(self.pledged_src_size, None) } - /// Frees the Zstd encoder/decoder state without changing mode. - /// Use close() for full cleanup that also sets mode to NONE. - fn deinit_state(&mut self) { - let _ = match self.mode { - // SAFETY: state was allocated by ZSTD_create{C,D}Ctx and not yet freed. - NodeMode::ZSTD_COMPRESS => unsafe { c::ZSTD_freeCCtx(self.state_ptr().cast()) }, - // SAFETY: state was allocated by ZSTD_createDCtx and not yet freed. - NodeMode::ZSTD_DECOMPRESS => unsafe { c::ZSTD_freeDCtx(self.state_ptr().cast()) }, - _ => unreachable!(), - }; - self.state = None; - } - pub(crate) fn set_buffers(&mut self, in_: Option<&[u8]>, out: Option<&mut [u8]>) { - self.input.src = in_.map_or(ptr::null(), |p| p.as_ptr().cast()); - self.input.size = in_.map_or(0, |p| p.len()); - self.input.pos = 0; - match out { - Some(p) => { - self.output.size = p.len(); - self.output.dst = p.as_mut_ptr().cast(); - } - None => { - self.output.size = 0; - self.output.dst = ptr::null_mut(); - } - } - self.output.pos = 0; + self.input = in_.unwrap_or_default(); + self.input_pos = 0; + self.output = out.unwrap_or_default(); + self.output_pos = 0; } pub(crate) fn flush_value_is_valid(flush: u32) -> bool { @@ -453,165 +357,88 @@ mod _impl { } pub(crate) fn set_flush(&mut self, flush: c_int) { - self.flush = flush; + self.flush = match flush { + 0 => EndDirective::Continue, + 1 => EndDirective::Flush, + 2 => EndDirective::End, + n => unreachable!("invalid ZSTD_EndDirective {n}"), + }; } const ZSTD_MAGICNUMBER: [u8; 4] = 0xFD2FB528u32.to_le_bytes(); const ZSTD_MAGIC_SKIPPABLE: [u8; 4] = 0x184D2A50u32.to_le_bytes(); - /// True when the unconsumed input begins a zstd/skippable frame magic (or a prefix of one). - fn next_input_is_frame(&self) -> bool { - let n = (self.input.size - self.input.pos).min(4); + /// True when `input` begins a zstd/skippable frame magic (or a prefix of one). + fn is_frame_start(input: &[u8]) -> bool { + let n = input.len().min(4); if n == 0 { return false; } - let mut head = [0u8; 4]; - // SAFETY: input.src[pos..pos+n] lies within the slice installed by set_buffers. - unsafe { - ptr::copy_nonoverlapping( - self.input.src.cast::().add(self.input.pos), - head.as_mut_ptr(), - n, - ); - } - head[..n] == Self::ZSTD_MAGICNUMBER[..n] + let head = &input[..n]; + *head == Self::ZSTD_MAGICNUMBER[..n] || (head[0] & 0xF0 == Self::ZSTD_MAGIC_SKIPPABLE[0] - && head[1..n] == Self::ZSTD_MAGIC_SKIPPABLE[1..n]) + && head[1..] == Self::ZSTD_MAGIC_SKIPPABLE[1..n]) } pub(crate) fn do_work(&mut self) { - // A handle driven before `init()` has no CCtx/DCtx; zstd - // dereferences the context pointer unconditionally. - if self.state.is_none() { + // A handle driven before `init()` has no context. + let Some(state) = &mut self.state else { return; - } - self.remaining = match self.mode { - // SAFETY: state is a valid CCtx; input/output point to caller-kept-alive buffers (set_buffers). - NodeMode::ZSTD_COMPRESS => unsafe { - c::ZSTD_compressStream2( - self.state_ptr().cast(), - &raw mut self.output, - &raw mut self.input, - // cast c_int → ZSTD_EndDirective (c_uint) - self.flush as c_uint, - ) - }, - NodeMode::ZSTD_DECOMPRESS => { - // SAFETY: state is a valid DCtx. - let mut ret = unsafe { - c::ZSTD_decompressStream( - self.state_ptr().cast(), - &raw mut self.output, - &raw mut self.input, - ) - }; - // ret == 0 is frame-complete; mirrors NativeZlib::do_work_inflate's GUNZIP loop. - while ret == 0 - && self.output.pos < self.output.size - && self.next_input_is_frame() + }; + let mut input = InBuffer { + // SAFETY: `set_buffers` got it from a slice that the caller keeps alive until the write is over. + src: unsafe { &*self.input }, + pos: self.input_pos, + }; + let mut output = OutBuffer { + // SAFETY: as above, and nothing else writes to it until then. + dst: unsafe { &mut *self.output }, + pos: self.output_pos, + }; + let result = match state { + State::Compress(state) => { + state.compress_stream2(&mut output, &mut input, self.flush) + } + State::Decompress(state) => { + let mut ret = state.decompress_stream(&mut output, &mut input); + // 0 is frame-complete; mirrors NativeZlib::do_work_inflate's GUNZIP loop. + while ret == Ok(0) + && output.pos < output.dst.len() + && Self::is_frame_start(&input.src[input.pos..]) { - // SAFETY: state is a valid DCtx; input/output point to caller-kept-alive buffers. - ret = unsafe { - c::ZSTD_decompressStream( - self.state_ptr().cast(), - &raw mut self.output, - &raw mut self.input, - ) - }; + ret = state.decompress_stream(&mut output, &mut input); } ret } - _ => unreachable!(), - } as u64; + }; + self.input_pos = input.pos; + self.output_pos = output.pos; + self.error = result.err(); } pub(crate) fn write_result(&self) -> [u32; 2] { [ - self.output.size - self.output.pos, - self.input.size - self.input.pos, + self.output.len() - self.output_pos, + self.input.len() - self.input_pos, ] .map(|n| u32::try_from(n).expect("int cast")) } pub(crate) fn get_error_info(&mut self) -> Error { - // Compute result, then clear `remaining`, then return. - let err = c::ZSTD_getErrorCode(self.remaining as usize); - let result = if err == 0 { - Error::OK - } else { - Error { + match self.error.take() { + None => Error::OK, + Some(err) => Error { err: err as c_int, - msg: c::ZSTD_getErrorString(err), - code: match err { - c::ZSTD_error_no_error => c"ZSTD_error_no_error", - c::ZSTD_error_GENERIC => c"ZSTD_error_GENERIC", - c::ZSTD_error_prefix_unknown => c"ZSTD_error_prefix_unknown", - c::ZSTD_error_version_unsupported => c"ZSTD_error_version_unsupported", - c::ZSTD_error_frameParameter_unsupported => { - c"ZSTD_error_frameParameter_unsupported" - } - c::ZSTD_error_frameParameter_windowTooLarge => { - c"ZSTD_error_frameParameter_windowTooLarge" - } - c::ZSTD_error_corruption_detected => c"ZSTD_error_corruption_detected", - c::ZSTD_error_checksum_wrong => c"ZSTD_error_checksum_wrong", - c::ZSTD_error_literals_headerWrong => c"ZSTD_error_literals_headerWrong", - c::ZSTD_error_dictionary_corrupted => c"ZSTD_error_dictionary_corrupted", - c::ZSTD_error_dictionary_wrong => c"ZSTD_error_dictionary_wrong", - c::ZSTD_error_dictionaryCreation_failed => { - c"ZSTD_error_dictionaryCreation_failed" - } - c::ZSTD_error_parameter_unsupported => c"ZSTD_error_parameter_unsupported", - c::ZSTD_error_parameter_combination_unsupported => { - c"ZSTD_error_parameter_combination_unsupported" - } - c::ZSTD_error_parameter_outOfBound => c"ZSTD_error_parameter_outOfBound", - c::ZSTD_error_tableLog_tooLarge => c"ZSTD_error_tableLog_tooLarge", - c::ZSTD_error_maxSymbolValue_tooLarge => { - c"ZSTD_error_maxSymbolValue_tooLarge" - } - c::ZSTD_error_maxSymbolValue_tooSmall => { - c"ZSTD_error_maxSymbolValue_tooSmall" - } - c::ZSTD_error_stabilityCondition_notRespected => { - c"ZSTD_error_stabilityCondition_notRespected" - } - c::ZSTD_error_stage_wrong => c"ZSTD_error_stage_wrong", - c::ZSTD_error_init_missing => c"ZSTD_error_init_missing", - c::ZSTD_error_memory_allocation => c"ZSTD_error_memory_allocation", - c::ZSTD_error_workSpace_tooSmall => c"ZSTD_error_workSpace_tooSmall", - c::ZSTD_error_dstSize_tooSmall => c"ZSTD_error_dstSize_tooSmall", - c::ZSTD_error_srcSize_wrong => c"ZSTD_error_srcSize_wrong", - c::ZSTD_error_dstBuffer_null => c"ZSTD_error_dstBuffer_null", - c::ZSTD_error_noForwardProgress_destFull => { - c"ZSTD_error_noForwardProgress_destFull" - } - c::ZSTD_error_noForwardProgress_inputEmpty => { - c"ZSTD_error_noForwardProgress_inputEmpty" - } - _ => c"ZSTD_error_GENERIC", - } - .as_ptr(), - } - }; - self.remaining = 0; - result + msg: err.message().as_ptr(), + code: err.name().as_ptr(), + }, + } } pub(crate) fn close(&mut self) { - // Idempotent: a handle that was never (successfully) initialized, - // or that was already closed, has no CCtx/DCtx to free. - if self.state.is_some() { - self.deinit_state(); - } + self.state = None; self.mode = NodeMode::NONE; } - - #[inline] - fn state_ptr(&self) -> *mut c_void { - // Passes null through if unset. - self.state.unwrap_or(ptr::null_mut()) - } } crate::__impl_compression_stream!(NativeZstd, Context, js_NativeZstd); diff --git a/src/runtime/webcore/CompressionStreamCoder.rs b/src/runtime/webcore/CompressionStreamCoder.rs index 1d22e6e6b50f..560feb5e9af5 100644 --- a/src/runtime/webcore/CompressionStreamCoder.rs +++ b/src/runtime/webcore/CompressionStreamCoder.rs @@ -25,7 +25,7 @@ use bun_jsc::{ErrorCode, JSGlobalObject, JSUint8Array, JSValue, PinnedArrayBuffe use bun_brotli::c as brotli; use bun_zlib as zlib; -use bun_zstd::c as zstd; +use bun_zstd as zstd; /// Matches `Bun::WebStreams::CompressionFormat` in `StreamsForward.h`. #[repr(u8)] @@ -74,9 +74,17 @@ fn spare(out: &mut Vec, cap: usize) -> Result<&mut [core::mem::MaybeUninit CodecError { - if zstd::ZSTD_getErrorCode(rc) == zstd::ZSTD_error_memory_allocation { +/// [`spare`], as zeros at the end of `out`, for a codec that takes initialized memory. Returns where they start: +/// the caller truncates `out` to what was written. +fn zeroed_spare(out: &mut Vec, cap: usize) -> Result { + let start = out.len(); + let len = spare(out, cap)?.len(); + out.resize(start + len, 0); + Ok(start) +} + +fn zstd_error(code: zstd::ErrorCode, message: &'static str) -> CodecError { + if code == zstd::ErrorCode::MemoryAllocation { CodecError::OutOfMemory } else { CodecError::Message(message) @@ -113,8 +121,8 @@ enum Backend { }, BrotliEncode(NonNull), BrotliDecode(NonNull), - ZstdEncode(NonNull), - ZstdDecode(NonNull), + ZstdEncode(Box), + ZstdDecode(Box), } #[derive(bun_ptr::ThreadSafeRefCounted)] @@ -166,12 +174,7 @@ impl Drop for CompressionStreamCoder { } Backend::BrotliEncode(p) => brotli::BrotliEncoderDestroyInstance(p.as_ptr()), Backend::BrotliDecode(p) => brotli::BrotliDecoderDestroyInstance(p.as_ptr()), - Backend::ZstdEncode(p) => { - zstd::ZSTD_freeCCtx(p.as_ptr()); - } - Backend::ZstdDecode(p) => { - zstd::ZSTD_freeDCtx(p.as_ptr()); - } + Backend::ZstdEncode(_) | Backend::ZstdDecode(_) => {} } } } @@ -252,26 +255,15 @@ impl CompressionStreamCoder { Backend::BrotliDecode(p) } (Format::Zstd, false) => { - let p = NonNull::new(zstd::ZSTD_createCCtx()) - .ok_or(CodecError::Message("failed to initialize zstd encoder"))?; - if let Some(lvl) = level { - // SAFETY: `p` was just created and is exclusively owned here. - let rc = unsafe { - zstd::ZSTD_CCtx_setParameter(p.as_ptr(), zstd::ZSTD_c_compressionLevel, lvl) - }; - if zstd::ZSTD_isError(rc) != 0 { - // SAFETY: `p` was created above and not stored anywhere. - unsafe { zstd::ZSTD_freeCCtx(p.as_ptr()) }; - return Err(CodecError::Message("failed to set zstd level")); - } + let mut cctx = Box::new(zstd::CCtx::new()); + if let Some(lvl) = level + && (cctx.set_parameter(zstd::c_param::COMPRESSION_LEVEL, lvl)).is_err() + { + return Err(CodecError::Message("failed to set zstd level")); } - Backend::ZstdEncode(p) - } - (Format::Zstd, true) => { - let p = NonNull::new(zstd::ZSTD_createDCtx()) - .ok_or(CodecError::Message("failed to initialize zstd decoder"))?; - Backend::ZstdDecode(p) + Backend::ZstdEncode(cctx) } + (Format::Zstd, true) => Backend::ZstdDecode(Box::new(zstd::DCtx::new())), }; Ok(Box::new(Self { backend, @@ -596,12 +588,14 @@ impl CompressionStreamCoder { } } } - Backend::ZstdEncode(p) => { - // ZSTD_EndDirective: 0 = ZSTD_e_continue, 2 = ZSTD_e_end. - let end: core::ffi::c_uint = if finish { 2 } else { 0 }; - let mut input_buf = zstd::ZSTD_inBuffer { - src: input.as_ptr().cast(), - size: input.len(), + Backend::ZstdEncode(cctx) => { + let end = if finish { + zstd::EndDirective::End + } else { + zstd::EndDirective::Continue + }; + let mut input_buf = zstd::InBuffer { + src: input, pos: 0, }; loop { @@ -610,39 +604,26 @@ impl CompressionStreamCoder { consumed: input_buf.pos, }); } - let spare = spare(out, cap)?; - let mut output_buf = zstd::ZSTD_outBuffer { - dst: spare.as_mut_ptr().cast(), - size: spare.len(), + let start = zeroed_spare(out, cap)?; + let mut output_buf = zstd::OutBuffer { + dst: &mut out[start..], pos: 0, }; - // SAFETY: `p` is a live CCtx; the buffers borrow locals / - // spare for this one call. - let remaining = unsafe { - zstd::ZSTD_compressStream2( - p.as_ptr(), - &raw mut output_buf, - &raw mut input_buf, - end, - ) - }; - // SAFETY: compressStream2 wrote exactly `output_buf.pos` - // bytes. - unsafe { out.set_len(out.len() + output_buf.pos) }; - if zstd::ZSTD_isError(remaining) != 0 { - return Err(zstd_error(remaining, "zstd encode failed")); - } - if input_buf.pos == input_buf.size && (!finish || remaining == 0) { + let remaining = cctx.compress_stream2(&mut output_buf, &mut input_buf, end); + let written = output_buf.pos; + out.truncate(start + written); + let remaining = + remaining.map_err(|code| zstd_error(code, "zstd encode failed"))?; + if input_buf.pos == input.len() && (!finish || remaining == 0) { return Ok(Progress::Done); } } } - Backend::ZstdDecode(p) => { + Backend::ZstdDecode(dctx) => { // After a frame completes, whatever follows must be another frame // magic (see `zstd_head`); `step` has already re-attached a split one. - let mut input_buf = zstd::ZSTD_inBuffer { - src: input.as_ptr().cast(), - size: input.len(), + let mut input_buf = zstd::InBuffer { + src: input, pos: 0, }; loop { @@ -663,13 +644,8 @@ impl CompressionStreamCoder { self.zstd_head_len = head.len() as u8; return Ok(Progress::Done); } - // SAFETY: `p` is a live DCtx. - unsafe { - zstd::ZSTD_DCtx_reset( - p.as_ptr(), - zstd::ZSTD_reset_session_and_parameters, - ) - }; + // Between frames, this cannot fail. + let _ = dctx.reset(zstd::ResetDirective::SessionAndParameters); self.ended = false; } if out.len() >= cap { @@ -677,32 +653,21 @@ impl CompressionStreamCoder { consumed: input_buf.pos, }); } - let spare = spare(out, cap)?; - let mut output_buf = zstd::ZSTD_outBuffer { - dst: spare.as_mut_ptr().cast(), - size: spare.len(), + let start = zeroed_spare(out, cap)?; + let mut output_buf = zstd::OutBuffer { + dst: &mut out[start..], pos: 0, }; - // SAFETY: `p` is a live DCtx; buffers borrow locals / spare - // for this one call. - let remaining = unsafe { - zstd::ZSTD_decompressStream( - p.as_ptr(), - &raw mut output_buf, - &raw mut input_buf, - ) - }; - // SAFETY: decompressStream wrote exactly `output_buf.pos` - // bytes. - unsafe { out.set_len(out.len() + output_buf.pos) }; - if zstd::ZSTD_isError(remaining) != 0 { - return Err(zstd_error(remaining, "zstd decode failed")); - } + let remaining = dctx.decompress_stream(&mut output_buf, &mut input_buf); + let (written, room) = (output_buf.pos, output_buf.dst.len()); + out.truncate(start + written); + let remaining = + remaining.map_err(|code| zstd_error(code, "zstd decode failed"))?; if remaining == 0 { self.ended = true; continue; } - if input_buf.pos == input_buf.size && output_buf.pos < output_buf.size { + if input_buf.pos == input.len() && written < room { if finish { return Err(CodecError::Message("unexpected end of file")); } diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs index c629839ae9ea..249d29d402d3 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs @@ -3557,21 +3557,11 @@ pub(crate) fn serialize_json_source_map_for_standalone( let offset = string_payload.len(); - let bound = bun_zstd::compress_bound(utf8.len()); - // `ZSTD_compressBound` returns an *error code* (a value near - // `usize::MAX`) when the input size exceeds `ZSTD_MAX_INPUT_SIZE`; - // feeding that to `Vec::reserve` below would abort with a capacity - // overflow instead of failing the build. - if bun_zstd::is_error(bound) { - return Err(crate::Error::SourceMapTooLarge); - } - string_payload.reserve(bound); - if let bun_zstd::Result::Err(err_msg) = - bun_zstd::compress_append(string_payload, utf8, Some(1)) - { + string_payload.reserve(bun_zstd::compress_bound(utf8.len())); + if let Err(err) = bun_zstd::compress_append(string_payload, utf8, 1) { Output::panic(format_args!( "Unexpected error compressing sourcemap: {}", - bstr::BStr::new(err_msg.as_bytes()) + bstr::BStr::new(err.message().to_bytes()) )); } diff --git a/src/zstd/common/mod.rs b/src/zstd/common/mod.rs index dd38865b1200..7637ec7546b5 100644 --- a/src/zstd/common/mod.rs +++ b/src/zstd/common/mod.rs @@ -29,6 +29,7 @@ pub(crate) const ZSTD_DID_FIELD_SIZE: [usize; 4] = [0, 1, 2, 4]; pub(crate) const ZSTD_FRAMEIDSIZE: usize = 4; pub(crate) const ZSTD_BLOCKHEADERSIZE: usize = 3; +pub(crate) const ZSTD_FRAMEHEADERSIZE_MAX: usize = 18; /// `blockType_e` #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -68,6 +69,8 @@ pub(crate) const LONGNBSEQ: usize = 0x7F00; pub(crate) const MINMATCH: u32 = 3; +pub(crate) const LITBITS: u32 = 8; +pub(crate) const LIT_HUF_LOG: u32 = 11; pub(crate) const MAX_ML: u32 = 52; pub(crate) const MAX_LL: u32 = 35; pub(crate) const DEFAULT_MAX_OFF: u32 = 28; diff --git a/src/zstd/compress/block_entropy.rs b/src/zstd/compress/block_entropy.rs new file mode 100644 index 000000000000..2701b1af9e44 --- /dev/null +++ b/src/zstd/compress/block_entropy.rs @@ -0,0 +1,690 @@ +//! The part of `zstd_compress.c` that turns the sequences and the literals of a block into its content. + +use super::fse::{FseCTable, FseRepeat}; +use super::huf::{self, HUF_OPTIMAL_DEPTH_THRESHOLD, HufFlags, HufRepeat}; +use super::internal::{ + EntropyCTables, EntropyCTablesMetadata, FseCTables, FseCTablesMetadata, HufCTables, + HufCTablesMetadata, LongLengthType, SeqChunk, SeqStore, Strategy, ll_code, min_gain, ml_code, +}; +use super::literals::compress_literals; +use super::params::CCtxParams; +use super::sequences::{ + build_ctable, cross_entropy_cost, encode_sequences, fse_bit_cost, select_encoding_type, +}; +use super::{hist, huf::HufCTable}; +use crate::common::{ + DEFAULT_MAX_OFF, HUF_SYMBOLVALUE_MAX, LIT_HUF_LOG, LL_BITS, LL_DEFAULT_NORM, + LL_DEFAULT_NORM_LOG, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MINMATCH, + ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OF_DEFAULT_NORM, + OF_DEFAULT_NORM_LOG, OFF_FSE_LOG, SymbolEncodingType, ZSTD_BLOCKHEADERSIZE, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, highbit32, write_le16}; + +/// `ZSTD_literalsCompressionIsDisabled` +pub(crate) fn literals_compression_is_disabled(cctx_params: &CCtxParams) -> bool { + cctx_params.c_params.strat() == Strategy::Fast && cctx_params.c_params.target_length > 0 +} + +/// `ZSTD_seqToCodes` +pub(crate) fn seq_to_codes(store: &mut SeqStore, chunk: &SeqChunk) -> Result<()> { + let range = chunk.seq_start..chunk.seq_end; + let sequences = store.sequences.sub(range.clone())?; + let ll_codes = store.ll_code.sub_mut(range.clone())?; + let of_codes = store.of_code.sub_mut(range.clone())?; + let ml_codes = store.ml_code.sub_mut(range)?; + for (((seq, ll), of), ml) in sequences.iter().zip(ll_codes).zip(of_codes).zip(ml_codes) { + *ll = ll_code(u32::from(seq.lit_length)) as u8; + *of = highbit32(seq.off_base) as u8; + *ml = ml_code(u32::from(seq.ml_base)) as u8; + } + let long_length_pos = chunk.seq_start + chunk.long_length_pos as usize; + match chunk.long_length_type { + LongLengthType::None => {} + LongLengthType::LiteralLength => store.ll_code.put(long_length_pos, MAX_LL as u8)?, + LongLengthType::MatchLength => store.ml_code.put(long_length_pos, MAX_ML as u8)?, + } + Ok(()) +} + +/// `ZSTD_symbolEncodingTypeStats_t` +struct SymbolEncodingTypeStats { + ll_type: SymbolEncodingType, + off_type: SymbolEncodingType, + ml_type: SymbolEncodingType, + /// Of the headers of the tables. + size: usize, + /// The size of the last header written for a table, 0 if there is none. + last_count_size: usize, +} + +/// `ZSTD_buildSequencesStatistics`: makes the tables for the sequences of `chunk`, of which there is at least one, and +/// writes their headers to `dst`. +fn build_sequences_statistics( + store: &mut SeqStore, + chunk: &SeqChunk, + prev_entropy: &FseCTables, + next_entropy: &mut FseCTables, + dst: &mut [u8], + strategy: Strategy, +) -> Result { + let mut count = [0u32; MAX_SEQ as usize + 1]; + let nb_seq = chunk.nb_seq(); + let range = chunk.seq_start..chunk.seq_end; + let mut op = 0usize; + let mut last_count_size = 0; + + seq_to_codes(store, chunk)?; + let of_code_table = store.of_code.sub(range.clone())?; + let ll_code_table = store.ll_code.sub(range.clone())?; + let ml_code_table = store.ml_code.sub(range)?; + + let ll_type = { + let mut max = MAX_LL; + let most_frequent = hist::count_fast_wksp(&mut count, &mut max, ll_code_table)?; + next_entropy.litlength_repeat_mode = prev_entropy.litlength_repeat_mode; + let ll_type = select_encoding_type( + &mut next_entropy.litlength_repeat_mode, + &count, + max, + most_frequent, + nb_seq, + LL_FSE_LOG, + &prev_entropy.litlength_c_table, + &LL_DEFAULT_NORM, + LL_DEFAULT_NORM_LOG, + true, + strategy, + )?; + let count_size = build_ctable( + dst.sub_mut(op..)?, + &mut next_entropy.litlength_c_table, + LL_FSE_LOG, + ll_type, + &mut count, + max, + ll_code_table, + &LL_DEFAULT_NORM, + LL_DEFAULT_NORM_LOG, + MAX_LL, + &prev_entropy.litlength_c_table, + )?; + if ll_type == SymbolEncodingType::Compressed { + last_count_size = count_size; + } + op += count_size; + ll_type + }; + let off_type = { + let mut max = MAX_OFF; + let most_frequent = hist::count_fast_wksp(&mut count, &mut max, of_code_table)?; + // The predefined table has no code for the largest offsets. + let default_allowed = max <= DEFAULT_MAX_OFF; + next_entropy.offcode_repeat_mode = prev_entropy.offcode_repeat_mode; + let off_type = select_encoding_type( + &mut next_entropy.offcode_repeat_mode, + &count, + max, + most_frequent, + nb_seq, + OFF_FSE_LOG, + &prev_entropy.offcode_c_table, + &OF_DEFAULT_NORM, + OF_DEFAULT_NORM_LOG, + default_allowed, + strategy, + )?; + let count_size = build_ctable( + dst.sub_mut(op..)?, + &mut next_entropy.offcode_c_table, + OFF_FSE_LOG, + off_type, + &mut count, + max, + of_code_table, + &OF_DEFAULT_NORM, + OF_DEFAULT_NORM_LOG, + DEFAULT_MAX_OFF, + &prev_entropy.offcode_c_table, + )?; + if off_type == SymbolEncodingType::Compressed { + last_count_size = count_size; + } + op += count_size; + off_type + }; + let ml_type = { + let mut max = MAX_ML; + let most_frequent = hist::count_fast_wksp(&mut count, &mut max, ml_code_table)?; + next_entropy.matchlength_repeat_mode = prev_entropy.matchlength_repeat_mode; + let ml_type = select_encoding_type( + &mut next_entropy.matchlength_repeat_mode, + &count, + max, + most_frequent, + nb_seq, + ML_FSE_LOG, + &prev_entropy.matchlength_c_table, + &ML_DEFAULT_NORM, + ML_DEFAULT_NORM_LOG, + true, + strategy, + )?; + let count_size = build_ctable( + dst.sub_mut(op..)?, + &mut next_entropy.matchlength_c_table, + ML_FSE_LOG, + ml_type, + &mut count, + max, + ml_code_table, + &ML_DEFAULT_NORM, + ML_DEFAULT_NORM_LOG, + MAX_ML, + &prev_entropy.matchlength_c_table, + )?; + if ml_type == SymbolEncodingType::Compressed { + last_count_size = count_size; + } + op += count_size; + ml_type + }; + Ok(SymbolEncodingTypeStats { + ll_type, + off_type, + ml_type, + size: op, + last_count_size, + }) +} + +/// Writes how many sequences a block has. `dst` has room for 3 bytes. Returns how many it takes. +pub(crate) fn write_nb_seq(dst: &mut [u8], nb_seq: usize) -> Result { + if nb_seq < 128 { + dst.put(0, nb_seq as u8)?; + Ok(1) + } else if nb_seq < LONGNBSEQ { + dst.put(0, ((nb_seq >> 8) + 0x80) as u8)?; + dst.put(1, nb_seq as u8)?; + Ok(2) + } else { + dst.put(0, 0xFF)?; + write_le16(dst, 1, (nb_seq - LONGNBSEQ) as u16)?; + Ok(3) + } +} + +/// `ZSTD_entropyCompressSeqStore_internal`: returns 0 if the block is better left uncompressed. +fn entropy_compress_seq_store_internal( + dst: &mut [u8], + store: &mut SeqStore, + chunk: &SeqChunk, + prev_entropy: &EntropyCTables, + next_entropy: &mut EntropyCTables, + cctx_params: &CCtxParams, +) -> Result { + const SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO: usize = 20; + let strategy = cctx_params.c_params.strat(); + let nb_seq = chunk.nb_seq(); + let lit_size = chunk.lit_size(); + let mut op = 0usize; + + { + let suspect_uncompressible = + nb_seq == 0 || lit_size / nb_seq >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO; + op += compress_literals( + dst, + store.lit_buffer.sub(chunk.lit_start..chunk.lit_end)?, + &prev_entropy.huf, + &mut next_entropy.huf, + strategy, + literals_compression_is_disabled(cctx_params), + suspect_uncompressible, + )?; + } + + if dst.len() - op < 3 + 1 { + return Err(ErrorCode::DstSizeTooSmall); + } + op += write_nb_seq(dst.sub_mut(op..)?, nb_seq)?; + if nb_seq == 0 { + next_entropy.fse.clone_from(&prev_entropy.fse); + return Ok(op); + } + let last_count_size = { + let seq_head = op; + op += 1; + let stats = build_sequences_statistics( + store, + chunk, + &prev_entropy.fse, + &mut next_entropy.fse, + dst.sub_mut(op..)?, + strategy, + )?; + dst.put( + seq_head, + (((stats.ll_type as u32) << 6) + + ((stats.off_type as u32) << 4) + + ((stats.ml_type as u32) << 2)) as u8, + )?; + op += stats.size; + stats.last_count_size + }; + + let range = chunk.seq_start..chunk.seq_end; + let bitstream_size = encode_sequences( + dst.sub_mut(op..)?, + &next_entropy.fse.matchlength_c_table, + store.ml_code.sub(range.clone())?, + &next_entropy.fse.offcode_c_table, + store.of_code.sub(range.clone())?, + &next_entropy.fse.litlength_c_table, + store.ll_code.sub(range.clone())?, + store.sequences.sub(range)?, + )?; + op += bitstream_size; + // Decoders up to 1.3.4 fail on a table header with fewer than 4 bytes after it. + if last_count_size != 0 && last_count_size + bitstream_size < 4 { + return Ok(0); + } + Ok(op) +} + +/// `ZSTD_entropyCompressSeqStore`: the content of a compressed block for `chunk`, which stands for `block_size` bytes. +/// Returns 0 if the block is better left uncompressed. +pub(crate) fn entropy_compress_seq_store( + store: &mut SeqStore, + chunk: &SeqChunk, + prev_entropy: &EntropyCTables, + next_entropy: &mut EntropyCTables, + cctx_params: &CCtxParams, + dst: &mut [u8], + block_size: usize, +) -> Result { + let c_size = match entropy_compress_seq_store_internal( + dst, + store, + chunk, + prev_entropy, + next_entropy, + cctx_params, + ) { + // There is room for the block as it is. + Err(ErrorCode::DstSizeTooSmall) if block_size <= dst.len() => return Ok(0), + c_size => c_size?, + }; + let max_c_size = block_size.wrapping_sub(min_gain(block_size, cctx_params.c_params.strat())); + if c_size >= max_c_size { + return Ok(0); + } + Ok(c_size) +} + +/// What the C code has for the size of something that failed: an error code, which is more than any size. +fn size_or_error(size: Result) -> usize { + size.unwrap_or_else(|code| 0usize.wrapping_sub(code as usize)) +} + +/// `ZSTD_buildBlockEntropyStats_literals`: makes the table for the literals `src`, and its description in +/// `huf_metadata`. Returns the size of the description. +fn build_block_entropy_stats_literals( + src: &[u8], + prev_huf: &HufCTables, + next_huf: &mut HufCTables, + huf_metadata: &mut HufCTablesMetadata, + literals_compression_is_disabled: bool, + huf_flags: HufFlags, +) -> Result { + const COMPRESS_LITERALS_SIZE_MIN: usize = 63; + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + let src_size = src.len(); + let mut max_symbol_value = HUF_SYMBOLVALUE_MAX; + let mut repeat = prev_huf.repeat_mode; + + next_huf.clone_from(prev_huf); + + if literals_compression_is_disabled { + huf_metadata.h_type = SymbolEncodingType::Basic; + return Ok(0); + } + + let min_lit_size = if prev_huf.repeat_mode == HufRepeat::Valid { + 6 + } else { + COMPRESS_LITERALS_SIZE_MIN + }; + if src_size <= min_lit_size { + huf_metadata.h_type = SymbolEncodingType::Basic; + return Ok(0); + } + + let largest = hist::count_wksp(&mut count, &mut max_symbol_value, src)?; + if largest == src_size { + huf_metadata.h_type = SymbolEncodingType::Rle; + return Ok(0); + } + if largest <= (src_size >> 7) + 4 { + huf_metadata.h_type = SymbolEncodingType::Basic; + return Ok(0); + } + + if repeat == HufRepeat::Check + && !huf::validate_ctable(&prev_huf.c_table, &count, max_symbol_value) + { + repeat = HufRepeat::None; + } + + next_huf.c_table = HufCTable::new(); + let huff_log = huf::optimal_table_log( + LIT_HUF_LOG, + src_size, + max_symbol_value, + &mut next_huf.c_table, + &count, + huf_flags, + ); + let huff_log = + huf::build_ctable(&mut next_huf.c_table, &count, max_symbol_value, huff_log)? as u32; + + let new_c_size = huf::estimate_compressed_size(&next_huf.c_table, &count, max_symbol_value); + let h_size_result = huf::write_ctable( + &mut huf_metadata.huf_des_buffer, + &next_huf.c_table, + max_symbol_value, + huff_log, + ); + let h_size = size_or_error(h_size_result); + if repeat != HufRepeat::None { + let old_c_size = huf::estimate_compressed_size(&prev_huf.c_table, &count, max_symbol_value); + if old_c_size < src_size + && (old_c_size <= h_size.wrapping_add(new_c_size) + || h_size.wrapping_add(12) >= src_size) + { + next_huf.clone_from(prev_huf); + huf_metadata.h_type = SymbolEncodingType::Repeat; + return Ok(0); + } + } + if new_c_size.wrapping_add(h_size) >= src_size { + next_huf.clone_from(prev_huf); + huf_metadata.h_type = SymbolEncodingType::Basic; + return Ok(0); + } + huf_metadata.h_type = SymbolEncodingType::Compressed; + next_huf.repeat_mode = HufRepeat::Check; + h_size_result +} + +/// `ZSTD_buildBlockEntropyStats_sequences`: makes the tables for the sequences of `chunk`, and their description in +/// `fse_metadata`. Returns the size of the description. +fn build_block_entropy_stats_sequences( + store: &mut SeqStore, + chunk: &SeqChunk, + prev_entropy: &FseCTables, + next_entropy: &mut FseCTables, + cctx_params: &CCtxParams, + fse_metadata: &mut FseCTablesMetadata, +) -> Result { + let stats = if chunk.nb_seq() != 0 { + build_sequences_statistics( + store, + chunk, + prev_entropy, + next_entropy, + &mut fse_metadata.fse_tables_buffer, + cctx_params.c_params.strat(), + )? + } else { + next_entropy.litlength_repeat_mode = FseRepeat::None; + next_entropy.offcode_repeat_mode = FseRepeat::None; + next_entropy.matchlength_repeat_mode = FseRepeat::None; + SymbolEncodingTypeStats { + ll_type: SymbolEncodingType::Basic, + off_type: SymbolEncodingType::Basic, + ml_type: SymbolEncodingType::Basic, + size: 0, + last_count_size: 0, + } + }; + fse_metadata.ll_type = stats.ll_type; + fse_metadata.of_type = stats.off_type; + fse_metadata.ml_type = stats.ml_type; + fse_metadata.last_count_size = stats.last_count_size; + Ok(stats.size) +} + +/// `ZSTD_buildBlockEntropyStats`: makes the tables for `chunk` in `next_entropy`, and what it takes to write them in +/// `entropy_metadata`. +pub(crate) fn build_block_entropy_stats( + store: &mut SeqStore, + chunk: &SeqChunk, + prev_entropy: &EntropyCTables, + next_entropy: &mut EntropyCTables, + cctx_params: &CCtxParams, + entropy_metadata: &mut EntropyCTablesMetadata, +) -> Result<()> { + let huf_flags = HufFlags { + optimal_depth: cctx_params.c_params.strat() >= HUF_OPTIMAL_DEPTH_THRESHOLD, + ..HufFlags::default() + }; + entropy_metadata.huf_metadata.huf_des_size = build_block_entropy_stats_literals( + store.lit_buffer.sub(chunk.lit_start..chunk.lit_end)?, + &prev_entropy.huf, + &mut next_entropy.huf, + &mut entropy_metadata.huf_metadata, + literals_compression_is_disabled(cctx_params), + huf_flags, + )?; + entropy_metadata.fse_metadata.fse_tables_size = build_block_entropy_stats_sequences( + store, + chunk, + &prev_entropy.fse, + &mut next_entropy.fse, + cctx_params, + &mut entropy_metadata.fse_metadata, + )?; + Ok(()) +} + +/// `ZSTD_estimateBlockSize_literal` +fn estimate_block_size_literal( + literals: &[u8], + huf: &HufCTables, + huf_metadata: &HufCTablesMetadata, + write_entropy: bool, +) -> usize { + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + let mut max_symbol_value = HUF_SYMBOLVALUE_MAX; + let lit_size = literals.len(); + let literal_section_header_size = + 3 + usize::from(lit_size >= 1 << 10) + usize::from(lit_size >= 16 << 10); + let single_stream = lit_size < 256; + + match huf_metadata.h_type { + SymbolEncodingType::Basic => lit_size, + SymbolEncodingType::Rle => 1, + SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { + if hist::count_wksp(&mut count, &mut max_symbol_value, literals).is_err() { + return lit_size; + } + let mut c_lit_size_estimate = + huf::estimate_compressed_size(&huf.c_table, &count, max_symbol_value); + if write_entropy { + c_lit_size_estimate += huf_metadata.huf_des_size; + } + if !single_stream { + // The jump table. + c_lit_size_estimate += 6; + } + c_lit_size_estimate + literal_section_header_size + } + } +} + +/// `ZSTD_estimateBlockSize_symbolType` +fn estimate_block_size_symbol_type( + ty: SymbolEncodingType, + code_table: &[u8], + max_code: u32, + fse_c_table: &FseCTable, + additional_bits: Option<&[u8]>, + default_norm: &[i16], + default_norm_log: u32, +) -> Result { + let mut count = [0u32; MAX_SEQ as usize + 1]; + let mut max = max_code; + hist::count_fast_wksp(&mut count, &mut max, code_table)?; + let estimate_in_bits = match ty { + SymbolEncodingType::Basic => cross_entropy_cost(default_norm, default_norm_log, &count, max), + SymbolEncodingType::Rle => Ok(0), + SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { + fse_bit_cost(fse_c_table, &count, max) + } + }; + let Ok(mut estimate_in_bits) = estimate_in_bits else { + return Ok(code_table.len() * 10); + }; + for &code in code_table { + estimate_in_bits += usize::from(match additional_bits { + Some(bits) => bits.at(usize::from(code))?, + None => code, + }); + } + Ok(estimate_in_bits >> 3) +} + +/// `ZSTD_estimateBlockSize_sequences` +fn estimate_block_size_sequences( + of_code_table: &[u8], + ll_code_table: &[u8], + ml_code_table: &[u8], + fse_tables: &FseCTables, + fse_metadata: &FseCTablesMetadata, + write_entropy: bool, +) -> Result { + let nb_seq = of_code_table.len(); + let sequences_section_header_size = + 1 + 1 + usize::from(nb_seq >= 128) + usize::from(nb_seq >= LONGNBSEQ); + let mut c_seq_size_estimate = estimate_block_size_symbol_type( + fse_metadata.of_type, + of_code_table, + MAX_OFF, + &fse_tables.offcode_c_table, + None, + &OF_DEFAULT_NORM, + OF_DEFAULT_NORM_LOG, + )?; + c_seq_size_estimate += estimate_block_size_symbol_type( + fse_metadata.ll_type, + ll_code_table, + MAX_LL, + &fse_tables.litlength_c_table, + Some(&LL_BITS), + &LL_DEFAULT_NORM, + LL_DEFAULT_NORM_LOG, + )?; + c_seq_size_estimate += estimate_block_size_symbol_type( + fse_metadata.ml_type, + ml_code_table, + MAX_ML, + &fse_tables.matchlength_c_table, + Some(&ML_BITS), + &ML_DEFAULT_NORM, + ML_DEFAULT_NORM_LOG, + )?; + if write_entropy { + c_seq_size_estimate += fse_metadata.fse_tables_size; + } + Ok(c_seq_size_estimate + sequences_section_header_size) +} + +/// `ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize`: about how large a block for `chunk` would be. Overwrites +/// `next_entropy`. +pub(crate) fn build_entropy_statistics_and_estimate_sub_block_size( + store: &mut SeqStore, + chunk: &SeqChunk, + prev_entropy: &EntropyCTables, + next_entropy: &mut EntropyCTables, + cctx_params: &CCtxParams, + entropy_metadata: &mut EntropyCTablesMetadata, +) -> Result { + build_block_entropy_stats( + store, + chunk, + prev_entropy, + next_entropy, + cctx_params, + entropy_metadata, + )?; + let range = chunk.seq_start..chunk.seq_end; + let literals_size = estimate_block_size_literal( + store.lit_buffer.sub(chunk.lit_start..chunk.lit_end)?, + &next_entropy.huf, + &entropy_metadata.huf_metadata, + entropy_metadata.huf_metadata.h_type == SymbolEncodingType::Compressed, + ); + let seq_size = estimate_block_size_sequences( + store.of_code.sub(range.clone())?, + store.ll_code.sub(range.clone())?, + store.ml_code.sub(range)?, + &next_entropy.fse, + &entropy_metadata.fse_metadata, + true, + )?; + Ok(seq_size + literals_size + ZSTD_BLOCKHEADERSIZE) +} + +/// `ZSTD_countSeqStoreLiteralsBytes` +pub(crate) fn count_seq_store_literals_bytes(store: &SeqStore, chunk: &SeqChunk) -> Result { + let sequences = store.sequences.sub(chunk.seq_start..chunk.seq_end)?; + let mut literals_bytes: usize = sequences.iter().map(|seq| usize::from(seq.lit_length)).sum(); + if chunk.long_length_type == LongLengthType::LiteralLength + && (chunk.long_length_pos as usize) < sequences.len() + { + literals_bytes += 0x10000; + } + Ok(literals_bytes) +} + +/// `ZSTD_countSeqStoreMatchBytes` +pub(crate) fn count_seq_store_match_bytes(store: &SeqStore, chunk: &SeqChunk) -> Result { + let sequences = store.sequences.sub(chunk.seq_start..chunk.seq_end)?; + let mut match_bytes: usize = (sequences.iter()) + .map(|seq| usize::from(seq.ml_base) + MINMATCH as usize) + .sum(); + if chunk.long_length_type == LongLengthType::MatchLength + && (chunk.long_length_pos as usize) < sequences.len() + { + match_bytes += 0x10000; + } + Ok(match_bytes) +} + +/// `ZSTD_deriveSeqStoreChunk`: the sequences of `store` from `start_idx` to `end_idx`. +pub(crate) fn derive_seq_store_chunk( + store: &SeqStore, + start_idx: usize, + end_idx: usize, +) -> Result { + let mut result = store.whole(); + if start_idx > 0 { + result.seq_end = start_idx; + result.lit_start += count_seq_store_literals_bytes(store, &result)?; + } + if store.long_length_type != LongLengthType::None { + let long_length_pos = store.long_length_pos as usize; + if long_length_pos < start_idx || long_length_pos > end_idx { + result.long_length_type = LongLengthType::None; + } else { + result.long_length_pos -= start_idx as u32; + } + } + result.seq_start = start_idx; + result.seq_end = end_idx; + if end_idx != store.sequences.len() { + // Only the last chunk has the literals that follow the last sequence. + result.lit_end = result.lit_start + count_seq_store_literals_bytes(store, &result)?; + } + Ok(result) +} diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs new file mode 100644 index 000000000000..3023d25df465 --- /dev/null +++ b/src/zstd/compress/cctx.rs @@ -0,0 +1,1727 @@ +//! The part of `zstd_compress.c` that writes frames: `ZSTD_compressBegin`, `ZSTD_compressContinue`, `ZSTD_compressEnd` +//! and what they need. + +use std::sync::Arc; + +use super::block_entropy::{ + build_entropy_statistics_and_estimate_sub_block_size, count_seq_store_literals_bytes, + count_seq_store_match_bytes, derive_seq_store_chunk, entropy_compress_seq_store, +}; +use super::fse::FseRepeat; +use super::huf::{self, HufRepeat}; +use super::internal::{ + BlockSrc, BufId, Buffers, CompressedBlockState, CompressionParameters, DictMatchState, + DictTableLoadMethod, EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, + MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqChunk, SeqStore, Strategy, + TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, + ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, min_gain, + no_compress_block, offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, +}; +use super::params::{ + CCtxParams, CParamMode, Format, MatchStateSizes, Workspace, WorkspaceNeeds, + adjust_c_params_internal, allocate_chain_table, cdict_indices_are_tagged, compress_bound, + cycle_log, get_c_params_from_cctx_params, max_nb_seq, resolve_row_match_finder_mode, + row_match_finder_used, +}; +use super::superblock::compress_super_block; +use super::{compress_block, double_fast, fast, lazy, ldm, opt, presplit}; +use crate::common::entropy::read_ncount; +use crate::common::xxhash::Xxh64; +use crate::common::{ + BlockType, LITBITS, LL_FSE_LOG, MAX_LL, MAX_ML, MAX_OFF, MIN_CBLOCK_SIZE, ML_FSE_LOG, + OFF_FSE_LOG, ZSTD_BLOCKHEADERSIZE, ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_UNKNOWN, + ZSTD_FRAMEHEADERSIZE_MAX, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGICNUMBER, ZSTD_REP_NUM, + ZSTD_WINDOWLOG_ABSOLUTEMIN, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{ + SliceExt, highbit32, read_le32, try_zeroed, write_le16, write_le24, write_le32, write_le64, +}; + +/// `ZSTD_compressionStage_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum CompressionStage { + Created, + Init, + Ongoing, + Ending, +} + +/// `ZSTD_compResetPolicy_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum CompResetPolicy { + MakeClean, + /// The caller overwrites the tables. + LeaveDirty, +} + +/// `ZSTD_indexResetPolicy_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum IndexResetPolicy { + /// Indices go on from where they are, which makes the ones in the tables invalid without erasing them. + Continue, + Reset, +} + +/// `ZSTD_resetTarget_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum ResetTarget { + CDict, + CCtx, +} + +/// `ZSTD_dictContentType_e`. Nothing asks for `ZSTD_dct_fullDict`. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DictContentType { + Auto, + RawContent, +} + +/// Some bytes of one of the [`Buffers`]. +#[derive(Clone, Copy)] +pub(crate) struct Span { + pub(crate) buf: BufId, + pub(crate) start: usize, + pub(crate) end: usize, +} + +impl Span { + #[inline] + const fn len(&self) -> usize { + self.end - self.start + } + + #[inline] + const fn start_addr(&self) -> usize { + self.buf.start() + self.start + } + + #[inline] + const fn end_addr(&self) -> usize { + self.buf.start() + self.end + } + + #[inline] + fn bytes<'a>(&self, bufs: &Buffers<'a>) -> Result<&'a [u8]> { + bufs.get(self.buf).sub(self.start..self.end) + } +} + +/// What the pointers of a `ZSTD_CCtx` lead to, that it does not own. +#[derive(Clone, Copy)] +pub(crate) struct Input<'a> { + pub(crate) bufs: Buffers<'a>, + pub(crate) cdict: Option<&'a CDict>, +} + +fn window_prefix<'a>(window: &Window, bufs: &Buffers<'a>) -> Prefix<'a> { + Prefix { + buf: bufs.get(window.base_buf), + base: window.base.wrapping_sub(window.base_buf.start()), + } +} + +/// `ZSTD_bitmix` +const fn bitmix(mut val: u64, len: u64) -> u64 { + val ^= val.rotate_right(49) ^ val.rotate_right(24); + val = val.wrapping_mul(0x9FB2_1C65_1E98_DF25); + val ^= (val >> 35).wrapping_add(len); + val = val.wrapping_mul(0x9FB2_1C65_1E98_DF25); + val ^ (val >> 28) +} + +/// `ZSTD_invalidateMatchState` +fn invalidate_match_state(ms: &mut MatchState) { + ms.window.clear(); + ms.next_to_update = ms.window.dict_limit; + ms.loaded_dict_end = 0; + ms.opt.lit_length_sum = 0; + ms.has_dict_match_state = false; +} + +/// Gives `table` the length `len`. What it holds is 0, or if `clean` is false possibly what it held before. +pub(crate) fn reset_table( + table: &mut Vec, + len: usize, + clean: bool, +) -> Result<()> { + if table.len() != len { + // First, so that both are not allocated at once. + *table = Vec::new(); + *table = try_zeroed(len)?; + } else if clean { + table.fill(T::default()); + } + Ok(()) +} + +/// The same for a table of structures. +pub(crate) fn reset_struct_table( + table: &mut Vec, + len: usize, + clean: bool, +) -> Result<()> { + if table.len() != len { + *table = Vec::new(); + table + .try_reserve_exact(len) + .map_err(|_| ErrorCode::MemoryAllocation)?; + table.resize(len, T::default()); + } else if clean { + table.fill(T::default()); + } + Ok(()) +} + +/// `ZSTD_reset_matchState` +fn reset_match_state( + ms: &mut MatchState, + c_params: &CompressionParameters, + use_row_match_finder: ParamSwitch, + crp: CompResetPolicy, + force_reset_index: IndexResetPolicy, + for_who: ResetTarget, +) -> Result<()> { + let sizes = MatchStateSizes::new( + c_params, + use_row_match_finder, + for_who == ResetTarget::CCtx, + ); + + if force_reset_index == IndexResetPolicy::Reset { + ms.window = Window::new(); + } + ms.hash_log3 = sizes.hash_log3; + ms.lazy_skipping = false; + invalidate_match_state(ms); + + // As long as indices go on, the ones in the tables are too low to be valid. + let clean = force_reset_index == IndexResetPolicy::Reset && crp != CompResetPolicy::LeaveDirty; + reset_table(&mut ms.hash_table, sizes.h_size, clean)?; + reset_table(&mut ms.chain_table, sizes.chain_size, clean)?; + reset_table(&mut ms.hash_table3, sizes.h3_size, clean)?; + + if row_match_finder_used(c_params.strat(), use_row_match_finder) { + if for_who == ResetTarget::CCtx { + // A new salt instead of erasing the tags. + reset_table(&mut ms.tag_table, sizes.h_size, false)?; + ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(u64::from(ms.hash_salt_entropy), 4); + } else { + reset_table(&mut ms.tag_table, sizes.h_size, true)?; + ms.hash_salt = 0; + } + let row_log = c_params.search_log.clamp(4, 6); + ms.row_hash_log = c_params.hash_log - row_log; + } else { + ms.tag_table = Vec::new(); + } + + if for_who == ResetTarget::CCtx && c_params.strat() >= Strategy::BtOpt { + reset_table(&mut ms.opt.lit_freq, 1 << LITBITS, false)?; + reset_table(&mut ms.opt.lit_length_freq, MAX_LL as usize + 1, false)?; + reset_table(&mut ms.opt.match_length_freq, MAX_ML as usize + 1, false)?; + reset_table(&mut ms.opt.off_code_freq, MAX_OFF as usize + 1, false)?; + reset_struct_table(&mut ms.opt.match_table, ZSTD_OPT_SIZE, false)?; + reset_struct_table(&mut ms.opt.price_table, ZSTD_OPT_SIZE, false)?; + } else { + ms.opt.match_table = Vec::new(); + ms.opt.price_table = Vec::new(); + } + + ms.c_params = *c_params; + Ok(()) +} + +/// `ZSTD_indexTooCloseToMax`: starting a frame this close to the limit would mean bringing indices down during it. +fn index_too_close_to_max(w: &Window) -> bool { + const ZSTD_INDEXOVERFLOW_MARGIN: usize = 16 << 20; + w.next_src.wrapping_sub(w.base) > ZSTD_CURRENT_MAX as usize - ZSTD_INDEXOVERFLOW_MARGIN +} + +/// `ZSTD_reduceTable_internal` +fn reduce_table(table: &mut [u32], reducer_value: u32, preserve_mark: bool) { + let reducer_threshold = reducer_value.wrapping_add(ZSTD_WINDOW_START_INDEX); + for cell in table { + *cell = if preserve_mark && *cell == ZSTD_DUBT_UNSORTED_MARK { + ZSTD_DUBT_UNSORTED_MARK + } else if *cell < reducer_threshold { + 0 + } else { + *cell - reducer_value + }; + } +} + +/// `ZSTD_reduceIndex` +fn reduce_index(ms: &mut MatchState, params: &CCtxParams, reducer_value: u32) { + reduce_table(&mut ms.hash_table, reducer_value, false); + if allocate_chain_table(params.c_params.strat(), params.use_row_match_finder) { + let preserve_mark = params.c_params.strat() == Strategy::BtLazy2; + reduce_table(&mut ms.chain_table, reducer_value, preserve_mark); + } + if ms.hash_log3 != 0 { + reduce_table(&mut ms.hash_table3, reducer_value, false); + } +} + +/// `ZSTD_overflowCorrectIfNeeded` +fn overflow_correct_if_needed(ms: &mut MatchState, params: &CCtxParams, ip: usize, iend: usize) { + let cycle_log = cycle_log(params.c_params.chain_log, params.c_params.strat()); + let max_dist = 1u32 << params.c_params.window_log; + if ms.window.need_overflow_correction(iend) { + let correction = ms.window.correct_overflow(cycle_log, max_dist, ip); + reduce_index(ms, params, correction); + ms.next_to_update = ms.next_to_update.saturating_sub(correction); + // The dictionary is out of reach by now. + ms.loaded_dict_end = 0; + ms.has_dict_match_state = false; + } +} + +/// `ZSTD_loadDictionaryContent`: makes `src` the start of the window, and indexes it. +fn load_dictionary_content( + ms: &mut MatchState, + ls: Option<&mut LdmState>, + params: &CCtxParams, + bufs: &Buffers<'_>, + mut src: Span, + dtlm: DictTableLoadMethod, + tfp: TableFillPurpose, +) -> Result<()> { + { + // Indices have to fit, along with a tag in the tables that have one. + let mut max_dict_size = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; + if cdict_indices_are_tagged(¶ms.c_params) && tfp == TableFillPurpose::ForCDict { + let short_cache_max_dict_size = + (1u32 << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; + max_dict_size = max_dict_size.min(short_cache_max_dict_size); + } + if src.len() > max_dict_size as usize { + src.start = src.end - max_dict_size as usize; + } + } + + ms.window.update(src.start_addr(), src.buf, src.len(), false); + + if let Some(ls) = ls + && params.ldm_params.enable_ldm == ParamSwitch::Enable + { + ls.window.update(src.start_addr(), src.buf, src.len(), false); + ls.loaded_dict_end = if params.force_window { + 0 + } else { + ls.window.index_of(src.end_addr()) + }; + let prefix = window_prefix(&ls.window, bufs); + ldm::fill_hash_table(ls, prefix, src.start, src.end, ¶ms.ldm_params)?; + } + + { + // More than the tables can hold is not worth indexing. + let log = (params.c_params.hash_log + 3) + .max(params.c_params.chain_log + 1) + .min(31); + let max_dict_size = 1usize << log; + if src.len() > max_dict_size { + src.start = src.end - max_dict_size; + } + } + + ms.next_to_update = ms.window.index_of(src.start_addr()); + ms.loaded_dict_end = if params.force_window { + 0 + } else { + ms.window.index_of(src.end_addr()) + }; + ms.force_non_contiguous = false; + + if src.len() <= HASH_READ_SIZE { + return Ok(()); + } + + overflow_correct_if_needed(ms, params, src.start_addr(), src.end_addr()); + + let prefix = window_prefix(&ms.window, bufs); + match params.c_params.strat() { + Strategy::Fast => fast::fill_hash_table(ms, prefix, src.end, dtlm, tfp)?, + Strategy::DFast => double_fast::fill_double_hash_table(ms, prefix, src.end, dtlm, tfp)?, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { + if params.use_row_match_finder == ParamSwitch::Enable { + ms.tag_table.fill(0); + lazy::row_update(ms, prefix, src.end - HASH_READ_SIZE)?; + } else { + lazy::insert_and_find_first_index(ms, prefix, src.end - HASH_READ_SIZE)?; + } + } + Strategy::BtLazy2 | Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { + opt::update_tree(ms, prefix, src.end - HASH_READ_SIZE, src.end)?; + } + } + + ms.next_to_update = ms.window.index_of(src.end_addr()); + Ok(()) +} + +/// `ZSTD_dictNCountRepeat`: a table that has a code for every symbol can be used without checking. +fn dict_ncount_repeat( + normalized_counter: &[i16], + dict_max_symbol_value: u32, + max_symbol_value: u32, +) -> FseRepeat { + if dict_max_symbol_value < max_symbol_value { + return FseRepeat::Check; + } + let counts = normalized_counter.iter().take(max_symbol_value as usize + 1); + if counts.into_iter().any(|&count| count == 0) { + return FseRepeat::Check; + } + FseRepeat::Valid +} + +/// `ZSTD_loadCEntropy`: reads the tables of the dictionary `dict` into `bs`. Returns where its content starts. +fn load_c_entropy(bs: &mut CompressedBlockState, dict: &[u8]) -> Result { + const CORRUPTED: ErrorCode = ErrorCode::DictionaryCorrupted; + let mut offcode_ncount = [0i16; MAX_OFF as usize + 1]; + let mut dict_ptr = 8usize; + bs.entropy.huf.repeat_mode = HufRepeat::Check; + + { + let stats = huf::read_ctable(&mut bs.entropy.huf.c_table, 255, dict.sub(dict_ptr..)?) + .map_err(|_| CORRUPTED)?; + // A table that has a code for every byte can be used without checking. + if !stats.has_zero_weights && stats.max_symbol_value == 255 { + bs.entropy.huf.repeat_mode = HufRepeat::Valid; + } + dict_ptr += stats.size; + } + + let offcode_max_value = { + let header = read_ncount(&mut offcode_ncount, MAX_OFF, dict.sub(dict_ptr..)?) + .map_err(|_| CORRUPTED)?; + if header.table_log > OFF_FSE_LOG { + return Err(CORRUPTED); + } + (bs.entropy.fse.offcode_c_table) + .build(&offcode_ncount, MAX_OFF, header.table_log) + .map_err(|_| CORRUPTED)?; + dict_ptr += header.size; + header.max_symbol_value + }; + + { + let mut matchlength_ncount = [0i16; MAX_ML as usize + 1]; + let header = read_ncount(&mut matchlength_ncount, MAX_ML, dict.sub(dict_ptr..)?) + .map_err(|_| CORRUPTED)?; + if header.table_log > ML_FSE_LOG { + return Err(CORRUPTED); + } + (bs.entropy.fse.matchlength_c_table) + .build( + &matchlength_ncount, + header.max_symbol_value, + header.table_log, + ) + .map_err(|_| CORRUPTED)?; + bs.entropy.fse.matchlength_repeat_mode = + dict_ncount_repeat(&matchlength_ncount, header.max_symbol_value, MAX_ML); + dict_ptr += header.size; + } + + { + let mut litlength_ncount = [0i16; MAX_LL as usize + 1]; + let header = read_ncount(&mut litlength_ncount, MAX_LL, dict.sub(dict_ptr..)?) + .map_err(|_| CORRUPTED)?; + if header.table_log > LL_FSE_LOG { + return Err(CORRUPTED); + } + (bs.entropy.fse.litlength_c_table) + .build(&litlength_ncount, header.max_symbol_value, header.table_log) + .map_err(|_| CORRUPTED)?; + bs.entropy.fse.litlength_repeat_mode = + dict_ncount_repeat(&litlength_ncount, header.max_symbol_value, MAX_LL); + dict_ptr += header.size; + } + + if dict_ptr + 12 > dict.len() { + return Err(CORRUPTED); + } + bs.rep = [ + read_le32(dict, dict_ptr)?, + read_le32(dict, dict_ptr + 4)?, + read_le32(dict, dict_ptr + 8)?, + ]; + dict_ptr += 12; + + { + let dict_content_size = dict.len() - dict_ptr; + let mut offcode_max = MAX_OFF; + if dict_content_size <= u32::MAX as usize - (128 << 10) { + // The largest offset that a first block can have. + let max_offset = dict_content_size as u32 + (128 << 10); + offcode_max = highbit32(max_offset); + } + bs.entropy.fse.offcode_repeat_mode = dict_ncount_repeat( + &offcode_ncount, + offcode_max_value, + offcode_max.min(MAX_OFF), + ); + + for rep in bs.rep { + if rep == 0 || rep as usize > dict_content_size { + return Err(CORRUPTED); + } + } + } + + Ok(dict_ptr) +} + +/// `ZSTD_compress_insertDictionary`: returns the ID of the dictionary, which is 0 if it has none. +fn compress_insert_dictionary( + bs: &mut CompressedBlockState, + ms: &mut MatchState, + ls: Option<&mut LdmState>, + params: &CCtxParams, + bufs: &Buffers<'_>, + dict: Option, + dict_content_type: DictContentType, + dtlm: DictTableLoadMethod, + tfp: TableFillPurpose, +) -> Result { + let Some(dict) = dict.filter(|dict| dict.len() >= 8) else { + return Ok(0); + }; + + bs.reset(); + + let bytes = dict.bytes(bufs)?; + if dict_content_type == DictContentType::RawContent + || read_le32(bytes, 0)? != ZSTD_MAGIC_DICTIONARY + { + load_dictionary_content(ms, ls, params, bufs, dict, dtlm, tfp)?; + return Ok(0); + } + + // `ZSTD_loadZstdDictionary` + let dict_id = if params.f_params.no_dict_id_flag { + 0 + } else { + read_le32(bytes, 4)? + }; + let e_size = load_c_entropy(bs, bytes)?; + let content = Span { + start: dict.start + e_size, + ..dict + }; + load_dictionary_content(ms, None, params, bufs, content, dtlm, tfp)?; + Ok(dict_id) +} + +/// `ZSTD_CDict`: a dictionary that is indexed, ready to start frames with. +pub(crate) struct CDict { + dict_content: Arc<[u8]>, + match_state: MatchState, + c_block_state: Box, + dict_id: u32, + use_row_match_finder: ParamSwitch, +} + +impl CDict { + /// `ZSTD_createCDict_advanced2` + pub(crate) fn new(dict: Arc<[u8]>, original_cctx_params: &CCtxParams) -> Result { + let mut params = *original_cctx_params; + params.c_params = get_c_params_from_cctx_params( + ¶ms, + ZSTD_CONTENTSIZE_UNKNOWN, + dict.len(), + CParamMode::CreateCDict, + ); + params.use_row_match_finder = + resolve_row_match_finder_mode(params.use_row_match_finder, ¶ms.c_params); + + let mut cdict = Self { + dict_content: dict, + match_state: MatchState::new(), + c_block_state: Box::new(CompressedBlockState::new()), + dict_id: 0, + use_row_match_finder: params.use_row_match_finder, + }; + reset_match_state( + &mut cdict.match_state, + ¶ms.c_params, + params.use_row_match_finder, + CompResetPolicy::MakeClean, + IndexResetPolicy::Reset, + ResetTarget::CDict, + )?; + let bufs = Buffers { + in_buff: &[], + user: &[], + dict: &cdict.dict_content, + }; + cdict.dict_id = compress_insert_dictionary( + &mut cdict.c_block_state, + &mut cdict.match_state, + None, + ¶ms, + &bufs, + Some(Span { + buf: BufId::Dict, + start: 0, + end: cdict.dict_content.len(), + }), + DictContentType::Auto, + DictTableLoadMethod::Full, + TableFillPurpose::ForCDict, + )?; + Ok(cdict) + } + + pub(crate) fn dict_content(&self) -> &[u8] { + &self.dict_content + } + + pub(crate) const fn dict_id(&self) -> u32 { + self.dict_id + } + + fn dict_match_state(&self) -> DictMatchState<'_> { + let window = &self.match_state.window; + let start = BufId::Dict.start(); + DictMatchState { + ms: &self.match_state, + buf: &self.dict_content, + base: window.base.wrapping_sub(start), + end: window.next_src.wrapping_sub(start), + } + } + + /// `ZSTD_shouldAttachDict`: whether to search the tables of the dictionary, rather than start from a copy of them. + pub(crate) fn should_attach(&self, params: &CCtxParams, pledged_src_size: u64) -> bool { + let cutoff: u64 = match self.match_state.c_params.strat() { + Strategy::Fast | Strategy::BtUltra | Strategy::BtUltra2 => 8 << 10, + Strategy::DFast => 16 << 10, + _ => 32 << 10, + }; + (pledged_src_size <= cutoff || pledged_src_size == ZSTD_CONTENTSIZE_UNKNOWN) + && !params.force_window + } +} + +/// `ZSTD_isRLE`: `src` is not empty. +fn is_rle(src: &[u8]) -> bool { + let Some((&value, rest)) = src.split_first() else { + return false; + }; + let (chunks, tail) = rest.as_chunks::<32>(); + chunks.iter().all(|chunk| *chunk == [value; 32]) && tail.iter().all(|&b| b == value) +} + +/// `ZSTD_maybeRLE` +fn maybe_rle(seq_store: &SeqStore) -> bool { + seq_store.sequences.len() < 4 && seq_store.lit_len < 10 +} + +/// `writeBlockHeader` +fn write_block_header( + op: &mut [u8], + c_size: usize, + block_size: usize, + last_block: bool, +) -> Result<()> { + let c_block_header = if c_size == 1 { + u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((block_size as u32) << 3) + } else { + u32::from(last_block) + ((BlockType::Compressed as u32) << 1) + ((c_size as u32) << 3) + }; + write_le24(op, 0, c_block_header) +} + +/// `ZSTD_resolveRepcodeToRawOffset` +fn resolve_repcode_to_raw_offset(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> u32 { + let [rep0, ..] = *rep; + let adjusted_rep_code = off_base - 1 + u32::from(ll0); + match rep.get(adjusted_rep_code as usize) { + Some(&offset) => offset, + None => rep0.wrapping_sub(1), + } +} + +/// `ZSTD_seqStore_resolveOffCodes`: a block that ends up uncompressed leaves the repeat offsets of the decoder, +/// `d_repcodes`, behind those that the sequences were found with, `c_repcodes`. Where they disagree, the repeat codes +/// of `chunk` are replaced by the offsets they stand for. +fn seq_store_resolve_off_codes( + d_repcodes: &mut [u32; ZSTD_REP_NUM], + c_repcodes: &mut [u32; ZSTD_REP_NUM], + store: &mut SeqStore, + chunk: &SeqChunk, +) -> Result<()> { + let sequences = store.sequences.sub_mut(chunk.seq_start..chunk.seq_end)?; + let long_lit_len_idx = if chunk.long_length_type == LongLengthType::LiteralLength { + chunk.long_length_pos as usize + } else { + sequences.len() + }; + for (idx, seq) in sequences.iter_mut().enumerate() { + let ll0 = seq.lit_length == 0 && idx != long_lit_len_idx; + let off_base = seq.off_base; + if offbase_is_repcode(off_base) { + let d_raw_offset = resolve_repcode_to_raw_offset(d_repcodes, off_base, ll0); + let c_raw_offset = resolve_repcode_to_raw_offset(c_repcodes, off_base, ll0); + if d_raw_offset != c_raw_offset { + seq.off_base = offset_to_offbase(c_raw_offset); + } + } + update_rep(d_repcodes, seq.off_base, ll0); + update_rep(c_repcodes, off_base, ll0); + } + Ok(()) +} + +/// `ZSTD_writeFrameHeader` +fn write_frame_header( + dst: &mut [u8], + params: &CCtxParams, + pledged_src_size: u64, + dict_id: u32, +) -> Result { + let dict_id_size_code_length = + u32::from(dict_id > 0) + u32::from(dict_id >= 256) + u32::from(dict_id >= 65536); + let dict_id_size_code = if params.f_params.no_dict_id_flag { + 0 + } else { + dict_id_size_code_length + }; + let checksum_flag = u32::from(params.f_params.checksum_flag); + let window_size = 1u32 << params.c_params.window_log; + let single_segment = + params.f_params.content_size_flag && u64::from(window_size) >= pledged_src_size; + let window_log_byte = ((params.c_params.window_log - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3) as u8; + let fcs_code = if params.f_params.content_size_flag { + u32::from(pledged_src_size >= 256) + + u32::from(pledged_src_size >= 65536 + 256) + + u32::from(pledged_src_size >= 0xFFFF_FFFF) + } else { + 0 + }; + let frame_header_description_byte = (dict_id_size_code + + (checksum_flag << 2) + + (u32::from(single_segment) << 5) + + (fcs_code << 6)) as u8; + let mut pos = 0usize; + + if dst.len() < ZSTD_FRAMEHEADERSIZE_MAX { + return Err(ErrorCode::DstSizeTooSmall); + } + if params.format == Format::Zstd1 { + write_le32(dst, 0, ZSTD_MAGICNUMBER)?; + pos = 4; + } + dst.put(pos, frame_header_description_byte)?; + pos += 1; + if !single_segment { + dst.put(pos, window_log_byte)?; + pos += 1; + } + match dict_id_size_code { + 0 => {} + 1 => { + dst.put(pos, dict_id as u8)?; + pos += 1; + } + 2 => { + write_le16(dst, pos, dict_id as u16)?; + pos += 2; + } + _ => { + write_le32(dst, pos, dict_id)?; + pos += 4; + } + } + match fcs_code { + 0 => { + if single_segment { + dst.put(pos, pledged_src_size as u8)?; + pos += 1; + } + } + 1 => { + write_le16(dst, pos, (pledged_src_size - 256) as u16)?; + pos += 2; + } + 2 => { + write_le32(dst, pos, pledged_src_size as u32)?; + pos += 4; + } + _ => { + write_le64(dst, pos, pledged_src_size)?; + pos += 8; + } + } + Ok(pos) +} + +/// `ZSTD_writeLastEmptyBlock` +pub(crate) fn write_last_empty_block(dst: &mut [u8]) -> Result { + if dst.len() < ZSTD_BLOCKHEADERSIZE { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, 1 + ((BlockType::Raw as u32) << 1))?; + Ok(ZSTD_BLOCKHEADERSIZE) +} + +/// `ZSTD_BuildSeqStore_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum BuildSeqStore { + Compress, + NoCompress, +} + +/// The C code stops splitting at 196 parts, but the calls under way at that time each still add one, past the end of +/// its array, in memory that happens to be free. +const ZSTD_MAX_NB_BLOCK_SPLITS: usize = 196; +const PARTITIONS_LEN: usize = ZSTD_MAX_NB_BLOCK_SPLITS + 20; + +/// `seqStoreSplits` +struct SeqStoreSplits { + split_locations: [u32; PARTITIONS_LEN], + idx: usize, +} + +/// The sizes of the buffers that streaming takes. +pub(crate) struct BufferSizes { + pub(crate) buff_in_size: usize, + pub(crate) buff_out_size: usize, +} + +/// The part of a `ZSTD_CCtx` that writes frames from blocks of input. +pub(crate) struct FrameCompressor { + pub(crate) stage: CompressionStage, + pub(crate) applied_params: CCtxParams, + pub(crate) dict_id: u32, + pub(crate) dict_content_size: usize, + workspace: Workspace, + pub(crate) block_size_max: usize, + pub(crate) pledged_src_size_plus_one: u64, + pub(crate) consumed_src_size: u64, + pub(crate) produced_c_size: u64, + xxh_state: Xxh64, + is_first_block: bool, + initialized: bool, + seq_store: SeqStore, + ldm_state: LdmState, + ldm_sequences: RawSeqStore, + extern_seq_store: RawSeqStore, + prev_c_block: Box, + next_c_block: Box, + match_state: MatchState, +} + +const fn empty_raw_seq_store() -> RawSeqStore { + RawSeqStore { + seq: Vec::new(), + pos: 0, + pos_in_sequence: 0, + size: 0, + capacity: 0, + } +} + +impl FrameCompressor { + pub(crate) fn new() -> Self { + Self { + stage: CompressionStage::Created, + applied_params: CCtxParams::new(0), + dict_id: 0, + dict_content_size: 0, + workspace: Workspace::new(), + block_size_max: 0, + pledged_src_size_plus_one: 0, + consumed_src_size: 0, + produced_c_size: 0, + xxh_state: Xxh64::new(), + is_first_block: true, + initialized: false, + seq_store: SeqStore::new(), + ldm_state: LdmState::new(), + ldm_sequences: empty_raw_seq_store(), + extern_seq_store: empty_raw_seq_store(), + prev_c_block: Box::new(CompressedBlockState::new()), + next_c_block: Box::new(CompressedBlockState::new()), + match_state: MatchState::new(), + } + } + + /// `ZSTD_resetCCtx_internal` + fn reset_internal( + &mut self, + params: &CCtxParams, + pledged_src_size: u64, + loaded_dict_size: usize, + crp: CompResetPolicy, + buffered: bool, + ) -> Result { + self.is_first_block = true; + self.applied_params = *params; + if self.applied_params.ldm_params.enable_ldm == ParamSwitch::Enable { + ldm::adjust_parameters(&mut self.applied_params.ldm_params, ¶ms.c_params); + } + let params = self.applied_params; + + let window_size = (1u64 << params.c_params.window_log) + .min(pledged_src_size) + .max(1) as usize; + let block_size = ZSTD_BLOCKSIZE_MAX.min(window_size); + let max_nb_seq = max_nb_seq(block_size, params.c_params.min_match); + let sizes = BufferSizes { + buff_out_size: if buffered { + compress_bound(block_size) + 1 + } else { + 0 + }, + buff_in_size: if buffered { + window_size + block_size + } else { + 0 + }, + }; + let max_nb_ldm_seq = ldm::get_max_nb_seq(params.ldm_params, block_size); + + let index_too_close = index_too_close_to_max(&self.match_state.window); + let dict_too_big = loaded_dict_size > ZSTD_CHUNKSIZE_MAX as usize; + let mut needs_index_reset = if index_too_close || dict_too_big || !self.initialized { + IndexResetPolicy::Reset + } else { + IndexResetPolicy::Continue + }; + if self.workspace.reset(&WorkspaceNeeds { + c_params: ¶ms.c_params, + ldm_params: ¶ms.ldm_params, + use_row_match_finder: params.use_row_match_finder, + buff_in_size: sizes.buff_in_size, + buff_out_size: sizes.buff_out_size, + block_size, + }) { + needs_index_reset = IndexResetPolicy::Reset; + } + + self.match_state.c_params = params.c_params; + self.pledged_src_size_plus_one = pledged_src_size.wrapping_add(1); + self.consumed_src_size = 0; + self.produced_c_size = 0; + if pledged_src_size == ZSTD_CONTENTSIZE_UNKNOWN { + self.applied_params.f_params.content_size_flag = false; + } + self.block_size_max = block_size; + + self.xxh_state = Xxh64::new(); + self.stage = CompressionStage::Init; + self.dict_id = 0; + self.dict_content_size = 0; + + self.prev_c_block.reset(); + + reset_match_state( + &mut self.match_state, + ¶ms.c_params, + params.use_row_match_finder, + crp, + needs_index_reset, + ResetTarget::CCtx, + )?; + + if params.ldm_params.enable_ldm == ParamSwitch::Enable { + let ldm_h_size = 1usize << params.ldm_params.hash_log; + reset_struct_table(&mut self.ldm_state.hash_table, ldm_h_size, true)?; + reset_struct_table(&mut self.ldm_sequences.seq, max_nb_ldm_seq, false)?; + self.ldm_sequences.capacity = max_nb_ldm_seq; + self.ldm_state.window = Window::new(); + self.ldm_state.loaded_dict_end = 0; + let num_buckets = + 1usize << (params.ldm_params.hash_log - params.ldm_params.bucket_size_log); + reset_table(&mut self.ldm_state.bucket_offsets, num_buckets, true)?; + } else { + self.ldm_state.hash_table = Vec::new(); + self.ldm_state.bucket_offsets = Vec::new(); + self.ldm_sequences = empty_raw_seq_store(); + } + + if self.seq_store.max_nb_seq != max_nb_seq || self.seq_store.max_nb_lit != block_size { + self.seq_store.reserve(max_nb_seq, block_size)?; + } + self.reference_external_sequences(Vec::new()); + + self.initialized = true; + Ok(sizes) + } + + /// `ZSTD_invalidateRepCodes` + pub(crate) fn invalidate_rep_codes(&mut self) { + self.prev_c_block.rep = [0; ZSTD_REP_NUM]; + } + + /// `ZSTD_resetCCtx_byAttachingCDict` + fn reset_by_attaching_cdict( + &mut self, + cdict: &CDict, + mut params: CCtxParams, + pledged_src_size: u64, + buffered: bool, + ) -> Result { + let window_log = params.c_params.window_log; + // The tables only have to hold the input. + params.c_params = adjust_c_params_internal( + cdict.match_state.c_params, + pledged_src_size, + cdict.dict_content.len(), + CParamMode::AttachDict, + params.use_row_match_finder, + ); + params.c_params.window_log = window_log; + params.use_row_match_finder = cdict.use_row_match_finder; + let sizes = self.reset_internal( + ¶ms, + pledged_src_size, + 0, + CompResetPolicy::MakeClean, + buffered, + )?; + + let cdict_window = &cdict.match_state.window; + let cdict_end = cdict_window.index_of(cdict_window.next_src); + let cdict_len = cdict_end.wrapping_sub(cdict_window.dict_limit); + if cdict_len != 0 { + let ms = &mut self.match_state; + ms.has_dict_match_state = true; + // The indices of the input have to be above those of the dictionary. + if ms.window.dict_limit < cdict_end { + ms.window.next_src = ms.window.base.wrapping_add(cdict_end as usize); + ms.window.clear(); + } + ms.loaded_dict_end = ms.window.dict_limit; + } + + self.dict_id = cdict.dict_id; + self.dict_content_size = cdict.dict_content.len(); + self.prev_c_block.clone_from(&cdict.c_block_state); + Ok(sizes) + } + + /// `ZSTD_resetCCtx_byCopyingCDict` + fn reset_by_copying_cdict( + &mut self, + cdict: &CDict, + mut params: CCtxParams, + pledged_src_size: u64, + buffered: bool, + ) -> Result { + /// `ZSTD_copyCDictTableIntoCCtx` + fn copy_table(dst: &mut [u32], src: &[u32], tagged: bool) -> Result<()> { + if tagged { + for (dst, &tagged_index) in dst.iter_mut().zip(src) { + *dst = tagged_index >> ZSTD_SHORT_CACHE_TAG_BITS; + } + Ok(()) + } else { + dst.write_at(0, src) + } + } + + let cdict_c_params = cdict.match_state.c_params; + let window_log = params.c_params.window_log; + params.c_params = cdict_c_params; + params.c_params.window_log = window_log; + params.use_row_match_finder = cdict.use_row_match_finder; + let sizes = self.reset_internal( + ¶ms, + pledged_src_size, + 0, + CompResetPolicy::LeaveDirty, + buffered, + )?; + + let src = &cdict.match_state; + let dst = &mut self.match_state; + let tagged = cdict_indices_are_tagged(&cdict_c_params); + copy_table(&mut dst.hash_table, &src.hash_table, tagged)?; + copy_table(&mut dst.chain_table, &src.chain_table, tagged)?; + if row_match_finder_used(cdict_c_params.strat(), cdict.use_row_match_finder) { + dst.tag_table.write_at(0, &src.tag_table)?; + dst.hash_salt = src.hash_salt; + } + dst.hash_table3.fill(0); + + dst.window = src.window; + dst.next_to_update = src.next_to_update; + dst.loaded_dict_end = src.loaded_dict_end; + + self.dict_id = cdict.dict_id; + self.dict_content_size = cdict.dict_content.len(); + self.prev_c_block.clone_from(&cdict.c_block_state); + Ok(sizes) + } + + /// `ZSTD_compressBegin_internal`: `dict` is content to start the window with, when there is no `input.cdict`. + pub(crate) fn begin( + &mut self, + input: &Input<'_>, + dict: Option, + dict_content_type: DictContentType, + dtlm: DictTableLoadMethod, + params: &CCtxParams, + pledged_src_size: u64, + buffered: bool, + ) -> Result { + if let Some(cdict) = input.cdict + && !cdict.dict_content.is_empty() + { + // `ZSTD_resetCCtx_usingCDict` + return if cdict.should_attach(params, pledged_src_size) { + self.reset_by_attaching_cdict(cdict, *params, pledged_src_size, buffered) + } else { + self.reset_by_copying_cdict(cdict, *params, pledged_src_size, buffered) + }; + } + + let dict_content_size = dict.map_or(0, |dict| dict.len()); + let sizes = self.reset_internal( + params, + pledged_src_size, + dict_content_size, + CompResetPolicy::MakeClean, + buffered, + )?; + self.dict_id = compress_insert_dictionary( + &mut self.prev_c_block, + &mut self.match_state, + Some(&mut self.ldm_state), + &self.applied_params, + &input.bufs, + dict, + dict_content_type, + dtlm, + TableFillPurpose::ForCCtx, + )?; + self.dict_content_size = dict_content_size; + Ok(sizes) + } + + /// `ZSTD_referenceExternalSequences`: long distance matches for the input to come. + pub(crate) fn reference_external_sequences(&mut self, seq: Vec) { + self.extern_seq_store = RawSeqStore { + pos: 0, + pos_in_sequence: 0, + size: seq.len(), + capacity: seq.len(), + seq, + }; + } + + /// Takes back what [`Self::reference_external_sequences`] was given. + pub(crate) fn take_external_sequences(&mut self) -> Vec { + core::mem::replace(&mut self.extern_seq_store, empty_raw_seq_store()).seq + } + + /// `ZSTD_blockState_confirmRepcodesAndEntropyTables` + fn confirm_repcodes_and_entropy_tables(&mut self) { + core::mem::swap(&mut self.prev_c_block, &mut self.next_c_block); + } + + /// A table for offsets that comes from a dictionary is only known to be valid for the first block. + fn downgrade_offcode_repeat_mode(&mut self) { + let mode = &mut self.prev_c_block.entropy.fse.offcode_repeat_mode; + if *mode == FseRepeat::Valid { + *mode = FseRepeat::Check; + } + } + + /// `ZSTD_buildSeqStore` + fn build_seq_store(&mut self, input: &Input<'_>, src: Span) -> Result { + let params = &self.applied_params; + let ms = &mut self.match_state; + let src_size = src.len(); + if src_size < MIN_CBLOCK_SIZE + ZSTD_BLOCKHEADERSIZE + 1 + 1 { + if params.c_params.strat() >= Strategy::BtOpt { + ldm::skip_raw_seq_store_bytes(&mut self.extern_seq_store, src_size)?; + } else { + ldm::skip_sequences( + &mut self.extern_seq_store, + src_size, + params.c_params.min_match, + )?; + } + return Ok(BuildSeqStore::NoCompress); + } + self.seq_store.reset(); + ms.opt.literal_compression_mode = ParamSwitch::Auto; + + { + // Not to spend too long catching up after incompressible data. + let curr = ms.window.index_of(src.start_addr()); + if curr > ms.next_to_update.wrapping_add(384) { + let behind = curr.wrapping_sub(ms.next_to_update).wrapping_sub(384); + ms.next_to_update = curr - behind.min(192); + } + } + + let window = ms.window; + let block = BlockSrc { + buf: input.bufs.get(window.base_buf), + base: window.base.wrapping_sub(window.base_buf.start()), + dict_buf: input.bufs.get(window.dict_buf), + dict_base: window.dict_base.wrapping_sub(window.dict_buf.start()), + start: src.start, + end: src.end, + dms: match input.cdict { + Some(cdict) if ms.has_dict_match_state => Some(cdict.dict_match_state()), + _ => None, + }, + symbol_costs: &self.prev_c_block.entropy, + ldm_seq_store: None, + }; + let dict_mode = ms.dict_mode(block.dms.as_ref()); + let use_row_match_finder = params.use_row_match_finder == ParamSwitch::Enable; + self.next_c_block.rep = self.prev_c_block.rep; + let rep = &mut self.next_c_block.rep; + + let last_ll_size = if self.extern_seq_store.pos < self.extern_seq_store.size { + ldm::block_compress( + &mut self.extern_seq_store, + ms, + &mut self.seq_store, + rep, + use_row_match_finder, + &block, + )? + } else if params.ldm_params.enable_ldm == ParamSwitch::Enable { + let ldm_seq_store = &mut self.ldm_sequences; + ldm_seq_store.pos = 0; + ldm_seq_store.pos_in_sequence = 0; + ldm_seq_store.size = 0; + ldm::generate_sequences( + &mut self.ldm_state, + ldm_seq_store, + ¶ms.ldm_params, + &input.bufs, + src.start, + src.end, + )?; + ldm::block_compress( + ldm_seq_store, + ms, + &mut self.seq_store, + rep, + use_row_match_finder, + &block, + )? + } else { + compress_block( + ms, + &mut self.seq_store, + rep, + &block, + params.c_params.strat(), + use_row_match_finder, + dict_mode, + )? + }; + let last_literals = block.buf.sub(src.end - last_ll_size..src.end)?; + self.seq_store.store_last_literals(last_literals)?; + Ok(BuildSeqStore::Compress) + } + + /// `ZSTD_compressSeqStore_singleBlock` + fn compress_seq_store_single_block( + &mut self, + chunk: &SeqChunk, + d_rep: &mut [u32; ZSTD_REP_NUM], + c_rep: &mut [u32; ZSTD_REP_NUM], + dst: &mut [u8], + src: &[u8], + last_block: bool, + is_partition: bool, + ) -> Result { + const RLE_MAX_LENGTH: usize = 25; + let d_rep_original = *d_rep; + if is_partition { + seq_store_resolve_off_codes(d_rep, c_rep, &mut self.seq_store, chunk)?; + } + + if dst.len() < ZSTD_BLOCKHEADERSIZE { + return Err(ErrorCode::DstSizeTooSmall); + } + let mut c_seqs_size = entropy_compress_seq_store( + &mut self.seq_store, + chunk, + &self.prev_c_block.entropy, + &mut self.next_c_block.entropy, + &self.applied_params, + dst.sub_mut(ZSTD_BLOCKHEADERSIZE..)?, + src.len(), + )?; + + if !self.is_first_block && c_seqs_size < RLE_MAX_LENGTH && is_rle(src) { + // Decoders up to 1.4.3 do not take a first block that is a run. + c_seqs_size = 1; + } + + let c_size = match c_seqs_size { + 0 => { + *d_rep = d_rep_original; + no_compress_block(dst, src, last_block)? + } + 1 => { + *d_rep = d_rep_original; + rle_compress_block(dst, src.at(0)?, src.len(), last_block)? + } + _ => { + self.confirm_repcodes_and_entropy_tables(); + write_block_header(dst, c_seqs_size, src.len(), last_block)?; + ZSTD_BLOCKHEADERSIZE + c_seqs_size + } + }; + + self.downgrade_offcode_repeat_mode(); + Ok(c_size) + } + + /// `ZSTD_deriveBlockSplitsHelper`: splits the sequences from `start_idx` to `end_idx` in two if that looks + /// smaller, then each half likewise. + fn derive_block_splits_helper( + &mut self, + splits: &mut SeqStoreSplits, + start_idx: usize, + end_idx: usize, + entropy_metadata: &mut EntropyCTablesMetadata, + ) -> Result<()> { + const MIN_SEQUENCES_BLOCK_SPLITTING: usize = 300; + let mid_idx = (start_idx + end_idx) / 2; + + if end_idx - start_idx < MIN_SEQUENCES_BLOCK_SPLITTING + || splits.idx >= ZSTD_MAX_NB_BLOCK_SPLITS + { + return Ok(()); + } + let mut estimate = |from: usize, to: usize| -> Result> { + let chunk = derive_seq_store_chunk(&self.seq_store, from, to)?; + Ok(build_entropy_statistics_and_estimate_sub_block_size( + &mut self.seq_store, + &chunk, + &self.prev_c_block.entropy, + &mut self.next_c_block.entropy, + &self.applied_params, + entropy_metadata, + )) + }; + let estimated_original_size = estimate(start_idx, end_idx)?; + let estimated_first_half_size = estimate(start_idx, mid_idx)?; + let estimated_second_half_size = estimate(mid_idx, end_idx)?; + let (Ok(original), Ok(first_half), Ok(second_half)) = ( + estimated_original_size, + estimated_first_half_size, + estimated_second_half_size, + ) else { + return Ok(()); + }; + if first_half + second_half < original { + self.derive_block_splits_helper(splits, start_idx, mid_idx, entropy_metadata)?; + splits.split_locations.put(splits.idx, mid_idx as u32)?; + splits.idx += 1; + self.derive_block_splits_helper(splits, mid_idx, end_idx, entropy_metadata)?; + } + Ok(()) + } + + /// `ZSTD_compressBlock_splitBlock_internal` + fn compress_block_split_block_internal( + &mut self, + dst: &mut [u8], + src: &[u8], + last_block: bool, + ) -> Result { + let nb_seq = self.seq_store.sequences.len(); + let block_size = src.len(); + let mut c_size = 0usize; + let mut ip = 0usize; + let mut src_bytes_total = 0usize; + + // `ZSTD_deriveBlockSplits` + let mut splits = SeqStoreSplits { + split_locations: [0; PARTITIONS_LEN], + idx: 0, + }; + if nb_seq > 4 { + let mut entropy_metadata = EntropyCTablesMetadata::new(); + self.derive_block_splits_helper(&mut splits, 0, nb_seq, &mut entropy_metadata)?; + splits.split_locations.put(splits.idx, nb_seq as u32)?; + } + let num_splits = splits.idx; + let partitions = &splits.split_locations; + + let mut d_rep = self.prev_c_block.rep; + let mut c_rep = self.prev_c_block.rep; + + if num_splits == 0 { + let whole = self.seq_store.whole(); + return self.compress_seq_store_single_block( + &whole, &mut d_rep, &mut c_rep, dst, src, last_block, false, + ); + } + + let mut curr_seq_store = + derive_seq_store_chunk(&self.seq_store, 0, partitions.at(0)? as usize)?; + for i in 0..=num_splits { + let last_partition = i == num_splits; + let mut last_block_entire_src = false; + + let mut src_bytes = count_seq_store_literals_bytes(&self.seq_store, &curr_seq_store)? + + count_seq_store_match_bytes(&self.seq_store, &curr_seq_store)?; + src_bytes_total += src_bytes; + let mut next_seq_store = curr_seq_store; + if last_partition { + // The last literals are in no sequence. + src_bytes += block_size - src_bytes_total; + last_block_entire_src = last_block; + } else { + next_seq_store = derive_seq_store_chunk( + &self.seq_store, + partitions.at(i)? as usize, + partitions.at(i + 1)? as usize, + )?; + } + + let c_size_chunk = self.compress_seq_store_single_block( + &curr_seq_store, + &mut d_rep, + &mut c_rep, + dst.sub_mut(c_size..)?, + src.span(ip, src_bytes)?, + last_block_entire_src, + true, + )?; + + ip += src_bytes; + c_size += c_size_chunk; + curr_seq_store = next_seq_store; + } + // What the decoder ends up with, which is not what the sequences were found with if a part is uncompressed. + self.prev_c_block.rep = d_rep; + Ok(c_size) + } + + /// `ZSTD_compressBlock_splitBlock` + fn compress_block_split_block( + &mut self, + dst: &mut [u8], + input: &Input<'_>, + src: Span, + last_block: bool, + ) -> Result { + let bytes = src.bytes(&input.bufs)?; + if self.build_seq_store(input, src)? == BuildSeqStore::NoCompress { + self.downgrade_offcode_repeat_mode(); + return no_compress_block(dst, bytes, last_block); + } + self.compress_block_split_block_internal(dst, bytes, last_block) + } + + /// `ZSTD_compressBlock_internal`: the content of a block, which is 1 byte for a run. Returns 0 if the block is + /// better left uncompressed. + fn compress_block_internal( + &mut self, + dst: &mut [u8], + input: &Input<'_>, + src: Span, + ) -> Result { + const RLE_MAX_LENGTH: usize = 25; + let bytes = src.bytes(&input.bufs)?; + + let c_size = match self.build_seq_store(input, src)? { + BuildSeqStore::NoCompress => Ok(0), + BuildSeqStore::Compress => { + let whole = self.seq_store.whole(); + let mut c_size = entropy_compress_seq_store( + &mut self.seq_store, + &whole, + &self.prev_c_block.entropy, + &mut self.next_c_block.entropy, + &self.applied_params, + dst, + bytes.len(), + ); + if !self.is_first_block + && c_size.is_ok_and(|c_size| c_size < RLE_MAX_LENGTH) + && is_rle(bytes) + { + dst.put(0, bytes.at(0)?)?; + c_size = Ok(1); + } + c_size + } + }; + + if c_size.is_ok_and(|c_size| c_size > 1) { + self.confirm_repcodes_and_entropy_tables(); + } + self.downgrade_offcode_repeat_mode(); + c_size + } + + /// `ZSTD_compressBlock_targetCBlockSize` + fn compress_block_target_c_block_size( + &mut self, + dst: &mut [u8], + input: &Input<'_>, + src: Span, + last_block: bool, + ) -> Result { + let bytes = src.bytes(&input.bufs)?; + let bss = self.build_seq_store(input, src)?; + let c_size = self.compress_block_target_c_block_size_body(dst, bytes, bss, last_block)?; + self.downgrade_offcode_repeat_mode(); + Ok(c_size) + } + + /// `ZSTD_compressBlock_targetCBlockSize_body` + fn compress_block_target_c_block_size_body( + &mut self, + dst: &mut [u8], + src: &[u8], + bss: BuildSeqStore, + last_block: bool, + ) -> Result { + if bss == BuildSeqStore::Compress { + if !self.is_first_block && maybe_rle(&self.seq_store) && is_rle(src) { + return rle_compress_block(dst, src.at(0)?, src.len(), last_block); + } + match compress_super_block( + &mut self.seq_store, + &self.prev_c_block, + &mut self.next_c_block, + &self.applied_params, + dst, + src, + last_block, + ) { + Err(ErrorCode::DstSizeTooSmall) => {} + c_size => { + let c_size = c_size?; + let max_c_size = src + .len() + .wrapping_sub(min_gain(src.len(), self.applied_params.c_params.strat())); + if c_size != 0 && c_size < max_c_size.wrapping_add(ZSTD_BLOCKHEADERSIZE) { + self.confirm_repcodes_and_entropy_tables(); + return Ok(c_size); + } + } + } + } + no_compress_block(dst, src, last_block) + } + + /// `ZSTD_optimalBlockSize`: `src` is what remains of the input. + fn optimal_block_size(&self, src: &[u8], savings: i64) -> Result { + const SPLIT_LEVELS: [i32; 10] = [0, 0, 1, 2, 2, 3, 3, 4, 4, 4]; + let block_size_max = self.block_size_max; + // Only full blocks are worth looking into. + if src.len() < 128 << 10 || block_size_max < 128 << 10 { + return Ok(src.len().min(block_size_max)); + } + // Data that does not compress has to be in full blocks, for the output to stay within `compress_bound`. + if savings < 3 { + return Ok(128 << 10); + } + let split_level = SPLIT_LEVELS.at(self.applied_params.c_params.strat() as usize)?; + presplit::split_block(src.sub(..block_size_max)?, split_level) + } + + /// `ZSTD_compress_frameChunk` + fn compress_frame_chunk( + &mut self, + dst: &mut [u8], + input: &Input<'_>, + src: Span, + last_frame_chunk: bool, + ) -> Result { + let bytes = src.bytes(&input.bufs)?; + let mut ip = src.start; + let mut op = 0usize; + let max_dist = 1u32 << self.applied_params.c_params.window_log; + let mut savings = self.consumed_src_size as i64 - self.produced_c_size as i64; + + if self.applied_params.f_params.checksum_flag && !bytes.is_empty() { + self.xxh_state.update(bytes); + } + + while ip < src.end { + let remaining = src.end - ip; + let block_size = self.optimal_block_size(bytes.sub(ip - src.start..)?, savings)?; + let last_block = last_frame_chunk && block_size == remaining; + let block = Span { + buf: src.buf, + start: ip, + end: ip + block_size, + }; + let out = dst.sub_mut(op..)?; + + if out.len() < ZSTD_BLOCKHEADERSIZE + MIN_CBLOCK_SIZE + 1 { + return Err(ErrorCode::DstSizeTooSmall); + } + + let ms = &mut self.match_state; + overflow_correct_if_needed( + ms, + &self.applied_params, + block.start_addr(), + block.end_addr(), + ); + if (ms.window).dict_is_invalid(block.end_addr(), max_dist, ms.loaded_dict_end) { + ms.loaded_dict_end = 0; + ms.has_dict_match_state = false; + } + if (ms.window).enforce_max_dist(block.start_addr(), max_dist, ms.loaded_dict_end) { + ms.loaded_dict_end = 0; + ms.has_dict_match_state = false; + } + ms.next_to_update = ms.next_to_update.max(ms.window.low_limit); + + let c_size = if self.applied_params.target_c_block_size != 0 { + self.compress_block_target_c_block_size(out, input, block, last_block)? + } else if self.applied_params.post_block_splitter == ParamSwitch::Enable { + self.compress_block_split_block(out, input, block, last_block)? + } else { + let c_size = self.compress_block_internal( + out.sub_mut(ZSTD_BLOCKHEADERSIZE..)?, + input, + block, + )?; + if c_size == 0 { + no_compress_block(out, block.bytes(&input.bufs)?, last_block)? + } else { + write_block_header(out, c_size, block_size, last_block)?; + c_size + ZSTD_BLOCKHEADERSIZE + } + }; + + savings += block_size as i64 - c_size as i64; + ip += block_size; + op += c_size; + self.is_first_block = false; + } + + if last_frame_chunk && op > 0 { + self.stage = CompressionStage::Ending; + } + Ok(op) + } + + /// `ZSTD_compressContinue_internal` + fn compress_continue_internal( + &mut self, + dst: &mut [u8], + input: &Input<'_>, + src: Span, + last_frame_chunk: bool, + ) -> Result { + let mut fh_size = 0usize; + if self.stage == CompressionStage::Created { + return Err(ErrorCode::StageWrong); + } + + if self.stage == CompressionStage::Init { + fh_size = write_frame_header( + dst, + &self.applied_params, + self.pledged_src_size_plus_one.wrapping_sub(1), + self.dict_id, + )?; + self.stage = CompressionStage::Ongoing; + } + + if src.len() == 0 { + return Ok(fh_size); + } + + let ms = &mut self.match_state; + if !(ms.window).update(src.start_addr(), src.buf, src.len(), ms.force_non_contiguous) { + ms.force_non_contiguous = false; + ms.next_to_update = ms.window.dict_limit; + } + if self.applied_params.ldm_params.enable_ldm == ParamSwitch::Enable { + (self.ldm_state.window).update(src.start_addr(), src.buf, src.len(), false); + } + + let c_size = + self.compress_frame_chunk(dst.sub_mut(fh_size..)?, input, src, last_frame_chunk)?; + self.consumed_src_size += src.len() as u64; + self.produced_c_size += (c_size + fh_size) as u64; + if self.pledged_src_size_plus_one != 0 + && self.consumed_src_size + 1 > self.pledged_src_size_plus_one + { + return Err(ErrorCode::SrcSizeWrong); + } + Ok(c_size + fh_size) + } + + /// `ZSTD_compressContinue` + pub(crate) fn compress_continue( + &mut self, + dst: &mut [u8], + input: &Input<'_>, + src: Span, + ) -> Result { + self.compress_continue_internal(dst, input, src, false) + } + + /// `ZSTD_writeEpilogue` + fn write_epilogue(&mut self, dst: &mut [u8]) -> Result { + let mut op = 0usize; + if self.stage == CompressionStage::Created { + return Err(ErrorCode::StageWrong); + } + + // An empty frame. + if self.stage == CompressionStage::Init { + op += write_frame_header(dst, &self.applied_params, 0, 0)?; + self.stage = CompressionStage::Ongoing; + } + + if self.stage != CompressionStage::Ending { + // One last empty block, to mark the end of the frame. + if dst.len() - op < 3 { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, op, 1 + ((BlockType::Raw as u32) << 1))?; + op += ZSTD_BLOCKHEADERSIZE; + } + + if self.applied_params.f_params.checksum_flag { + let checksum = self.xxh_state.digest() as u32; + if dst.len() - op < 4 { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le32(dst, op, checksum)?; + op += 4; + } + + self.stage = CompressionStage::Created; + Ok(op) + } + + /// `ZSTD_compressEnd` + pub(crate) fn compress_end( + &mut self, + dst: &mut [u8], + input: &Input<'_>, + src: Span, + ) -> Result { + let c_size = self.compress_continue_internal(dst, input, src, true)?; + let end_result = self.write_epilogue(dst.sub_mut(c_size..)?)?; + if self.pledged_src_size_plus_one != 0 + && self.pledged_src_size_plus_one != self.consumed_src_size + 1 + { + return Err(ErrorCode::SrcSizeWrong); + } + Ok(c_size + end_result) + } +} diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs new file mode 100644 index 000000000000..0b3fd60ef1d7 --- /dev/null +++ b/src/zstd/compress/double_fast.rs @@ -0,0 +1,785 @@ +//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. + +use super::fast::{FAST_HASH_FILL_STEP, Located, count_backwards, restore_rep}; +use super::internal::{ + BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, + write_tagged_index, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, read_le32, read_le64}; + +/// `ZSTD_fillDoubleHashTableForCDict` +fn fill_double_hash_table_for_cdict( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits_l = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let h_bits_s = ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; + let mls = ms.c_params.min_match; + let hash_large = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + // The positions in between only go in the table of long hashes, where there is nothing yet. + while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { + let curr = prefix.index_of(ip); + for i in 0..FAST_HASH_FILL_STEP { + let sm_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; + let lg_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; + if i == 0 { + write_tagged_index(hash_small, sm_hash_and_tag, curr + i)?; + } + if i == 0 || hash_large.at(lg_hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { + write_tagged_index(hash_large, lg_hash_and_tag, curr + i)?; + } + if dtlm == DictTableLoadMethod::Fast { + break; + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillDoubleHashTableForCCtx` +fn fill_double_hash_table_for_cctx( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let mls = ms.c_params.min_match; + let hash_large = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + // The positions in between only go in the table of long hashes, where there is nothing yet. + while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { + let curr = prefix.index_of(ip); + for i in 0..FAST_HASH_FILL_STEP { + let sm_hash = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; + let lg_hash = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; + if i == 0 { + hash_small.put(sm_hash, curr + i)?; + } + if i == 0 || hash_large.at(lg_hash)? == 0 { + hash_large.put(lg_hash, curr + i)?; + } + if dtlm == DictTableLoadMethod::Fast { + break; + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_double_hash_table( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, + tfp: TableFillPurpose, +) -> Result<()> { + match tfp { + TableFillPurpose::ForCDict => fill_double_hash_table_for_cdict(ms, prefix, end, dtlm), + TableFillPurpose::ForCCtx => fill_double_hash_table_for_cctx(ms, prefix, end, dtlm), + } +} + +/// `ZSTD_compressBlock_doubleFast_noDict_generic` +fn compress_block_double_fast_no_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; + + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let window_log = ms.c_params.window_log; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, window_log); + let prefix_lowest = src.pos_of(prefix_lowest_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip = istart + usize::from(istart == prefix_lowest); + let [mut offset_1, mut offset_2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let current = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(current, window_log); + let max_rep = current.wrapping_sub(window_low); + if offset_2 > max_rep { + offset_saved2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved1 = offset_1; + offset_1 = 0; + } + } + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + // One turn per match. + 'start: loop { + let mut step = 1usize; + let mut next_step = ip + K_STEP_INCR; + let mut ip1 = ip + step; + if ip1 > ilimit { + break 'start; + } + + let mut hl0 = hash_ptr(buf, ip, h_bits_l, 8)?; + let mut idxl0 = hash_long.at(hl0)?; + let mut curr; + + let m_length = 'match_stored: { + // One turn per position. Ends with the offset and the length of a match, which `ip` is moved to the start + // of. + let (offset, m_length, hl1) = loop { + let hs0 = hash_ptr(buf, ip, h_bits_s, MLS)?; + let idxs0 = hash_small.at(hs0)?; + curr = src.index_of(ip); + hash_long.put(hl0, curr)?; + hash_small.put(hs0, curr)?; + + if (offset_1 > 0) + & (read_le32(buf, (ip + 1).wrapping_sub(offset_1 as usize))? + == read_le32(buf, ip + 1)?) + { + let m_length = count( + buf, + ip + 1 + 4, + buf, + (ip + 1 + 4).wrapping_sub(offset_1 as usize), + iend, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break 'match_stored m_length; + } + + let hl1 = hash_ptr(buf, ip1, h_bits_l, 8)?; + + let matchl0 = src.pos_of(idxl0); + if idxl0 >= prefix_lowest_index && read_le64(buf, matchl0)? == read_le64(buf, ip)? { + let m_length = count(buf, ip + 8, buf, matchl0 + 8, iend)? + 8; + let offset = ip.wrapping_sub(matchl0) as u32; + let back = count_backwards(buf, ip, anchor, buf, matchl0, prefix_lowest); + ip -= back; + break (offset, m_length + back, hl1); + } + + let idxl1 = hash_long.at(hl1)?; + + let mut matchs0 = src.pos_of(idxs0); + if idxs0 >= prefix_lowest_index && read_le32(buf, matchs0)? == read_le32(buf, ip)? { + // A short match: a longer one at the next position is preferred. + let mut m_length = count(buf, ip + 4, buf, matchs0 + 4, iend)? + 4; + let mut offset = ip.wrapping_sub(matchs0) as u32; + let matchl1 = src.pos_of(idxl1); + if idxl1 > prefix_lowest_index + && read_le64(buf, matchl1)? == read_le64(buf, ip1)? + { + let l1len = count(buf, ip1 + 8, buf, matchl1 + 8, iend)? + 8; + if l1len > m_length { + ip = ip1; + m_length = l1len; + offset = ip.wrapping_sub(matchl1) as u32; + matchs0 = matchl1; + } + } + let back = count_backwards(buf, ip, anchor, buf, matchs0, prefix_lowest); + ip -= back; + break (offset, m_length + back, hl1); + } + + if ip1 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + ip = ip1; + ip1 += step; + + hl0 = hl1; + idxl0 = idxl1; + if ip1 > ilimit { + break 'start; + } + }; + + offset_2 = offset_1; + offset_1 = offset; + // With a larger step, ip1 may be past the end of the match. + if step < 4 { + hash_long.put(hl1, src.index_of(ip1))?; + } + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + m_length + }; + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit + && (offset_2 > 0) + & (read_le32(buf, ip)? == read_le32(buf, ip.wrapping_sub(offset_2 as usize))?) + { + let r_length = count( + buf, + ip + 4, + buf, + (ip + 4).wrapping_sub(offset_2 as usize), + iend, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, src.index_of(ip))?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, src.index_of(ip))?; + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; + ip += r_length; + anchor = ip; + } + } + } + + restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` +fn compress_block_double_fast_dict_match_state_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, ms.c_params.window_log); + let prefix_lowest = src.pos_of(prefix_lowest_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + let [mut offset_1, mut offset_2, _] = *rep; + + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_hash_long = dms.ms.hash_table.as_slice(); + let dict_hash_small = dms.ms.chain_table.as_slice(); + let dict_start_index = dms.ms.window.dict_limit; + let dict_start = dms.pos_of(dict_start_index); + let dict_end = dms.end; + // What to subtract from an index below prefix_lowest_index to get the index in the dictionary. + let dict_index_delta = prefix_lowest_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); + let dict_h_bits_l = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let dict_h_bits_s = dms.ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; + let dict_and_prefix_length = istart + .wrapping_sub(prefix_lowest) + .wrapping_add(dict_end.wrapping_sub(dict_start)) as u32; + + let locate = |index: u32| { + if index < prefix_lowest_index { + Located { + buf: dms.buf, + pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_lowest, + end: iend, + } + } + }; + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip = istart + usize::from(dict_and_prefix_length == 0); + + 'search: while ip < ilimit { + let h2 = hash_ptr(buf, ip, h_bits_l, 8)?; + let h = hash_ptr(buf, ip, h_bits_s, MLS)?; + let dict_hash_and_tag_l = hash_ptr(buf, ip, dict_h_bits_l, 8)?; + let dict_hash_and_tag_s = hash_ptr(buf, ip, dict_h_bits_s, MLS)?; + let dict_match_index_and_tag_l = + dict_hash_long.at(dict_hash_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_match_index_and_tag_s = + dict_hash_small.at(dict_hash_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_tags_match_l = + compare_packed_tags(dict_match_index_and_tag_l as usize, dict_hash_and_tag_l); + let dict_tags_match_s = + compare_packed_tags(dict_match_index_and_tag_s as usize, dict_hash_and_tag_s); + let curr = src.index_of(ip); + let match_index_l = hash_long.at(h2)?; + let mut match_index_s = hash_small.at(h)?; + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + hash_long.put(h2, curr)?; + hash_small.put(h, curr)?; + + let m_length = 'match_stored: { + if index_overlap_check(prefix_lowest_index, rep_index) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? + { + let m_length = count_2segments( + buf, + ip + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_lowest, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break 'match_stored m_length; + } + + // The offset and the length of a match, which `ip` is moved to the start of. + let (offset, m_length) = 'match_found: { + let match_long = src.pos_of(match_index_l); + if match_index_l >= prefix_lowest_index + && read_le64(buf, match_long)? == read_le64(buf, ip)? + { + let m_length = count(buf, ip + 8, buf, match_long + 8, iend)? + 8; + let offset = ip.wrapping_sub(match_long) as u32; + let back = count_backwards(buf, ip, anchor, buf, match_long, prefix_lowest); + ip -= back; + break 'match_found (offset, m_length + back); + } else if dict_tags_match_l { + let dict_match_index_l = + dict_match_index_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match_l = dms.pos_of(dict_match_index_l); + if dict_match_index_l > dict_start_index + && read_le64(dms.buf, dict_match_l)? == read_le64(buf, ip)? + { + let m_length = count_2segments( + buf, + ip + 8, + dms.buf, + dict_match_l + 8, + iend, + dict_end, + prefix_lowest, + )? + 8; + let offset = curr + .wrapping_sub(dict_match_index_l) + .wrapping_sub(dict_index_delta); + let back = + count_backwards(buf, ip, anchor, dms.buf, dict_match_l, dict_start); + ip -= back; + break 'match_found (offset, m_length + back); + } + } + + let short_found = if match_index_s >= prefix_lowest_index { + read_le32(buf, src.pos_of(match_index_s))? == read_le32(buf, ip)? + } else if dict_tags_match_s { + let dict_match_index_s = + dict_match_index_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS; + match_index_s = dict_match_index_s.wrapping_add(dict_index_delta); + dict_match_index_s > dict_start_index + && read_le32(dms.buf, dms.pos_of(dict_match_index_s))? + == read_le32(buf, ip)? + } else { + false + }; + if !short_found { + ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; + continue 'search; + } + + // A short match: a long one at the next position is preferred. + let hl3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; + let dict_hash_and_tag_l3 = hash_ptr(buf, ip + 1, dict_h_bits_l, 8)?; + let match_index_l3 = hash_long.at(hl3)?; + let dict_match_index_and_tag_l3 = + dict_hash_long.at(dict_hash_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_tags_match_l3 = + compare_packed_tags(dict_match_index_and_tag_l3 as usize, dict_hash_and_tag_l3); + let match_l3 = src.pos_of(match_index_l3); + hash_long.put(hl3, curr + 1)?; + + if match_index_l3 >= prefix_lowest_index + && read_le64(buf, match_l3)? == read_le64(buf, ip + 1)? + { + let m_length = count(buf, ip + 9, buf, match_l3 + 8, iend)? + 8; + ip += 1; + let offset = ip.wrapping_sub(match_l3) as u32; + let back = count_backwards(buf, ip, anchor, buf, match_l3, prefix_lowest); + ip -= back; + break 'match_found (offset, m_length + back); + } else if dict_tags_match_l3 { + let dict_match_index_l3 = + dict_match_index_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match_l3 = dms.pos_of(dict_match_index_l3); + if dict_match_index_l3 > dict_start_index + && read_le64(dms.buf, dict_match_l3)? == read_le64(buf, ip + 1)? + { + let m_length = count_2segments( + buf, + ip + 1 + 8, + dms.buf, + dict_match_l3 + 8, + iend, + dict_end, + prefix_lowest, + )? + 8; + ip += 1; + let offset = curr + .wrapping_add(1) + .wrapping_sub(dict_match_index_l3) + .wrapping_sub(dict_index_delta); + let back = + count_backwards(buf, ip, anchor, dms.buf, dict_match_l3, dict_start); + ip -= back; + break 'match_found (offset, m_length + back); + } + } + + let match_s = locate(match_index_s); + let m_length = count_2segments( + buf, + ip + 4, + match_s.buf, + match_s.pos + 4, + iend, + match_s.end, + prefix_lowest, + )? + 4; + let offset = curr.wrapping_sub(match_index_s); + let back = + count_backwards(buf, ip, anchor, match_s.buf, match_s.pos, match_s.start); + ip -= back; + (offset, m_length + back) + }; + + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + m_length + }; + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit { + let current2 = src.index_of(ip); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !(index_overlap_check(prefix_lowest_index, rep_index2) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_lowest, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; + ip += rep_length2; + anchor = ip; + } + } + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast` +pub(crate) fn compress_block_double_fast( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_no_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_no_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_no_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_no_dict_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState` +pub(crate) fn compress_block_double_fast_dict_match_state( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_doubleFast_extDict_generic` +fn compress_block_double_fast_ext_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); + let dict_start_index = low_limit; + let dict_limit = ms.window.dict_limit; + let prefix_start_index = dict_limit.max(low_limit); + let prefix_start = src.pos_of(prefix_start_index); + let dict_start = src.dict_pos_of(dict_start_index); + let dict_end = src.dict_pos_of(prefix_start_index); + let [mut offset_1, mut offset_2, _] = *rep; + + // Nothing of the older segment is within reach. + if prefix_start_index == dict_start_index { + return compress_block_double_fast(ms, seq_store, rep, src); + } + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: src.dict_buf, + pos: src.dict_pos_of(index), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip = istart; + + while ip < ilimit { + let h_small = hash_ptr(buf, ip, h_bits_s, MLS)?; + let match_index = hash_small.at(h_small)?; + let match_s = locate(match_index); + + let h_long = hash_ptr(buf, ip, h_bits_l, 8)?; + let match_long_index = hash_long.at(h_long)?; + let match_long = locate(match_long_index); + + let curr = src.index_of(ip); + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + hash_long.put(h_long, curr)?; + hash_small.put(h_small, curr)?; + + let m_length; + if (index_overlap_check(prefix_start_index, rep_index) + & (offset_1 <= curr.wrapping_add(1).wrapping_sub(dict_start_index))) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? + { + m_length = count_2segments( + buf, + ip + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_start, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + } else { + // The match, how much of it is known, and its offset. `ip` is moved to where it is for. + let (found, known, offset) = if match_long_index > dict_start_index + && read_le64(match_long.buf, match_long.pos)? == read_le64(buf, ip)? + { + (match_long, 8, curr.wrapping_sub(match_long_index)) + } else if match_index > dict_start_index + && read_le32(match_s.buf, match_s.pos)? == read_le32(buf, ip)? + { + // A short match: a long one at the next position is preferred. + let h3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; + let match_index3 = hash_long.at(h3)?; + let match3 = locate(match_index3); + hash_long.put(h3, curr + 1)?; + if match_index3 > dict_start_index + && read_le64(match3.buf, match3.pos)? == read_le64(buf, ip + 1)? + { + ip += 1; + (match3, 8, curr.wrapping_add(1).wrapping_sub(match_index3)) + } else { + (match_s, 4, curr.wrapping_sub(match_index)) + } + } else { + ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; + continue; + }; + let forwards = count_2segments( + buf, + ip + known, + found.buf, + found.pos + known, + iend, + found.end, + prefix_start, + )? + known; + let back = count_backwards(buf, ip, anchor, found.buf, found.pos, found.start); + ip -= back; + m_length = forwards + back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + } + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit { + let current2 = src.index_of(ip); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !((index_overlap_check(prefix_start_index, rep_index2) + & (offset_2 <= current2.wrapping_sub(dict_start_index))) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; + ip += rep_length2; + anchor = ip; + } + } + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast_extDict` +pub(crate) fn compress_block_double_fast_ext_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), + } +} diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs new file mode 100644 index 000000000000..4bec128a6786 --- /dev/null +++ b/src/zstd/compress/fast.rs @@ -0,0 +1,829 @@ +//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, + write_tagged_index, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, read_le32}; + +pub(super) const FAST_HASH_FILL_STEP: u32 = 3; + +/// Where an index leads, when it can be in one of two segments. +#[derive(Clone, Copy)] +pub(super) struct Located<'a> { + /// The buffer that the segment is in. + pub(super) buf: &'a [u8], + /// The position of the index in `buf`. + pub(super) pos: usize, + /// Where the segment starts in `buf`. + pub(super) start: usize, + /// Where it ends. + pub(super) end: usize, +} + +/// How many bytes before `ip`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. +#[inline(always)] +pub(super) fn count_backwards( + buf: &[u8], + ip: usize, + anchor: usize, + match_buf: &[u8], + match_pos: usize, + match_low: usize, +) -> usize { + let before_ip = buf.get(anchor..ip).unwrap_or_default(); + let before_match = match_buf.get(match_low..match_pos).unwrap_or_default(); + before_ip + .iter() + .rev() + .zip(before_match.iter().rev()) + .take_while(|(a, b)| a == b) + .count() +} + +/// The repeat offsets to leave at the end of a block, at the start of which those that reached too far were saved and +/// set to 0. +#[inline(always)] +pub(super) fn restore_rep( + rep: &mut [u32; ZSTD_REP_NUM], + offset_1: u32, + offset_2: u32, + offset_saved1: u32, + offset_saved2: u32, +) { + // If offset_1 started invalid and became valid, offset_saved1 is what rotates into the second place. + let offset_saved2 = if offset_saved1 != 0 && offset_1 != 0 { + offset_saved1 + } else { + offset_saved2 + }; + let [rep0, rep1, _] = rep; + *rep0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved1 + }; + *rep1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved2 + }; +} + +/// `ZSTD_fillHashTableForCDict` +fn fill_hash_table_for_cdict( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let mls = ms.c_params.min_match; + let hash_table = ms.hash_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { + let curr = prefix.index_of(ip); + write_tagged_index(hash_table, hash_ptr(buf, ip, h_bits, mls)?, curr)?; + if dtlm == DictTableLoadMethod::Full { + // The positions in between only go where there is nothing yet. + for p in 1..FAST_HASH_FILL_STEP { + let hash_and_tag = hash_ptr(buf, ip + p as usize, h_bits, mls)?; + if hash_table.at(hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { + write_tagged_index(hash_table, hash_and_tag, curr + p)?; + } + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillHashTableForCCtx` +fn fill_hash_table_for_cctx( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits = ms.c_params.hash_log; + let mls = ms.c_params.min_match; + let hash_table = ms.hash_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { + let curr = prefix.index_of(ip); + hash_table.put(hash_ptr(buf, ip, h_bits, mls)?, curr)?; + if dtlm == DictTableLoadMethod::Full { + // The positions in between only go where there is nothing yet. + for p in 1..FAST_HASH_FILL_STEP { + let hash = hash_ptr(buf, ip + p as usize, h_bits, mls)?; + if hash_table.at(hash)? == 0 { + hash_table.put(hash, curr + p)?; + } + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_hash_table( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, + tfp: TableFillPurpose, +) -> Result<()> { + match tfp { + TableFillPurpose::ForCDict => fill_hash_table_for_cdict(ms, prefix, end, dtlm), + TableFillPurpose::ForCCtx => fill_hash_table_for_cctx(ms, prefix, end, dtlm), + } +} + +/// `ZSTD_match4Found_branch`, and `ZSTD_match4Found_cmov`, which only differs in how it avoids reading at an index that +/// is not valid. +#[inline(always)] +fn match4_found( + buf: &[u8], + current: usize, + match_pos: usize, + match_idx: u32, + idx_low_limit: u32, +) -> Result { + Ok(match_idx >= idx_low_limit && read_le32(buf, current)? == read_le32(buf, match_pos)?) +} + +/// `ZSTD_compressBlock_fast_noDict_generic` +/// +/// Each turn of the inner loop looks at two positions, `ip0` and `ip1`, for a match, and at `ip2` for a repeat. The +/// hash of a position is computed, and the table is read, one position ahead of the comparison. +fn compress_block_fast_no_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); + + let hlog = ms.c_params.hash_log; + let window_log = ms.c_params.window_log; + let target_length = ms.c_params.target_length; + // The least step is 2, as two positions are looked at per turn. + let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_start_index = ms.get_lowest_prefix_index(end_index, window_log); + let prefix_start = src.pos_of(prefix_start_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip0 = istart + usize::from(istart == prefix_start); + let [mut rep_offset1, mut rep_offset2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let curr = src.index_of(ip0); + let window_low = ms.get_lowest_prefix_index(curr, window_log); + let max_rep = curr.wrapping_sub(window_low); + if rep_offset2 > max_rep { + offset_saved2 = rep_offset2; + rep_offset2 = 0; + } + if rep_offset1 > max_rep { + offset_saved1 = rep_offset1; + rep_offset1 = 0; + } + } + let hash_table = ms.hash_table.as_mut_slice(); + + 'start: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let mut ip1 = ip0 + 1; + let mut ip2 = ip0 + step; + let mut ip3 = ip2 + 1; + if ip3 >= ilimit { + break 'start; + } + + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let mut match_idx = hash_table.at(hash0)?; + let mut current0; + + // Ends with the start and the length so far of a repeat match, or with `None` for a match at `match_idx`. + let rep_match = loop { + let rval = read_le32(buf, ip2.wrapping_sub(rep_offset1 as usize))?; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if (read_le32(buf, ip2)? == rval) & (rep_offset1 > 0) { + ip0 = ip2; + let match0 = ip0.wrapping_sub(rep_offset1 as usize); + let back = usize::from(buf.at(ip0 - 1)? == buf.at(match0.wrapping_sub(1))?); + ip0 -= back; + // The hash of ip1 is there, and ip1 is before the end of the match. + hash_table.put(hash1, src.index_of(ip1))?; + break Some((match0.wrapping_sub(back), 4 + back)); + } + + if match4_found( + buf, + ip0, + src.pos_of(match_idx), + match_idx, + prefix_start_index, + )? { + hash_table.put(hash1, src.index_of(ip1))?; + break None; + } + + match_idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if match4_found( + buf, + ip0, + src.pos_of(match_idx), + match_idx, + prefix_start_index, + )? { + // ip1 is certain to be before the end of the match only if the step is small. + if step <= 4 { + hash_table.put(hash1, src.index_of(ip1))?; + } + break None; + } + + match_idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + if ip2 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + if ip3 >= ilimit { + break 'start; + } + }; + + let (offcode, match0, mut m_length) = match rep_match { + Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), + None => { + let match0 = src.pos_of(match_idx); + rep_offset2 = rep_offset1; + rep_offset1 = ip0.wrapping_sub(match0) as u32; + let back = count_backwards(buf, ip0, anchor, buf, match0, prefix_start); + ip0 -= back; + (offset_to_offbase(rep_offset1), match0 - back, 4 + back) + } + }; + + m_length += count(buf, ip0 + m_length, buf, match0 + m_length, iend)?; + seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; + ip0 += m_length; + anchor = ip0; + + if ip0 <= ilimit { + hash_table.put( + hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, + current0 + 2, + )?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + if rep_offset2 > 0 { + while ip0 <= ilimit + && read_le32(buf, ip0)? + == read_le32(buf, ip0.wrapping_sub(rep_offset2 as usize))? + { + let r_length = count( + buf, + ip0 + 4, + buf, + (ip0 + 4).wrapping_sub(rep_offset2 as usize), + iend, + )? + 4; + core::mem::swap(&mut rep_offset1, &mut rep_offset2); + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; + ip0 += r_length; + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; + anchor = ip0; + } + } + } + } + + restore_rep(rep, rep_offset1, rep_offset2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast` +pub(crate) fn compress_block_fast( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_no_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_no_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_no_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_no_dict_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_fast_dictMatchState_generic` +fn compress_block_fast_dict_match_state_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; + + let hlog = ms.c_params.hash_log; + let target_length = ms.c_params.target_length; + let step_size = (target_length + u32::from(target_length == 0)) as usize; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let prefix_start_index = ms.window.dict_limit; + let prefix_start = src.pos_of(prefix_start_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + let [mut offset_1, mut offset_2, _] = *rep; + + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_hash_table = dms.ms.hash_table.as_slice(); + let dict_start_index = dms.ms.window.dict_limit; + let dict_start = dms.pos_of(dict_start_index); + let dict_end = dms.end; + // What to subtract from an index below prefix_start_index to get the index in the dictionary. + let dict_index_delta = prefix_start_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); + let dict_and_prefix_length = istart + .wrapping_sub(prefix_start) + .wrapping_add(dict_end) + .wrapping_sub(dict_start) as u32; + let dict_h_bits = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: dms.buf, + pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + let hash_table = ms.hash_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip0 = istart + usize::from(dict_and_prefix_length == 0); + let mut ip1 = istart + step_size; + + 'search: while ip1 <= ilimit { + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + + let dict_hash_and_tag0 = hash_ptr(buf, ip0, dict_h_bits, MLS)?; + let mut dict_match_index_and_tag = + dict_hash_table.at(dict_hash_and_tag0 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let mut dict_tags_match = + compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag0); + + let mut match_index = hash_table.at(hash0)?; + let mut curr = src.index_of(ip0); + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let m_length = loop { + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + let hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let dict_hash_and_tag1 = hash_ptr(buf, ip1, dict_h_bits, MLS)?; + hash_table.put(hash0, curr)?; + + if index_overlap_check(prefix_start_index, rep_index) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip0 + 1)? + { + let m_length = count_2segments( + buf, + ip0 + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_start, + )? + 4; + ip0 += 1; + seq_store.store_seq(ip0 - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break m_length; + } + + if dict_tags_match { + let dict_match_index = dict_match_index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match = dms.pos_of(dict_match_index); + // A match in the dictionary is only taken if there is no candidate in the prefix. + if dict_match_index > dict_start_index + && read_le32(dms.buf, dict_match)? == read_le32(buf, ip0)? + && match_index <= prefix_start_index + { + let offset = curr + .wrapping_sub(dict_match_index) + .wrapping_sub(dict_index_delta); + let mut m_length = count_2segments( + buf, + ip0 + 4, + dms.buf, + dict_match + 4, + iend, + dict_end, + prefix_start, + )? + 4; + let back = count_backwards(buf, ip0, anchor, dms.buf, dict_match, dict_start); + ip0 -= back; + m_length += back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip0 - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + break m_length; + } + } + + let match_pos = src.pos_of(match_index); + if match4_found(buf, ip0, match_pos, match_index, prefix_start_index)? { + let offset = ip0.wrapping_sub(match_pos) as u32; + let mut m_length = count(buf, ip0 + 4, buf, match_pos + 4, iend)? + 4; + let back = count_backwards(buf, ip0, anchor, buf, match_pos, prefix_start); + ip0 -= back; + m_length += back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip0 - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + break m_length; + } + + dict_match_index_and_tag = + dict_hash_table.at(dict_hash_and_tag1 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + dict_tags_match = + compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag1); + match_index = hash_table.at(hash1)?; + + if ip1 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + ip0 = ip1; + ip1 += step; + if ip1 > ilimit { + break 'search; + } + + curr = src.index_of(ip0); + hash0 = hash1; + }; + + ip0 += m_length; + anchor = ip0; + + if ip0 <= ilimit { + hash_table.put(hash_ptr(buf, src.pos_of(curr + 2), hlog, MLS)?, curr + 2)?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + while ip0 <= ilimit { + let current2 = src.index_of(ip0); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !(index_overlap_check(prefix_start_index, rep_index2) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip0 + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, current2)?; + ip0 += rep_length2; + anchor = ip0; + } + } + + ip1 = ip0 + step_size; + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast_dictMatchState` +pub(crate) fn compress_block_fast_dict_match_state( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_fast_extDict_generic` +fn compress_block_fast_ext_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); + + let hlog = ms.c_params.hash_log; + let target_length = ms.c_params.target_length; + let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); + let dict_start_index = low_limit; + let dict_start = src.dict_pos_of(dict_start_index); + let dict_limit = ms.window.dict_limit; + let prefix_start_index = dict_limit.max(low_limit); + let prefix_start = src.pos_of(prefix_start_index); + let dict_end = src.dict_pos_of(prefix_start_index); + + // Nothing of the older segment is within reach. + if prefix_start_index == dict_start_index { + return compress_block_fast(ms, seq_store, rep, src); + } + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip0 = istart; + let [mut offset_1, mut offset_2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let curr = src.index_of(ip0); + let max_rep = curr.wrapping_sub(dict_start_index); + if offset_2 >= max_rep { + offset_saved2 = offset_2; + offset_2 = 0; + } + if offset_1 >= max_rep { + offset_saved1 = offset_1; + offset_1 = 0; + } + } + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: src.dict_buf, + pos: src.dict_pos_of(index), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + // Whether there is a match for the position `ip` at `idx`, which is from the table. + let match_found = |ip: usize, idx: u32| -> Result { + let candidate = locate(idx); + Ok(idx >= dict_start_index + && read_le32(candidate.buf, candidate.pos)? == read_le32(buf, ip)?) + }; + let hash_table = ms.hash_table.as_mut_slice(); + + 'start: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let mut ip1 = ip0 + 1; + let mut ip2 = ip0 + step; + let mut ip3 = ip2 + 1; + if ip3 >= ilimit { + break 'start; + } + + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let mut idx = hash_table.at(hash0)?; + let mut current0; + + // Ends with the start and the length so far of a repeat match, or with `None` for a match at `idx`. + let rep_match = loop { + let current2 = src.index_of(ip2); + let rep_index = current2.wrapping_sub(offset_1); + let mut rep_match = locate(rep_index); + // The 4 bytes have to be in one segment, with a byte before them. + let rep_found = ((prefix_start_index.wrapping_sub(rep_index) >= 4) & (offset_1 > 0)) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip2)?; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if rep_found { + ip0 = ip2; + let back = usize::from( + buf.at(ip0 - 1)? == rep_match.buf.at(rep_match.pos.wrapping_sub(1))?, + ); + ip0 -= back; + rep_match.pos = rep_match.pos.wrapping_sub(back); + break Some((rep_match, 4 + back)); + } + + if match_found(ip0, idx)? { + break None; + } + + idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if match_found(ip0, idx)? { + break None; + } + + idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + if ip2 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + if ip3 >= ilimit { + break 'start; + } + }; + + let (offcode, match0, mut m_length) = match rep_match { + Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), + None => { + let offset = current0.wrapping_sub(idx); + let mut match0 = locate(idx); + offset_2 = offset_1; + offset_1 = offset; + let back = count_backwards(buf, ip0, anchor, match0.buf, match0.pos, match0.start); + ip0 -= back; + match0.pos -= back; + (offset_to_offbase(offset), match0, 4 + back) + } + }; + + m_length += count_2segments( + buf, + ip0 + m_length, + match0.buf, + match0.pos + m_length, + iend, + match0.end, + prefix_start, + )?; + seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; + ip0 += m_length; + anchor = ip0; + + if ip1 < ip0 { + hash_table.put(hash1, src.index_of(ip1))?; + } + + if ip0 <= ilimit { + hash_table.put( + hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, + current0 + 2, + )?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + while ip0 <= ilimit { + let rep_index2 = src.index_of(ip0).wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !((index_overlap_check(prefix_start_index, rep_index2) & (offset_2 > 0)) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip0 + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; + ip0 += rep_length2; + anchor = ip0; + } + } + } + + restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast_extDict` +pub(crate) fn compress_block_fast_ext_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), + } +} diff --git a/src/zstd/compress/fse.rs b/src/zstd/compress/fse.rs new file mode 100644 index 000000000000..82a4a8821b92 --- /dev/null +++ b/src/zstd/compress/fse.rs @@ -0,0 +1,672 @@ +//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. + +use crate::common::bitstream::BitCStream; +use crate::common::{FSE_MAX_TABLELOG, FSE_MIN_TABLELOG, MAX_SEQ, ML_FSE_LOG, fse_table_step}; +use crate::error::{ErrorCode, Result}; +use crate::util::{BUG, SliceExt, highbit32, write_le16}; + +pub(crate) const FSE_NCOUNTBOUND: usize = 512; +const FSE_DEFAULT_TABLELOG: u32 = 11; + +/// `FSE_BLOCKBOUND` +const fn block_bound(size: usize) -> usize { + size + (size >> 7) + 4 + size_of::() +} + +/// `FSE_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum FseRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `FSE_symbolCompressionTransform` +#[derive(Clone, Copy, Default)] +pub(crate) struct FseSymbolTransform { + pub(crate) delta_find_state: i32, + pub(crate) delta_nb_bits: u32, +} + +/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. +#[derive(Clone)] +pub(crate) struct FseCTable { + pub(crate) table_log: u32, + pub(crate) max_symbol_value: u32, + pub(crate) state_table: [u16; 1 << ML_FSE_LOG], + pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], +} + +impl FseCTable { + pub(crate) const fn new() -> Self { + Self { + table_log: 0, + max_symbol_value: 0, + state_table: [0; 1 << ML_FSE_LOG], + symbol_tt: [FseSymbolTransform { + delta_find_state: 0, + delta_nb_bits: 0, + }; MAX_SEQ as usize + 1], + } + } + + /// `FSE_buildCTable_wksp`: `normalized_counter` adds up to `1 << table_log`, a count of -1 standing for 1. + /// [`ErrorCode::TableLogTooLarge`] if the table has no room for it. + pub(crate) fn build( + &mut self, + normalized_counter: &[i16], + max_symbol_value: u32, + table_log: u32, + ) -> Result<()> { + if table_log > ML_FSE_LOG || max_symbol_value > MAX_SEQ { + return Err(ErrorCode::TableLogTooLarge); + } + let table_size = 1usize << table_log; + let table_mask = table_size - 1; + let step = fse_table_step(table_size); + let counts = normalized_counter.sub(..=max_symbol_value as usize)?; + + let mut cumul = [0u16; MAX_SEQ as usize + 1]; + let mut table_symbol = [0u8; 1 << ML_FSE_LOG]; + let mut high_threshold = table_size - 1; + + self.table_log = table_log; + self.max_symbol_value = max_symbol_value; + + // Where the states of each symbol start. The symbols of low probability get the highest positions. + let mut total = 0u16; + for (symbol, (&count, start)) in counts.iter().zip(&mut cumul).enumerate() { + *start = total; + if count == -1 { + total = total.wrapping_add(1); + table_symbol.put(high_threshold, symbol as u8)?; + high_threshold = high_threshold.wrapping_sub(1); + } else { + total = total.wrapping_add(count as u16); + } + } + + // Spread the symbols. + if high_threshold == table_size - 1 { + // No symbol of low probability: lay the symbols down in order, and then every position is used. + let mut spread = [0u8; (1 << ML_FSE_LOG) + 8]; + let mut pos = 0usize; + let mut sv = 0u64; + for &count in counts { + let n = usize::try_from(count).map_err(|_| BUG)?; + *spread.array_at_mut(pos)? = sv.to_le_bytes(); + for i in (8..n).step_by(8) { + *spread.array_at_mut(pos + i)? = sv.to_le_bytes(); + } + pos += n; + sv = sv.wrapping_add(0x0101_0101_0101_0101); + } + + let mut position = 0usize; + let (pairs, _) = spread.sub(..table_size)?.as_chunks::<2>(); + for &[first, second] in pairs { + table_symbol.put(position, first)?; + table_symbol.put((position + step) & table_mask, second)?; + position = (position + 2 * step) & table_mask; + } + } else { + let mut position = 0usize; + for (symbol, &freq) in counts.iter().enumerate() { + for _ in 0..freq { + table_symbol.put(position, symbol as u8)?; + position = (position + step) & table_mask; + while position > high_threshold { + position = (position + step) & table_mask; + } + } + } + } + + // The states, sorted by symbol. + for (u, &symbol) in table_symbol.sub(..table_size)?.iter().enumerate() { + let next = cumul.at_mut(usize::from(symbol))?; + self.state_table + .put(usize::from(*next), (table_size + u) as u16)?; + *next = next.wrapping_add(1); + } + + let mut total = 0u32; + for (&count, transform) in counts.iter().zip(&mut self.symbol_tt) { + match count { + // For `get_max_nb_bits` and `bit_cost`. + 0 => transform.delta_nb_bits = ((table_log + 1) << 16) - (1 << table_log), + -1 | 1 => { + transform.delta_nb_bits = (table_log << 16).wrapping_sub(1 << table_log); + transform.delta_find_state = total.wrapping_sub(1) as i32; + total += 1; + } + _ => { + let count = count as u32; + let max_bits_out = table_log.wrapping_sub(highbit32(count.wrapping_sub(1))); + let min_state_plus = count.wrapping_shl(max_bits_out); + transform.delta_nb_bits = (max_bits_out << 16).wrapping_sub(min_state_plus); + transform.delta_find_state = total.wrapping_sub(count) as i32; + total = total.wrapping_add(count); + } + } + } + Ok(()) + } + + /// `FSE_buildCTable_rle`: a table that encodes `symbol_value`, and nothing else, in 0 bits. + pub(crate) fn build_rle(&mut self, symbol_value: u8) -> Result<()> { + self.table_log = 0; + self.max_symbol_value = u32::from(symbol_value); + self.state_table.sub_mut(..2)?.fill(0); + self.symbol_tt + .put(usize::from(symbol_value), FseSymbolTransform::default()) + } + + /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log + /// if the table has no code for it. + #[inline(always)] + pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 + } + + /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. + /// The table has a code for it. + #[inline(always)] + pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + let min_nb_bits = transform.delta_nb_bits >> 16; + let threshold = (min_nb_bits + 1) << 16; + let table_size = 1u32 << self.table_log; + let delta_from_threshold = + threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); + // A linear interpolation, which is very approximate. + let normalized_delta_from_threshold = + (delta_from_threshold << accuracy_log) >> self.table_log; + let bit_multiplier = 1u32 << accuracy_log; + ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) + } +} + +/// `FSE_CState_t` +pub(crate) struct FseCState<'a> { + value: usize, + table: &'a FseCTable, +} + +impl<'a> FseCState<'a> { + /// `FSE_initCState` + #[inline] + pub(crate) fn new(table: &'a FseCTable) -> Self { + Self { + value: 1 << (table.table_log & 15), + table, + } + } + + /// `FSE_initCState2`: the first symbol costs the fewest bits this way. The table has a code for `symbol`. + #[inline] + pub(crate) fn new2(table: &'a FseCTable, symbol: u32) -> Self { + let mut state = Self::new(table); + let transform = table.symbol_tt.at(symbol as usize).unwrap_or_default(); + let nb_bits_out = transform.delta_nb_bits.wrapping_add(1 << 15) >> 16; + state.value = (nb_bits_out << 16).wrapping_sub(transform.delta_nb_bits) as usize; + state.step(nb_bits_out, transform.delta_find_state); + state + } + + #[inline(always)] + fn step(&mut self, nb_bits_out: u32, delta_find_state: i32) { + let state_table = &self.table.state_table; + let index = (self.value >> (nb_bits_out & (usize::BITS - 1))) + .wrapping_add_signed(delta_find_state as isize); + let next = state_table.get(index & (state_table.len() - 1)); + self.value = usize::from(next.copied().unwrap_or(0)); + } + + /// `FSE_encodeSymbol`: the table has a code for `symbol`. + #[inline(always)] + pub(crate) fn encode_symbol(&mut self, bit_c: &mut BitCStream<'_>, symbol: u32) { + let symbol_tt = &self.table.symbol_tt; + let transform = symbol_tt.at(symbol as usize).unwrap_or_default(); + let nb_bits_out = ((self.value + transform.delta_nb_bits as usize) >> 16) as u32; + bit_c.add_bits(self.value as u64, nb_bits_out); + self.step(nb_bits_out, transform.delta_find_state); + } + + /// `FSE_flushCState` + #[inline(always)] + pub(crate) fn flush(&self, bit_c: &mut BitCStream<'_>) { + bit_c.add_bits(self.value as u64, self.table.table_log); + bit_c.flush_bits(); + } +} + +/// `FSE_NCountWriteBound` +pub(crate) fn ncount_write_bound(max_symbol_value: u32, table_log: u32) -> usize { + let max_header_size = + ((max_symbol_value as usize + 1) * table_log as usize + 4 + 2) / 8 + 1 + 2; + if max_symbol_value != 0 { + max_header_size + } else { + FSE_NCOUNTBOUND + } +} + +/// `FSE_writeNCount_generic`: `table_log` is from 5 to 12. +fn write_ncount_generic( + header: &mut [u8], + normalized_counter: &[i16], + max_symbol_value: u32, + table_log: u32, + write_is_safe: bool, +) -> Result { + let counts = normalized_counter.sub(..=max_symbol_value as usize)?; + let alphabet_size = counts.len(); + let mut out = 0usize; + let table_size = 1i32 << table_log; + let mut bit_stream = table_log - FSE_MIN_TABLELOG; + let mut bit_count = 4i32; + let mut symbol = 0usize; + let mut previous_is_0 = false; + + // The lowest 16 bits of the stream leave it. + let mut write16 = |bit_stream: &mut u32, out: &mut usize| -> Result<()> { + if !write_is_safe && *out + 2 > header.len() { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le16(header, *out, *bit_stream as u16)?; + *out += 2; + *bit_stream >>= 16; + Ok(()) + }; + let shifted = |value: u32, bit_count: i32| value.wrapping_shl(bit_count as u32); + + let mut remaining = table_size + 1; + let mut threshold = table_size; + let mut nb_bits = table_log as i32 + 1; + + while symbol < alphabet_size && remaining > 1 { + if previous_is_0 { + // After a count of 0 comes how many more follow it. + let mut start = symbol; + while symbol < alphabet_size && counts.at(symbol)? == 0 { + symbol += 1; + } + if symbol == alphabet_size { + // The counts do not add up. + break; + } + while symbol >= start + 24 { + start += 24; + bit_stream = bit_stream.wrapping_add(shifted(0xFFFF, bit_count)); + write16(&mut bit_stream, &mut out)?; + } + while symbol >= start + 3 { + start += 3; + bit_stream = bit_stream.wrapping_add(shifted(3, bit_count)); + bit_count += 2; + } + bit_stream = bit_stream.wrapping_add(shifted((symbol - start) as u32, bit_count)); + bit_count += 2; + if bit_count > 16 { + write16(&mut bit_stream, &mut out)?; + bit_count -= 16; + } + } + { + let mut count = i32::from(counts.at(symbol)?); + symbol += 1; + let max = (2 * threshold - 1) - remaining; + remaining -= count.abs(); + // The stored value is the count plus one, since a count can be -1. + count += 1; + if count >= threshold { + count += max; + } + bit_stream = bit_stream.wrapping_add(shifted(count as u32, bit_count)); + bit_count += nb_bits; + bit_count -= i32::from(count < max); + previous_is_0 = count == 1; + if remaining < 1 { + return Err(ErrorCode::Generic); + } + while remaining < threshold { + nb_bits -= 1; + threshold >>= 1; + } + } + if bit_count > 16 { + write16(&mut bit_stream, &mut out)?; + bit_count -= 16; + } + } + + if remaining != 1 { + return Err(ErrorCode::Generic); + } + + let last = out; + write16(&mut bit_stream, &mut out)?; + Ok(last + ((bit_count + 7) / 8) as usize) +} + +/// `FSE_writeNCount`: writes the header that [`crate::common::entropy::read_ncount`] reads, and returns its size. +pub(crate) fn write_ncount( + buffer: &mut [u8], + normalized_counter: &[i16], + max_symbol_value: u32, + table_log: u32, +) -> Result { + if table_log > FSE_MAX_TABLELOG { + return Err(ErrorCode::TableLogTooLarge); + } + if table_log < FSE_MIN_TABLELOG { + return Err(ErrorCode::Generic); + } + let write_is_safe = buffer.len() >= ncount_write_bound(max_symbol_value, table_log); + write_ncount_generic( + buffer, + normalized_counter, + max_symbol_value, + table_log, + write_is_safe, + ) +} + +/// `FSE_minTableLog`: the smallest table log that can represent the distribution. `src_size` is more than 1. +fn min_table_log(src_size: usize, max_symbol_value: u32) -> u32 { + let min_bits_src = highbit32(src_size as u32) + 1; + let min_bits_symbols = highbit32(max_symbol_value) + 2; + min_bits_src.min(min_bits_symbols) +} + +/// `FSE_optimalTableLog_internal`: `src_size` is more than 1. +pub(crate) fn optimal_table_log_internal( + max_table_log: u32, + src_size: usize, + max_symbol_value: u32, + minus: u32, +) -> u32 { + let max_bits_src = highbit32(src_size.wrapping_sub(1) as u32).wrapping_sub(minus); + let mut table_log = max_table_log; + let min_bits = min_table_log(src_size, max_symbol_value); + if table_log == 0 { + table_log = FSE_DEFAULT_TABLELOG; + } + // More accuracy than the input has symbols for is of no use. + if max_bits_src < table_log { + table_log = max_bits_src; + } + if min_bits > table_log { + table_log = min_bits; + } + table_log.clamp(FSE_MIN_TABLELOG, FSE_MAX_TABLELOG) +} + +/// `FSE_optimalTableLog` +pub(crate) fn optimal_table_log(max_table_log: u32, src_size: usize, max_symbol_value: u32) -> u32 { + optimal_table_log_internal(max_table_log, src_size, max_symbol_value, 2) +} + +/// `FSE_normalizeM2`: the second normalization method, for when the first one would take too much from the most +/// frequent symbol. `norm` and `count` are of the same length. +fn normalize_m2( + norm: &mut [i16], + table_log: u32, + count: &[u32], + mut total: usize, + low_prob_count: i16, +) -> Result<()> { + const NOT_YET_ASSIGNED: i16 = -2; + let table_size = 1u32 << table_log; + let mut distributed = 0u32; + + // The symbols of low probability first. + let low_threshold = (total >> table_log) as u32; + let mut low_one = (total.wrapping_mul(3) >> (table_log + 1)) as u32; + + for (norm, &count) in norm.iter_mut().zip(count) { + if count == 0 { + *norm = 0; + } else if count <= low_threshold { + *norm = low_prob_count; + distributed += 1; + total = total.wrapping_sub(count as usize); + } else if count <= low_one { + *norm = 1; + distributed += 1; + total = total.wrapping_sub(count as usize); + } else { + *norm = NOT_YET_ASSIGNED; + } + } + let mut to_distribute = table_size.wrapping_sub(distributed); + + if to_distribute == 0 { + return Ok(()); + } + + if total / to_distribute as usize > low_one as usize { + // The rest would get too much each: more symbols get 1. + low_one = total + .wrapping_mul(3) + .checked_div(to_distribute.wrapping_mul(2) as usize) + .ok_or(BUG)? as u32; + for (norm, &count) in norm.iter_mut().zip(count) { + if *norm == NOT_YET_ASSIGNED && count <= low_one { + *norm = 1; + distributed += 1; + total = total.wrapping_sub(count as usize); + } + } + to_distribute = table_size.wrapping_sub(distributed); + } + + if distributed as usize == count.len() { + // All the values are small. The first symbol of the largest count gets what is left. + let mut max_v = 0usize; + let mut max_c = 0u32; + for (s, &count) in count.iter().enumerate() { + if count > max_c { + max_v = s; + max_c = count; + } + } + let norm = norm.at_mut(max_v)?; + *norm = norm.wrapping_add(to_distribute as i16); + return Ok(()); + } + + if total == 0 { + // All the symbols were of low enough probability to get 1 or less: what is left goes round. + if !norm.iter().any(|&norm| norm > 0) { + return Err(BUG); + } + while to_distribute > 0 { + for norm in norm.iter_mut().filter(|norm| **norm > 0) { + if to_distribute == 0 { + break; + } + to_distribute -= 1; + *norm = norm.wrapping_add(1); + } + } + return Ok(()); + } + + let v_step_log = 62 - table_log; + let mid = (1u64 << (v_step_log - 1)) - 1; + // A scale on what remains. + let r_step = ((1u64 << v_step_log) * u64::from(to_distribute) + mid) + .checked_div(u64::from(total as u32)) + .ok_or(BUG)?; + let mut tmp_total = mid; + for (norm, &count) in norm.iter_mut().zip(count) { + if *norm == NOT_YET_ASSIGNED { + let end = tmp_total.wrapping_add(u64::from(count).wrapping_mul(r_step)); + let s_start = (tmp_total >> v_step_log) as u32; + let s_end = (end >> v_step_log) as u32; + let weight = s_end.wrapping_sub(s_start); + if weight < 1 { + return Err(ErrorCode::Generic); + } + *norm = weight as i16; + tmp_total = end; + } + } + Ok(()) +} + +/// `FSE_normalizeCount`: scales `count`, which adds up to `total`, to add up to `1 << table_log`. Returns the table +/// log, or 0 if there is only one symbol, which is a case for run-length encoding. +pub(crate) fn normalize_count( + normalized_counter: &mut [i16], + mut table_log: u32, + count: &[u32], + total: usize, + max_symbol_value: u32, + use_low_prob_count: bool, +) -> Result { + if table_log == 0 { + table_log = FSE_DEFAULT_TABLELOG; + } + if table_log < FSE_MIN_TABLELOG { + return Err(ErrorCode::Generic); + } + if table_log > FSE_MAX_TABLELOG { + return Err(ErrorCode::TableLogTooLarge); + } + if table_log < min_table_log(total, max_symbol_value) { + return Err(ErrorCode::Generic); + } + + const RTB_TABLE: [u32; 8] = [0, 473195, 504333, 520860, 550000, 700000, 750000, 830000]; + let low_prob_count: i16 = if use_low_prob_count { -1 } else { 1 }; + let scale = 62 - table_log; + let step = (1u64 << 62) + .checked_div(u64::from(total as u32)) + .ok_or(BUG)?; + let v_step = 1u64 << (scale - 20); + let mut still_to_distribute = 1i32 << table_log; + let mut largest = 0usize; + let mut largest_p = 0i16; + let low_threshold = (total >> table_log) as u32; + + let count = count.sub(..=max_symbol_value as usize)?; + let norm = normalized_counter.sub_mut(..=max_symbol_value as usize)?; + + for (s, (norm, &count)) in norm.iter_mut().zip(count).enumerate() { + if count as usize == total { + return Ok(0); + } + if count == 0 { + *norm = 0; + } else if count <= low_threshold { + *norm = low_prob_count; + still_to_distribute -= 1; + } else { + let scaled = u64::from(count).wrapping_mul(step); + let mut proba = (scaled >> scale) as i16; + if proba < 8 { + let rest_to_beat = v_step * u64::from(RTB_TABLE.at(proba as usize)?); + proba += i16::from(scaled.wrapping_sub((proba as u64) << scale) > rest_to_beat); + } + if proba > largest_p { + largest_p = proba; + largest = s; + } + *norm = proba; + still_to_distribute -= i32::from(proba); + } + } + let largest = norm.at_mut(largest)?; + if -still_to_distribute >= i32::from(*largest >> 1) { + // The correction is too large for one symbol to take. + normalize_m2(norm, table_log, count, total, low_prob_count)?; + } else { + *largest = largest.wrapping_add(still_to_distribute as i16); + } + Ok(table_log as usize) +} + +/// `FSE_compress_usingCTable_generic` +#[inline(always)] +fn compress_using_ctable_generic( + dst: &mut [u8], + src: &[u8], + ct: &FseCTable, +) -> usize { + let flush = |bit_c: &mut BitCStream<'_>| { + if FAST { + bit_c.flush_bits_fast(); + } else { + bit_c.flush_bits(); + } + }; + + if src.len() <= 2 { + return 0; + } + let Ok(mut bit_c) = BitCStream::new(dst) else { + return 0; + }; + let encode = |state: &mut FseCState<'_>, bit_c: &mut BitCStream<'_>, symbol: u8| { + state.encode_symbol(bit_c, u32::from(symbol)); + }; + + // From the last symbol to the first, two states taking turns. + let mut state1; + let mut state2; + let mut rest; + if src.len() & 1 != 0 { + let [head @ .., c, b, a] = src else { + return 0; + }; + state1 = FseCState::new2(ct, u32::from(*a)); + state2 = FseCState::new2(ct, u32::from(*b)); + encode(&mut state1, &mut bit_c, *c); + flush(&mut bit_c); + rest = head; + } else { + let [head @ .., b, a] = src else { + return 0; + }; + state2 = FseCState::new2(ct, u32::from(*a)); + state1 = FseCState::new2(ct, u32::from(*b)); + rest = head; + } + + if let ([head @ .., b, a], true) = (rest, rest.len() & 2 != 0) { + encode(&mut state2, &mut bit_c, *a); + encode(&mut state1, &mut bit_c, *b); + flush(&mut bit_c); + rest = head; + } + + let (_, quads) = rest.as_rchunks::<4>(); + for &[d, c, b, a] in quads.iter().rev() { + encode(&mut state2, &mut bit_c, a); + encode(&mut state1, &mut bit_c, b); + encode(&mut state2, &mut bit_c, c); + encode(&mut state1, &mut bit_c, d); + flush(&mut bit_c); + } + + state2.flush(&mut bit_c); + state1.flush(&mut bit_c); + bit_c.close() +} + +/// `FSE_compress_usingCTable`: the table has a code for every symbol in `src`. Returns the size of the output, or 0 +/// if `src` has less than 3 symbols or `dst` is too small. +pub(crate) fn compress_using_ctable(dst: &mut [u8], src: &[u8], ct: &FseCTable) -> usize { + if dst.len() >= block_bound(src.len()) { + compress_using_ctable_generic::(dst, src, ct) + } else { + compress_using_ctable_generic::(dst, src, ct) + } +} diff --git a/src/zstd/compress/hist.rs b/src/zstd/compress/hist.rs new file mode 100644 index 000000000000..472292b90e2c --- /dev/null +++ b/src/zstd/compress/hist.rs @@ -0,0 +1,130 @@ +//! `hist.c`: how many times each byte value occurs. + +use crate::error::{ErrorCode, Result}; +use crate::util::{BUG, SliceExt}; + +/// `HIST_add`: `count` has room for every value in `src`. +pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { + for &byte in src { + *count.at_mut(usize::from(byte))? += 1; + } + Ok(()) +} + +/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. +/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. +pub(crate) fn count_simple( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + let mut max_symbol = *max_symbol_value as usize; + count.sub_mut(..=max_symbol)?.fill(0); + if src.is_empty() { + *max_symbol_value = 0; + return Ok(0); + } + add(count, src)?; + while count.at(max_symbol)? == 0 { + max_symbol = max_symbol.checked_sub(1).ok_or(BUG)?; + } + *max_symbol_value = max_symbol as u32; + Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) +} + +#[inline(always)] +fn increment(counting: &mut [u32; 256], byte: u8) { + if let Some(count) = counting.get_mut(usize::from(byte)) { + *count = count.wrapping_add(1); + } +} + +/// `HIST_count_parallel_wksp`: four tables, so that a run of one value does not wait for its own counter. +/// `max_symbol_value` is at most 255. With `check`, a value above it is an error rather than trusted not to occur. +fn count_parallel( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], + check: bool, +) -> Result { + let count_size = *max_symbol_value as usize + 1; + if src.is_empty() { + count.sub_mut(..count_size)?.fill(0); + *max_symbol_value = 0; + return Ok(0); + } + + let mut counting1 = [0u32; 256]; + let mut counting2 = [0u32; 256]; + let mut counting3 = [0u32; 256]; + let mut counting4 = [0u32; 256]; + + // Each word is read before the bytes of the one before it are counted. + let (words, rest) = src.as_chunks::<4>(); + if let Some((&first, words)) = words.split_first() { + let mut cached = u32::from_le_bytes(first); + let mut count_word = |next: [u8; 4]| { + let c = cached; + cached = u32::from_le_bytes(next); + increment(&mut counting1, c as u8); + increment(&mut counting2, (c >> 8) as u8); + increment(&mut counting3, (c >> 16) as u8); + increment(&mut counting4, (c >> 24) as u8); + }; + let (stripes, words) = words.as_chunks::<4>(); + for &[a, b, c, d] in stripes { + count_word(a); + count_word(b); + count_word(c); + count_word(d); + } + for &word in words { + count_word(word); + } + count_word([0; 4]); + } + for &byte in rest { + increment(&mut counting1, byte); + } + + let mut max = 0u32; + let others = counting2.iter().zip(&counting3).zip(&counting4); + for (sum, ((&c2, &c3), &c4)) in counting1.iter_mut().zip(others) { + *sum = sum.wrapping_add(c2).wrapping_add(c3).wrapping_add(c4); + max = max.max(*sum); + } + + let max_symbol = counting1.iter().rposition(|&c| c != 0).ok_or(BUG)? as u32; + if check && max_symbol > *max_symbol_value { + return Err(ErrorCode::MaxSymbolValueTooSmall); + } + *max_symbol_value = max_symbol; + count.write_at(0, counting1.sub(..count_size)?)?; + Ok(max as usize) +} + +/// `HIST_countFast_wksp`: as [`count_simple`], faster on large inputs. +pub(crate) fn count_fast_wksp( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + if src.len() < 1500 { + return Ok(count_simple(count, max_symbol_value, src)? as usize); + } + count_parallel(count, max_symbol_value, src, false) +} + +/// `HIST_count_wksp`: as [`count_fast_wksp`], but a value above `max_symbol_value` in `src` is +/// [`ErrorCode::MaxSymbolValueTooSmall`]. +pub(crate) fn count_wksp( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + if *max_symbol_value < 255 { + return count_parallel(count, max_symbol_value, src, true); + } + *max_symbol_value = 255; + count_fast_wksp(count, max_symbol_value, src) +} diff --git a/src/zstd/compress/huf.rs b/src/zstd/compress/huf.rs new file mode 100644 index 000000000000..85fe95701c70 --- /dev/null +++ b/src/zstd/compress/huf.rs @@ -0,0 +1,1142 @@ +//! `huf_compress.c`: Huffman coding of literals. + +use super::fse::{self, FseCTable}; +use super::hist; +use super::internal::Strategy; +use crate::common::entropy::huf_read_stats; +use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, highbit32, write_le16}; + +pub(crate) const HUF_BLOCKSIZE_MAX: usize = 128 * 1024; +pub(crate) const HUF_WORKSPACE_SIZE: usize = (8 << 10) + 512; +pub(crate) const HUF_TABLELOG_DEFAULT: u32 = 11; +/// The strategies from this one on look for the best depth of the tree. +pub(crate) const HUF_OPTIMAL_DEPTH_THRESHOLD: Strategy = Strategy::BtUltra; +const HUF_CTABLEBOUND: usize = 129; + +/// `HUF_flags_e` +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct HufFlags { + /// Try every depth of the tree and keep the one that gives the smallest output, instead of a guess. + pub(crate) optimal_depth: bool, + /// Use the previous table again whenever it is possible, without looking for a better one. + pub(crate) prefer_repeat: bool, + /// Sample the input first, and give up at once if it looks like it cannot be compressed. + pub(crate) suspect_uncompressible: bool, +} + +/// `HUF_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum HufRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the +/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in +/// its highest bits. +#[derive(Clone)] +pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); + +impl HufCTable { + pub(crate) const fn new() -> Self { + Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) + } + + /// The elements, by symbol. + #[inline(always)] + fn elements(&self) -> &[u64; HUF_SYMBOLVALUE_MAX as usize + 1] { + let [_, elements @ ..] = &self.0; + elements + } + + #[inline(always)] + fn elements_mut(&mut self) -> &mut [u64; HUF_SYMBOLVALUE_MAX as usize + 1] { + let [_, elements @ ..] = &mut self.0; + elements + } + + /// `HUF_writeCTableHeader` + fn write_header(&mut self, table_log: u32, max_symbol_value: u32) { + let [header, ..] = &mut self.0; + *header = u64::from(table_log as u8) | (u64::from(max_symbol_value as u8) << 8); + } + + /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. + #[inline] + pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { + let [header, elements @ ..] = &self.0; + let max_symbol_value = (header >> 8) as u8; + if symbol_value > u32::from(max_symbol_value) { + return 0; + } + (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 + } +} + +/// `HUF_CTableHeader` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct HufCTableHeader { + pub(crate) table_log: u8, + pub(crate) max_symbol_value: u8, +} + +/// `HUF_readCTableHeader` +#[inline] +pub(crate) fn read_ctable_header(ctable: &HufCTable) -> HufCTableHeader { + let [header, ..] = ctable.0; + HufCTableHeader { + table_log: header as u8, + max_symbol_value: (header >> 8) as u8, + } +} + +/// `HUF_getNbBits` +#[inline(always)] +const fn get_nb_bits(elt: u64) -> u64 { + elt & 0xFF +} + +/// `HUF_setValue`: `elt` has its number of bits, and nothing else. +#[inline(always)] +fn set_value(elt: &mut u64, value: u64) { + let nb_bits = get_nb_bits(*elt) as u32; + if nb_bits > 0 { + *elt |= value.wrapping_shl(u64::BITS.wrapping_sub(nb_bits)); + } +} + +/// Gives each symbol its code. `ct` has the number of bits of each, at most `table_log`, and `nb_per_rank` has how +/// many symbols there are of each number of bits. +fn set_values( + ct: &mut [u64], + nb_per_rank: &[u16; RANKS], + table_log: u32, +) -> Result<()> { + let mut val_per_rank = [0u16; RANKS]; + // The first code of each rank: the longest codes have the lowest values. + let mut min = 0u16; + for n in (1..=table_log as usize).rev() { + val_per_rank.put(n, min)?; + min = min.wrapping_add(nb_per_rank.at(n)?) >> 1; + } + for elt in ct { + let value = val_per_rank.at_mut(get_nb_bits(*elt) as usize)?; + set_value(elt, u64::from(*value)); + *value = value.wrapping_add(1); + } + Ok(()) +} + +const MAX_FSE_TABLELOG_FOR_HUFF_HEADER: u32 = 6; + +/// `HUF_compressWeights`: returns 0 if the weights cannot be compressed, and 1 if they are all the same. +fn compress_weights(dst: &mut [u8], weight_table: &[u8]) -> Result { + let mut max_symbol_value = HUF_TABLELOG_MAX; + let mut count = [0u32; HUF_TABLELOG_MAX as usize + 1]; + let mut norm = [0i16; HUF_TABLELOG_MAX as usize + 1]; + let wt_size = weight_table.len(); + + if wt_size <= 1 { + return Ok(0); + } + + let max_count = hist::count_simple(&mut count, &mut max_symbol_value, weight_table)?; + if max_count as usize == wt_size { + return Ok(1); + } + if max_count == 1 { + return Ok(0); + } + + let table_log = + fse::optimal_table_log(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, wt_size, max_symbol_value); + fse::normalize_count( + &mut norm, + table_log, + &count, + wt_size, + max_symbol_value, + false, + )?; + + let h_size = fse::write_ncount(dst, &norm, max_symbol_value, table_log)?; + + let mut ctable = FseCTable::new(); + ctable.build(&norm, max_symbol_value, table_log)?; + let c_size = fse::compress_using_ctable(dst.sub_mut(h_size..)?, weight_table, &ctable); + if c_size == 0 { + return Ok(0); + } + Ok(h_size + c_size) +} + +/// `HUF_writeCTable_wksp`: writes the header that [`huf_read_stats`] reads, and returns its size. `max_symbol_value` +/// and `huff_log` are those of `ctable`. +pub(crate) fn write_ctable( + dst: &mut [u8], + ctable: &HufCTable, + max_symbol_value: u32, + huff_log: u32, +) -> Result { + let mut bits_to_weight = [0u8; HUF_TABLELOG_MAX as usize + 1]; + let mut huff_weight = [0u8; HUF_SYMBOLVALUE_MAX as usize]; + + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return Err(ErrorCode::MaxSymbolValueTooLarge); + } + // The weight of the last symbol is not stored. + let nb_weights = max_symbol_value as usize; + + for n in 1..=huff_log { + bits_to_weight.put(n as usize, (huff_log + 1 - n) as u8)?; + } + let weights = huff_weight.sub_mut(..nb_weights)?; + for (weight, &elt) in weights.iter_mut().zip(ctable.elements()) { + *weight = bits_to_weight.at(get_nb_bits(elt) as usize)?; + } + + let max_dst_size = dst.len(); + let Some((first, rest)) = dst.split_first_mut() else { + return Err(ErrorCode::DstSizeTooSmall); + }; + let h_size = compress_weights(rest, weights)?; + if h_size > 1 && h_size < nb_weights / 2 { + *first = h_size as u8; + return Ok(h_size + 1); + } + + // Not compressed: 4 bits to a weight. + if max_symbol_value > 256 - 128 { + return Err(ErrorCode::Generic); + } + let raw_size = nb_weights.div_ceil(2); + if raw_size + 1 > max_dst_size { + return Err(ErrorCode::DstSizeTooSmall); + } + *first = 128u32.wrapping_add(max_symbol_value.wrapping_sub(1)) as u8; + // An odd number of weights is completed with a 0, which `huff_weight` still has there. + let (pairs, _) = huff_weight.sub(..raw_size * 2)?.as_chunks::<2>(); + for (out, &[high, low]) in rest.iter_mut().zip(pairs) { + *out = (high << 4).wrapping_add(low); + } + Ok(raw_size + 1) +} + +/// What [`read_ctable`] found besides the table. +pub(crate) struct HufCTableStats { + /// How many bytes of the input the header takes. + pub(crate) size: usize, + pub(crate) max_symbol_value: u32, + /// Some symbol up to `max_symbol_value` has no code. + pub(crate) has_zero_weights: bool, +} + +/// `HUF_readCTable`: [`ErrorCode::MaxSymbolValueTooSmall`] if the table has symbols above `max_symbol_value`. +pub(crate) fn read_ctable( + ctable: &mut HufCTable, + max_symbol_value: u32, + src: &[u8], +) -> Result { + let mut huff_weight = [0u8; HUF_SYMBOLVALUE_MAX as usize + 1]; + let mut rank_val = [0u32; HUF_TABLELOG_MAX as usize + 1]; + + let stats = huf_read_stats(&mut huff_weight, &mut rank_val, src)?; + let table_log = stats.table_log; + let [zero_weights, ..] = rank_val; + + if table_log > HUF_TABLELOG_MAX { + return Err(ErrorCode::TableLogTooLarge); + } + if stats.nb_symbols > max_symbol_value.wrapping_add(1) { + return Err(ErrorCode::MaxSymbolValueTooSmall); + } + let max_symbol_value = stats.nb_symbols - 1; + ctable.write_header(table_log, max_symbol_value); + + let mut nb_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 2]; + let ct = ctable.elements_mut().sub_mut(..stats.nb_symbols as usize)?; + for (elt, &weight) in ct.iter_mut().zip(&huff_weight) { + let nb_bits = if weight != 0 { + (table_log + 1).wrapping_sub(u32::from(weight)) as u8 + } else { + 0 + }; + *elt = u64::from(nb_bits); + *nb_per_rank.at_mut(usize::from(nb_bits))? += 1; + } + set_values(ct, &nb_per_rank, table_log)?; + + Ok(HufCTableStats { + size: stats.size, + max_symbol_value, + has_zero_weights: zero_weights > 0, + }) +} + +/// `nodeElt` +#[derive(Clone, Copy, Default)] +struct NodeElt { + count: u32, + parent: u16, + byte: u8, + nb_bits: u8, +} + +/// The first node that is not a leaf. +const STARTNODE: i32 = HUF_SYMBOLVALUE_MAX as i32 + 1; + +/// `huffNodeTable`: the leaves from 0 on, sorted from the most frequent symbol to the rarest, the other nodes from +/// [`STARTNODE`] on, and a sentinel at -1. +struct HuffNodes([NodeElt; 2 * (HUF_SYMBOLVALUE_MAX as usize + 1)]); + +impl HuffNodes { + #[inline(always)] + fn at(&self, n: i32) -> Result { + self.0.at(n.wrapping_add(1) as usize) + } + + #[inline(always)] + fn at_mut(&mut self, n: i32) -> Result<&mut NodeElt> { + self.0.at_mut(n.wrapping_add(1) as usize) + } + + #[inline(always)] + fn nb_bits(&self, n: i32) -> Result { + Ok(u32::from(self.at(n)?.nb_bits)) + } + + /// The nodes from 0 on. + #[inline(always)] + fn from_0(&mut self) -> &mut [NodeElt] { + let [_, nodes @ ..] = &mut self.0; + nodes + } +} + +/// `HUF_setMaxHeight`: makes no code longer than `target_nb_bits`, which is enough for a code for every symbol. +/// `last_non_null` is the rarest symbol, which has the longest code. Returns the length of the longest code. +fn set_max_height( + huff_node: &mut HuffNodes, + last_non_null: u32, + target_nb_bits: u32, +) -> Result { + let largest_bits = huff_node.nb_bits(last_non_null as i32)?; + if largest_bits <= target_nb_bits { + return Ok(largest_bits); + } + + // The unit of cost is the share of the code space that a code of `largest_bits` takes. + let cost_of = |nb_bits: u32| 1u32.wrapping_shl(largest_bits.wrapping_sub(nb_bits)); + let mut total_cost = 0i32; + let base_cost = cost_of(target_nb_bits); + let mut n = last_non_null as i32; + + // Shorten the codes that are too long. `total_cost` is how much more than the whole code space is taken then. + while huff_node.nb_bits(n)? > target_nb_bits { + let node = huff_node.at_mut(n)?; + let cost = base_cost.wrapping_sub(cost_of(u32::from(node.nb_bits))); + total_cost = total_cost.wrapping_add(cost as i32); + node.nb_bits = target_nb_bits as u8; + n -= 1; + } + // `n` ends at the rarest symbol with a code shorter than `target_nb_bits`. + while huff_node.nb_bits(n)? == target_nb_bits { + n -= 1; + } + + // From here on the unit is the share of a code of `target_nb_bits`. + total_cost = total_cost.wrapping_shr(largest_bits - target_nb_bits); + + // Repay the cost by making other codes longer. + const NO_SYMBOL: u32 = 0xF0F0_F0F0; + // The rarest symbol whose code is that many bits shorter than `target_nb_bits`. + let mut rank_last = [NO_SYMBOL; HUF_TABLELOG_MAX as usize + 2]; + + let mut current_nb_bits = target_nb_bits; + for pos in (0..=n).rev() { + let nb_bits = huff_node.nb_bits(pos)?; + if nb_bits >= current_nb_bits { + continue; + } + current_nb_bits = nb_bits; + rank_last.put((target_nb_bits - current_nb_bits) as usize, pos as u32)?; + } + + while total_cost > 0 { + // One more bit on a code that is `n_bits_to_decrease` shorter repays `1 << (n_bits_to_decrease - 1)`. + let mut n_bits_to_decrease = highbit32(total_cost as u32) as usize + 1; + while n_bits_to_decrease > 1 { + let high_pos = rank_last.at(n_bits_to_decrease)?; + let low_pos = rank_last.at(n_bits_to_decrease - 1)?; + if high_pos != NO_SYMBOL { + if low_pos == NO_SYMBOL { + break; + } + // It is cheaper to lengthen this one than two of the rank below. + let high_total = huff_node.at(high_pos as i32)?.count; + let low_total = huff_node.at(low_pos as i32)?.count.wrapping_mul(2); + if high_total <= low_total { + break; + } + } + n_bits_to_decrease -= 1; + } + while n_bits_to_decrease <= HUF_TABLELOG_MAX as usize + && rank_last.at(n_bits_to_decrease)? == NO_SYMBOL + { + n_bits_to_decrease += 1; + } + let pos = rank_last.at(n_bits_to_decrease)?; + total_cost -= 1 << (n_bits_to_decrease - 1); + let node = huff_node.at_mut(pos as i32)?; + node.nb_bits = node.nb_bits.wrapping_add(1); + + // The symbol is now of the rank below, where it is the rarest only if there was none. + let below = rank_last.at_mut(n_bits_to_decrease - 1)?; + if *below == NO_SYMBOL { + *below = pos; + } + // The next symbol takes its place, if it is of the same rank. + let same_rank = target_nb_bits.wrapping_sub(n_bits_to_decrease as u32); + let next = if pos != 0 && huff_node.nb_bits(pos as i32 - 1)? == same_rank { + pos - 1 + } else { + NO_SYMBOL + }; + rank_last.put(n_bits_to_decrease, next)?; + } + + // Too much may have been repaid: shorten codes of `target_nb_bits`, the most frequent symbols first. + let [_, rank_last_1, ..] = &mut rank_last; + while total_cost < 0 { + if *rank_last_1 == NO_SYMBOL { + while huff_node.nb_bits(n)? == target_nb_bits { + n -= 1; + } + let node = huff_node.at_mut(n + 1)?; + node.nb_bits = node.nb_bits.wrapping_sub(1); + *rank_last_1 = (n + 1) as u32; + total_cost += 1; + continue; + } + let node = huff_node.at_mut(rank_last_1.wrapping_add(1) as i32)?; + node.nb_bits = node.nb_bits.wrapping_sub(1); + *rank_last_1 += 1; + total_cost += 1; + } + + Ok(target_nb_bits) +} + +/// `rankPos` +#[derive(Clone, Copy, Default)] +struct RankPos { + base: u16, + curr: u16, +} + +/// How many buckets [`sort`] has. +const RANK_POSITION_TABLE_SIZE: usize = 192; +const RANK_POSITION_MAX_COUNT_LOG: u32 = 32; +const RANK_POSITION_LOG_BUCKETS_BEGIN: u32 = + (RANK_POSITION_TABLE_SIZE as u32 - 1) - RANK_POSITION_MAX_COUNT_LOG - 1; +/// The counts below this one have a bucket each. The others share one with the counts of the same binary logarithm. +const RANK_POSITION_DISTINCT_COUNT_CUTOFF: u32 = + RANK_POSITION_LOG_BUCKETS_BEGIN + highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN); + +/// `HUF_getIndex`: the bucket of a count. +#[inline(always)] +fn get_index(count: u32) -> usize { + if count < RANK_POSITION_DISTINCT_COUNT_CUTOFF { + count as usize + } else { + (highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN) as usize + } +} + +/// `HUF_swapNodes` +#[inline(always)] +fn swap_nodes(arr: &mut [NodeElt], a: i32, b: i32) -> Result<()> { + let tmp = arr.at(a as usize)?; + arr.put(a as usize, arr.at(b as usize)?)?; + arr.put(b as usize, tmp) +} + +/// `HUF_insertionSort`: by decreasing count, from `low` to `high` included. +#[inline] +fn insertion_sort(huff_node: &mut [NodeElt], low: i32, high: i32) -> Result<()> { + let size = high - low + 1; + let huff_node = huff_node.sub_mut(low as usize..)?; + for i in 1..size { + let key = huff_node.at(i as usize)?; + let mut j = i - 1; + while j >= 0 && huff_node.at(j as usize)?.count < key.count { + huff_node.put((j + 1) as usize, huff_node.at(j as usize)?)?; + j -= 1; + } + huff_node.put((j + 1) as usize, key)?; + } + Ok(()) +} + +/// `HUF_quickSortPartition`: the pivot is the last element. Returns where it ends up. +fn quick_sort_partition(arr: &mut [NodeElt], low: i32, high: i32) -> Result { + let pivot = arr.at(high as usize)?.count; + let mut i = low - 1; + for j in low..high { + if arr.at(j as usize)?.count > pivot { + i += 1; + swap_nodes(arr, i, j)?; + } + } + swap_nodes(arr, i + 1, high)?; + Ok(i + 1) +} + +/// `HUF_simpleQuickSort`: by decreasing count, from `low` to `high` included. It recurses into the smaller part only, +/// which bounds the depth. The order it leaves symbols of the same count in decides their codes. +fn simple_quick_sort(arr: &mut [NodeElt], mut low: i32, mut high: i32) -> Result<()> { + const INSERTION_SORT_THRESHOLD: i32 = 8; + if high - low < INSERTION_SORT_THRESHOLD { + return insertion_sort(arr, low, high); + } + while low < high { + let idx = quick_sort_partition(arr, low, high)?; + if idx - low < high - idx { + simple_quick_sort(arr, low, idx - 1)?; + low = idx + 1; + } else { + simple_quick_sort(arr, idx + 1, high)?; + high = idx - 1; + } + } + Ok(()) +} + +/// `HUF_sort`: fills `count` and `byte` of the first nodes with the symbols, by decreasing count. +fn sort(huff_node: &mut [NodeElt], count: &[u32]) -> Result<()> { + let mut rank_position = [RankPos::default(); RANK_POSITION_TABLE_SIZE]; + + // A bucket starts after the symbols of all the buckets above it. + for &c in count { + rank_position.at_mut(get_index(c))?.base += 1; + } + let mut above = 0u16; + for rank in rank_position.iter_mut().rev() { + rank.base += above; + rank.curr = rank.base; + above = rank.base; + } + + for (n, &c) in count.iter().enumerate() { + let rank = rank_position.at_mut(get_index(c) + 1)?; + let pos = usize::from(rank.curr); + rank.curr += 1; + let node = huff_node.at_mut(pos)?; + node.count = c; + node.byte = n as u8; + } + + let shared = RANK_POSITION_DISTINCT_COUNT_CUTOFF as usize..RANK_POSITION_TABLE_SIZE - 1; + for rank in rank_position.sub(shared)? { + let bucket_size = i32::from(rank.curr) - i32::from(rank.base); + if bucket_size > 1 { + let bucket = huff_node.sub_mut(usize::from(rank.base)..)?; + simple_quick_sort(bucket, 0, bucket_size - 1)?; + } + } + Ok(()) +} + +/// `HUF_buildTree`: `huff_node` is sorted, and some count is not 0. Sets the depth of every leaf, and returns the +/// rarest symbol whose count is not 0. +fn build_tree(huff_node: &mut HuffNodes, max_symbol_value: u32) -> Result { + let mut non_null_rank = max_symbol_value as i32; + while huff_node.at(non_null_rank)?.count == 0 { + non_null_rank -= 1; + } + let mut low_s = non_null_rank; + let mut node_nb = STARTNODE; + let node_root = node_nb + low_s - 1; + let mut low_n = node_nb; + + let mut join = |huff_node: &mut HuffNodes, n1: i32, n2: i32| -> Result<()> { + let count = huff_node + .at(n1)? + .count + .wrapping_add(huff_node.at(n2)?.count); + huff_node.at_mut(node_nb)?.count = count; + huff_node.at_mut(n1)?.parent = node_nb as u16; + huff_node.at_mut(n2)?.parent = node_nb as u16; + node_nb += 1; + Ok(()) + }; + + join(huff_node, low_s, low_s - 1)?; + low_s -= 2; + for n in STARTNODE + 1..=node_root { + huff_node.at_mut(n)?.count = 1 << 30; + } + // The sentinel is never the smaller of the two. + huff_node.at_mut(-1)?.count = 1 << 31; + + // Join the two nodes of the lowest counts, which are the next leaves or the next parents. + for _ in STARTNODE + 1..=node_root { + let mut lowest = |huff_node: &HuffNodes| -> Result { + Ok(if huff_node.at(low_s)?.count < huff_node.at(low_n)?.count { + low_s -= 1; + low_s + 1 + } else { + low_n += 1; + low_n - 1 + }) + }; + let n1 = lowest(huff_node)?; + let n2 = lowest(huff_node)?; + join(huff_node, n1, n2)?; + } + + // The parents come after their children, so that this goes from the root down. + huff_node.at_mut(node_root)?.nb_bits = 0; + for n in (STARTNODE..node_root).rev().chain(0..=non_null_rank) { + let parent = i32::from(huff_node.at(n)?.parent); + huff_node.at_mut(n)?.nb_bits = huff_node.at(parent)?.nb_bits.wrapping_add(1); + } + + Ok(non_null_rank) +} + +/// `HUF_buildCTableFromTree`: no leaf is deeper than `max_nb_bits`, which is at most 12. +fn build_ctable_from_tree( + ctable: &mut HufCTable, + huff_node: &[NodeElt], + non_null_rank: i32, + max_symbol_value: u32, + max_nb_bits: u32, +) -> Result<()> { + let alphabet_size = max_symbol_value as usize + 1; + let ct = ctable.elements_mut().sub_mut(..alphabet_size)?; + let mut nb_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 1]; + for node in huff_node.sub(..=non_null_rank as usize)? { + *nb_per_rank.at_mut(usize::from(node.nb_bits))? += 1; + } + for node in huff_node.sub(..alphabet_size)? { + ct.put(usize::from(node.byte), u64::from(node.nb_bits))?; + } + set_values(ct, &nb_per_rank, max_nb_bits)?; + ctable.write_header(max_nb_bits, max_symbol_value); + Ok(()) +} + +/// `HUF_buildCTable_wksp`: some count up to `max_symbol_value` is not 0, and `max_nb_bits` is enough for a code for +/// each that is not, or 0 for the default. Returns the length of the longest code. +pub(crate) fn build_ctable( + ctable: &mut HufCTable, + count: &[u32], + max_symbol_value: u32, + mut max_nb_bits: u32, +) -> Result { + if max_nb_bits == 0 { + max_nb_bits = HUF_TABLELOG_DEFAULT; + } + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return Err(ErrorCode::MaxSymbolValueTooLarge); + } + let mut huff_node = HuffNodes([NodeElt::default(); _]); + + sort(huff_node.from_0(), count.sub(..=max_symbol_value as usize)?)?; + let non_null_rank = build_tree(&mut huff_node, max_symbol_value)?; + max_nb_bits = set_max_height(&mut huff_node, non_null_rank as u32, max_nb_bits)?; + if max_nb_bits > HUF_TABLELOG_MAX { + return Err(ErrorCode::Generic); + } + build_ctable_from_tree( + ctable, + huff_node.from_0(), + non_null_rank, + max_symbol_value, + max_nb_bits, + )?; + Ok(max_nb_bits as usize) +} + +/// `HUF_estimateCompressedSize`: how many bytes the symbols counted in `count` take, without the last partial one. +pub(crate) fn estimate_compressed_size( + ctable: &HufCTable, + count: &[u32], + max_symbol_value: u32, +) -> usize { + let nb_bits: usize = (ctable.elements().iter().zip(count)) + .take(max_symbol_value as usize + 1) + .map(|(&elt, &count)| get_nb_bits(elt) as usize * count as usize) + .sum(); + nb_bits >> 3 +} + +/// `HUF_validateCTable`: whether the table has a code for every symbol whose count is not 0. +pub(crate) fn validate_ctable(ctable: &HufCTable, count: &[u32], max_symbol_value: u32) -> bool { + if u32::from(read_ctable_header(ctable).max_symbol_value) < max_symbol_value { + return false; + } + (ctable.elements().iter().zip(count)) + .take(max_symbol_value as usize + 1) + .all(|(&elt, &count)| count == 0 || get_nb_bits(elt) != 0) +} + +/// `HUF_compressBound` +pub(crate) const fn compress_bound(size: usize) -> usize { + HUF_CTABLEBOUND + size + (size >> 8) + 8 +} + +/// One `bitContainer` of `HUF_CStream_t` and its `bitPos`. +#[derive(Clone, Copy, Default)] +struct BitContainer { + /// The bits are at the top. The lowest bits can be dirty. + bits: u64, + /// The number of bits is in the lowest byte. The rest is dirty. + pos: u64, +} + +impl BitContainer { + /// `HUF_addBits`: there is room for the code of `elt`. With `FAST`, the lowest byte of `elt` is not cleared + /// first, which takes that there is room for 4 bits more. + #[inline(always)] + fn add_bits(&mut self, elt: u64) { + self.bits >>= get_nb_bits(elt) & 63; + self.bits |= if FAST { elt } else { elt & !0xFF }; + self.pos = self.pos.wrapping_add(elt); + } + + #[inline(always)] + const fn nb_bits(self) -> u64 { + self.pos & 0xFF + } +} + +/// `HUF_CStream_t`. Its second container is a local variable of the loop that uses it. +struct HufCStream<'a> { + container: BitContainer, + dst: &'a mut [u8], + pos: usize, + /// The last position a whole container can be written to. + end: usize, +} + +impl<'a> HufCStream<'a> { + /// `HUF_initCStream` + #[inline] + fn new(dst: &'a mut [u8]) -> Result { + if dst.len() <= size_of::() { + return Err(ErrorCode::DstSizeTooSmall); + } + let end = dst.len() - size_of::(); + Ok(Self { + container: BitContainer::default(), + dst, + pos: 0, + end, + }) + } + + /// `HUF_mergeIndex1`: there is room for the bits of `second`, which come after those the stream has. + #[inline(always)] + fn merge(&mut self, second: BitContainer) { + self.container.bits >>= second.nb_bits() & 63; + self.container.bits |= second.bits; + self.container.pos = self.container.pos.wrapping_add(second.pos); + } + + /// `HUF_flushBits`: writes the whole bytes. With `FAST`, the caller knows that the output is large enough. + /// Without, the stream stops advancing at the end of the output, which [`Self::close`] reports. + #[inline(always)] + fn flush_bits(&mut self) { + let nb_bits = self.container.nb_bits(); + let nb_bytes = (nb_bits >> 3) as usize; + let bits = self.container.bits >> (u64::from(u64::BITS).wrapping_sub(nb_bits) & 63); + self.container.pos &= 7; + if let Ok(out) = self.dst.array_at_mut(self.pos) { + *out = bits.to_le_bytes(); + } + self.pos += nb_bytes; + if !FAST && self.pos > self.end { + self.pos = self.end; + } + } + + /// `HUF_closeCStream`: the size of the stream, or 0 if it did not fit. + #[inline] + fn close(mut self) -> usize { + /// `HUF_endMark`: one bit, set. + const END_MARK: u64 = 1 | (1 << 63); + self.container.add_bits::(END_MARK); + self.flush_bits::(); + if self.pos >= self.end { + return 0; + } + self.pos + usize::from(self.container.nb_bits() > 0) + } +} + +type Elements = [u64; HUF_SYMBOLVALUE_MAX as usize + 1]; + +/// `HUF_encodeSymbol` +#[inline(always)] +fn encode_symbol(container: &mut BitContainer, symbol: u8, ct: &Elements) { + container.add_bits::(ct.get(usize::from(symbol)).copied().unwrap_or(0)); +} + +/// Encodes `symbols` from the last to the first. +#[inline(always)] +fn encode_symbols( + container: &mut BitContainer, + symbols: &[u8; UNROLL], + ct: &Elements, +) { + if let Some((&last, symbols)) = symbols.split_first() { + for &symbol in symbols.iter().rev() { + encode_symbol::(container, symbol, ct); + } + encode_symbol::(container, last, ct); + } +} + +/// `HUF_compress1X_usingCTable_internal_body_loop`: a container has room for `UNROLL` symbols on top of the 7 bits +/// that a flush can leave, and, with `LAST_FAST`, for 4 bits more. +#[inline(always)] +fn compress_1x_body_loop( + bit_c: &mut HufCStream<'_>, + src: &[u8], + ct: &Elements, +) { + let (groups, rem) = src.as_chunks::(); + if !rem.is_empty() { + for &symbol in rem.iter().rev() { + encode_symbol::(&mut bit_c.container, symbol, ct); + } + bit_c.flush_bits::(); + } + let (pairs, odd) = groups.as_chunks::<2>(); + if let [group] = odd { + encode_symbols::(&mut bit_c.container, group, ct); + bit_c.flush_bits::(); + } + for [second, first] in pairs.iter().rev() { + encode_symbols::(&mut bit_c.container, first, ct); + bit_c.flush_bits::(); + // The second group does not depend on the flush, since it has a container of its own. + let mut container = BitContainer::default(); + encode_symbols::(&mut container, second, ct); + bit_c.merge(container); + bit_c.flush_bits::(); + } +} + +/// `HUF_tightCompressBound`: no output is larger, given the longest code. +const fn tight_compress_bound(src_size: usize, table_log: usize) -> usize { + ((src_size * table_log) >> 3) + 8 +} + +/// `HUF_compress1X_usingCTable`: the table has a code for every symbol in `src`. Returns the size of the output, or 0 +/// if `dst` is too small. +pub(crate) fn compress_1x_using_ctable( + dst: &mut [u8], + src: &[u8], + ctable: &HufCTable, +) -> Result { + let table_log = read_ctable_header(ctable).table_log; + let ct = ctable.elements(); + let dst_size = dst.len(); + + let Ok(mut bit_c) = HufCStream::new(dst) else { + return Ok(0); + }; + let bit_c_ref = &mut bit_c; + + if dst_size < tight_compress_bound(src.len(), usize::from(table_log)) || table_log > 11 { + compress_1x_body_loop::<4, false, false>(bit_c_ref, src, ct); + } else { + match table_log { + 11 => compress_1x_body_loop::<5, true, false>(bit_c_ref, src, ct), + 10 => compress_1x_body_loop::<5, true, true>(bit_c_ref, src, ct), + 9 => compress_1x_body_loop::<6, true, false>(bit_c_ref, src, ct), + 8 => compress_1x_body_loop::<7, true, false>(bit_c_ref, src, ct), + 7 => compress_1x_body_loop::<8, true, false>(bit_c_ref, src, ct), + _ => compress_1x_body_loop::<9, true, true>(bit_c_ref, src, ct), + } + } + Ok(bit_c.close()) +} + +/// `HUF_compress4X_usingCTable`: four streams, after the sizes of the first three. The table has a code for every +/// symbol in `src`. Returns the size of the output, or 0 if `src` is too small for it, `dst` is, or a stream is too +/// large. +pub(crate) fn compress_4x_using_ctable( + dst: &mut [u8], + src: &[u8], + ctable: &HufCTable, +) -> Result { + let segment_size = src.len().div_ceil(4); + if dst.len() < 6 + 1 + 1 + 1 + 8 { + return Ok(0); + } + if src.len() < 12 { + return Ok(0); + } + let mut op = 6usize; + for (stream, segment) in src.chunks(segment_size).enumerate() { + let c_size = compress_1x_using_ctable(dst.sub_mut(op..)?, segment, ctable)?; + if c_size == 0 || c_size > 65535 { + return Ok(0); + } + if stream < 3 { + write_le16(dst, stream * 2, c_size as u16)?; + } + op += c_size; + } + Ok(op) +} + +/// `HUF_nbStreams_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum NbStreams { + Single, + Four, +} + +/// `HUF_compressCTable_internal`: writes from `op` on. Returns how much of `dst` is used, or 0 if that saves nothing. +fn compress_ctable_internal( + dst: &mut [u8], + op: usize, + src: &[u8], + nb_streams: NbStreams, + ctable: &HufCTable, +) -> Result { + let out = dst.sub_mut(op..)?; + let c_size = match nb_streams { + NbStreams::Single => compress_1x_using_ctable(out, src, ctable)?, + NbStreams::Four => compress_4x_using_ctable(out, src, ctable)?, + }; + if c_size == 0 { + return Ok(0); + } + if op + c_size >= src.len().wrapping_sub(1) { + return Ok(0); + } + Ok(op + c_size) +} + +const SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE: usize = 4096; +const SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO: usize = 10; + +/// `HUF_cardinality`: how many counts up to `max_symbol_value` are not 0. +pub(crate) fn cardinality(count: &[u32], max_symbol_value: u32) -> u32 { + let counts = count.iter().take(max_symbol_value as usize + 1); + counts.filter(|&&count| count != 0).count() as u32 +} + +/// `HUF_minTableLog`: the shortest that the longest code can be. `symbol_cardinality` is not 0. +pub(crate) fn min_table_log(symbol_cardinality: u32) -> u32 { + highbit32(symbol_cardinality) + 1 +} + +/// `HUF_optimalTableLog`: the limit to put on the length of the codes. `src_size` is more than 1. With +/// `flags.optimal_depth`, `table` is overwritten. +pub(crate) fn optimal_table_log( + max_table_log: u32, + src_size: usize, + max_symbol_value: u32, + table: &mut HufCTable, + count: &[u32], + flags: HufFlags, +) -> u32 { + if !flags.optimal_depth { + return fse::optimal_table_log_internal(max_table_log, src_size, max_symbol_value, 1); + } + + // Large enough that no step of `write_ctable` is short of room. + let mut dst = [0u8; 512]; + let symbol_cardinality = cardinality(count, max_symbol_value); + let min_table_log = min_table_log(symbol_cardinality); + let mut opt_size = usize::MAX - 1; + let mut opt_log = max_table_log; + + // Search until the size increases. + for opt_log_guess in min_table_log..=max_table_log { + let Ok(max_bits) = build_ctable(table, count, max_symbol_value, opt_log_guess) else { + continue; + }; + if max_bits < opt_log_guess as usize && opt_log_guess > min_table_log { + break; + } + let Ok(h_size) = write_ctable(&mut dst, table, max_symbol_value, max_bits as u32) else { + continue; + }; + let new_size = estimate_compressed_size(table, count, max_symbol_value) + h_size; + if new_size > opt_size + 1 { + break; + } + if new_size < opt_size { + opt_size = new_size; + opt_log = opt_log_guess; + } + } + opt_log +} + +/// `HUF_compress_internal` +fn compress_internal( + dst: &mut [u8], + src: &[u8], + mut max_symbol_value: u32, + mut huff_log: u32, + nb_streams: NbStreams, + old_huf_table: &mut HufCTable, + repeat: &mut HufRepeat, + flags: HufFlags, +) -> Result { + let src_size = src.len(); + let Some(&first_byte) = src.first() else { + return Ok(0); + }; + if dst.is_empty() { + return Ok(0); + } + if src_size > HUF_BLOCKSIZE_MAX { + return Err(ErrorCode::SrcSizeWrong); + } + if huff_log > HUF_TABLELOG_MAX { + return Err(ErrorCode::TableLogTooLarge); + } + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return Err(ErrorCode::MaxSymbolValueTooLarge); + } + if max_symbol_value == 0 { + max_symbol_value = HUF_SYMBOLVALUE_MAX; + } + if huff_log == 0 { + huff_log = HUF_TABLELOG_DEFAULT; + } + + if flags.prefer_repeat && *repeat == HufRepeat::Valid { + return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); + } + + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + + // If the input may well be incompressible, look at its start and its end before all of it. + if flags.suspect_uncompressible + && src_size >= SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO + { + let mut largest_total = 0usize; + for sample in [ + src.sub(..SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE)?, + src.sub(src_size - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE..)?, + ] { + let mut max_symbol_value_sample = max_symbol_value; + largest_total += + hist::count_simple(&mut count, &mut max_symbol_value_sample, sample)? as usize; + } + if largest_total <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7) + 4 { + return Ok(0); + } + } + + let largest = hist::count_wksp(&mut count, &mut max_symbol_value, src)?; + if largest == src_size { + // A single symbol: run-length encoding. + dst.put(0, first_byte)?; + return Ok(1); + } + if largest <= (src_size >> 7) + 4 { + // Probably not compressible enough. + return Ok(0); + } + + if *repeat == HufRepeat::Check && !validate_ctable(old_huf_table, &count, max_symbol_value) { + *repeat = HufRepeat::None; + } + if flags.prefer_repeat && *repeat != HufRepeat::None { + return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); + } + + let mut ctable = HufCTable::new(); + huff_log = optimal_table_log( + huff_log, + src_size, + max_symbol_value, + &mut ctable, + &count, + flags, + ); + huff_log = build_ctable(&mut ctable, &count, max_symbol_value, huff_log)? as u32; + + let h_size = write_ctable(dst, &ctable, max_symbol_value, huff_log)?; + // Is the previous table the better choice, since it takes no header? + if *repeat != HufRepeat::None { + let old_size = estimate_compressed_size(old_huf_table, &count, max_symbol_value); + let new_size = estimate_compressed_size(&ctable, &count, max_symbol_value); + if old_size <= h_size + new_size || h_size + 12 >= src_size { + return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); + } + } + + if h_size + 12 >= src_size { + return Ok(0); + } + *repeat = HufRepeat::None; + old_huf_table.clone_from(&ctable); + compress_ctable_internal(dst, h_size, src, nb_streams, &ctable) +} + +/// `HUF_compress1X_repeat`: one stream, after the header of its table, or without one if `huf_table` is used again, +/// which `repeat` says is allowed. If it is not used again, `repeat` becomes [`HufRepeat::None`] and `huf_table` the +/// new table. Returns the size of the output, 0 if `src` is not compressible, or 1 if it is one byte repeated, which +/// is then the output. +pub(crate) fn compress_1x_repeat( + dst: &mut [u8], + src: &[u8], + max_symbol_value: u32, + huff_log: u32, + huf_table: &mut HufCTable, + repeat: &mut HufRepeat, + flags: HufFlags, +) -> Result { + compress_internal( + dst, + src, + max_symbol_value, + huff_log, + NbStreams::Single, + huf_table, + repeat, + flags, + ) +} + +/// `HUF_compress4X_repeat`: as [`compress_1x_repeat`], with four streams. +pub(crate) fn compress_4x_repeat( + dst: &mut [u8], + src: &[u8], + max_symbol_value: u32, + huff_log: u32, + huf_table: &mut HufCTable, + repeat: &mut HufRepeat, + flags: HufFlags, +) -> Result { + compress_internal( + dst, + src, + max_symbol_value, + huff_log, + NbStreams::Four, + huf_table, + repeat, + flags, + ) +} diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs new file mode 100644 index 000000000000..74db66398416 --- /dev/null +++ b/src/zstd/compress/internal.rs @@ -0,0 +1,1267 @@ +//! `zstd_compress_internal.h`: what the parts of the compressor share. + +use super::fse::{FseCTable, FseRepeat}; +use super::huf::{HufCTable, HufRepeat}; +use crate::common::{ + BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, + ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; + +pub(crate) const K_SEARCH_STRENGTH: u32 = 8; +pub(crate) const HASH_READ_SIZE: usize = 8; +/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. +pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; + +pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; +pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; + +pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; +pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; + +/// `ZSTD_strategy` +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub(crate) enum Strategy { + Fast = 1, + DFast = 2, + Greedy = 3, + Lazy = 4, + Lazy2 = 5, + BtLazy2 = 6, + BtOpt = 7, + BtUltra = 8, + BtUltra2 = 9, +} + +impl Strategy { + pub(crate) const fn from_int(value: i32) -> Option { + Some(match value { + 1 => Self::Fast, + 2 => Self::DFast, + 3 => Self::Greedy, + 4 => Self::Lazy, + 5 => Self::Lazy2, + 6 => Self::BtLazy2, + 7 => Self::BtOpt, + 8 => Self::BtUltra, + 9 => Self::BtUltra2, + _ => return None, + }) + } +} + +/// `ZSTD_ParamSwitch_e` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum ParamSwitch { + Auto = 0, + Enable = 1, + Disable = 2, +} + +/// `ZSTD_compressionParameters` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct CompressionParameters { + pub(crate) window_log: u32, + pub(crate) chain_log: u32, + pub(crate) hash_log: u32, + pub(crate) search_log: u32, + pub(crate) min_match: u32, + pub(crate) target_length: u32, + /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. + pub(crate) strategy: Option, +} + +impl CompressionParameters { + /// The strategy of parameters that are ready to be applied. + #[inline(always)] + pub(crate) fn strat(&self) -> Strategy { + self.strategy.unwrap_or(Strategy::Fast) + } +} + +/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum DictMode { + NoDict, + ExtDict, + DictMatchState, +} + +/// `ZSTD_dictTableLoadMethod_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DictTableLoadMethod { + Fast, + Full, +} + +/// `ZSTD_tableFillPurpose_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TableFillPurpose { + ForCCtx, + ForCDict, +} + +// ── entropy tables ──────────────────────────────────────────────────────── + +/// `ZSTD_hufCTables_t` +#[derive(Clone)] +pub(crate) struct HufCTables { + pub(crate) c_table: HufCTable, + pub(crate) repeat_mode: HufRepeat, +} + +/// `ZSTD_fseCTables_t` +#[derive(Clone)] +pub(crate) struct FseCTables { + pub(crate) offcode_c_table: FseCTable, + pub(crate) matchlength_c_table: FseCTable, + pub(crate) litlength_c_table: FseCTable, + pub(crate) offcode_repeat_mode: FseRepeat, + pub(crate) matchlength_repeat_mode: FseRepeat, + pub(crate) litlength_repeat_mode: FseRepeat, +} + +/// `ZSTD_entropyCTables_t` +#[derive(Clone)] +pub(crate) struct EntropyCTables { + pub(crate) huf: HufCTables, + pub(crate) fse: FseCTables, +} + +/// `ZSTD_compressedBlockState_t` +#[derive(Clone)] +pub(crate) struct CompressedBlockState { + pub(crate) entropy: EntropyCTables, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +impl CompressedBlockState { + /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. + pub(crate) fn new() -> Self { + Self { + entropy: EntropyCTables { + huf: HufCTables { + c_table: HufCTable::new(), + repeat_mode: HufRepeat::None, + }, + fse: FseCTables { + offcode_c_table: FseCTable::new(), + matchlength_c_table: FseCTable::new(), + litlength_c_table: FseCTable::new(), + offcode_repeat_mode: FseRepeat::None, + matchlength_repeat_mode: FseRepeat::None, + litlength_repeat_mode: FseRepeat::None, + }, + }, + rep: REP_START_VALUE, + } + } + + /// `ZSTD_reset_compressedBlockState` + pub(crate) fn reset(&mut self) { + self.rep = REP_START_VALUE; + self.entropy.huf.repeat_mode = HufRepeat::None; + self.entropy.fse.offcode_repeat_mode = FseRepeat::None; + self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; + self.entropy.fse.litlength_repeat_mode = FseRepeat::None; + } +} + +/// `ZSTD_hufCTablesMetadata_t` +pub(crate) struct HufCTablesMetadata { + pub(crate) h_type: SymbolEncodingType, + pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], + pub(crate) huf_des_size: usize, +} + +/// `ZSTD_fseCTablesMetadata_t` +pub(crate) struct FseCTablesMetadata { + pub(crate) ll_type: SymbolEncodingType, + pub(crate) of_type: SymbolEncodingType, + pub(crate) ml_type: SymbolEncodingType, + pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], + pub(crate) fse_tables_size: usize, + /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. + pub(crate) last_count_size: usize, +} + +/// `ZSTD_entropyCTablesMetadata_t` +pub(crate) struct EntropyCTablesMetadata { + pub(crate) huf_metadata: HufCTablesMetadata, + pub(crate) fse_metadata: FseCTablesMetadata, +} + +impl EntropyCTablesMetadata { + pub(crate) const fn new() -> Self { + Self { + huf_metadata: HufCTablesMetadata { + h_type: SymbolEncodingType::Basic, + huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], + huf_des_size: 0, + }, + fse_metadata: FseCTablesMetadata { + ll_type: SymbolEncodingType::Basic, + of_type: SymbolEncodingType::Basic, + ml_type: SymbolEncodingType::Basic, + fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], + fse_tables_size: 0, + last_count_size: 0, + }, + } + } +} + +// ── sequences ───────────────────────────────────────────────────────────── + +/// `SeqDef` +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct SeqDef { + /// A repeat code (1 to 3), or an offset plus 3. + pub(crate) off_base: u32, + pub(crate) lit_length: u16, + /// The match length minus [`MINMATCH`]. + pub(crate) ml_base: u16, +} + +/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum LongLengthType { + None, + LiteralLength, + MatchLength, +} + +/// `SeqStore_t`: the sequences and the literals of a block. +pub(crate) struct SeqStore { + /// Never grows beyond the capacity it is given by [`Self::reserve`]. + pub(crate) sequences: Vec, + /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. + pub(crate) lit_buffer: Vec, + /// How much of `lit_buffer` is literals. + pub(crate) lit_len: usize, + pub(crate) ll_code: Vec, + pub(crate) ml_code: Vec, + pub(crate) of_code: Vec, + pub(crate) max_nb_seq: usize, + pub(crate) max_nb_lit: usize, + pub(crate) long_length_type: LongLengthType, + /// Which sequence has the long length. + pub(crate) long_length_pos: u32, +} + +/// `ZSTD_SequenceLength` +#[derive(Clone, Copy)] +pub(crate) struct SequenceLength { + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +/// Some consecutive sequences of a [`SeqStore`] and their literals, which the C code has a `SeqStore_t` that points +/// into the other one for. +#[derive(Clone, Copy)] +pub(crate) struct SeqChunk { + pub(crate) seq_start: usize, + pub(crate) seq_end: usize, + pub(crate) lit_start: usize, + pub(crate) lit_end: usize, + pub(crate) long_length_type: LongLengthType, + /// Counted from `seq_start`. + pub(crate) long_length_pos: u32, +} + +impl SeqChunk { + #[inline] + pub(crate) const fn nb_seq(&self) -> usize { + self.seq_end - self.seq_start + } + + #[inline] + pub(crate) const fn lit_size(&self) -> usize { + self.lit_end - self.lit_start + } +} + +impl SeqStore { + /// All of it. + pub(crate) const fn whole(&self) -> SeqChunk { + SeqChunk { + seq_start: 0, + seq_end: self.sequences.len(), + lit_start: 0, + lit_end: self.lit_len, + long_length_type: self.long_length_type, + long_length_pos: self.long_length_pos, + } + } + + /// `ZSTD_getSequenceLength` + #[inline] + pub(crate) fn get_sequence_length(&self, index: usize) -> Result { + let seq = self.sequences.at(index)?; + let mut seq_len = SequenceLength { + lit_length: u32::from(seq.lit_length), + match_length: u32::from(seq.ml_base) + MINMATCH, + }; + if self.long_length_pos as usize == index { + match self.long_length_type { + LongLengthType::None => {} + LongLengthType::LiteralLength => seq_len.lit_length += 0x10000, + LongLengthType::MatchLength => seq_len.match_length += 0x10000, + } + } + Ok(seq_len) + } + + pub(crate) const fn new() -> Self { + Self { + sequences: Vec::new(), + lit_buffer: Vec::new(), + lit_len: 0, + ll_code: Vec::new(), + ml_code: Vec::new(), + of_code: Vec::new(), + max_nb_seq: 0, + max_nb_lit: 0, + long_length_type: LongLengthType::None, + long_length_pos: 0, + } + } + + /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. + pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { + fn grow(vec: &mut Vec, len: usize) -> Result<()> { + vec.clear(); + vec.try_reserve_exact(len) + .map_err(|_| ErrorCode::MemoryAllocation)?; + vec.resize(len, T::default()); + Ok(()) + } + grow(&mut self.sequences, max_nb_seq)?; + grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; + grow(&mut self.ll_code, max_nb_seq)?; + grow(&mut self.ml_code, max_nb_seq)?; + grow(&mut self.of_code, max_nb_seq)?; + self.max_nb_seq = max_nb_seq; + self.max_nb_lit = max_nb_lit; + self.reset(); + Ok(()) + } + + /// `ZSTD_resetSeqStore` + #[inline] + pub(crate) fn reset(&mut self) { + self.lit_len = 0; + self.sequences.clear(); + self.long_length_type = LongLengthType::None; + } + + #[inline] + pub(crate) fn literals(&self) -> &[u8] { + self.lit_buffer.get(..self.lit_len).unwrap_or_default() + } + + /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. + #[inline(always)] + pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { + debug_assert!(self.sequences.len() < self.max_nb_seq); + if lit_length > 0xFFFF { + self.long_length_type = LongLengthType::LiteralLength; + self.long_length_pos = self.sequences.len() as u32; + } + let ml_base = match_length.wrapping_sub(MINMATCH as usize); + if ml_base > 0xFFFF { + self.long_length_type = LongLengthType::MatchLength; + self.long_length_pos = self.sequences.len() as u32; + } + self.sequences.push(SeqDef { + off_base, + lit_length: lit_length as u16, + ml_base: ml_base as u16, + }); + } + + /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. + #[inline(always)] + pub(crate) fn store_seq( + &mut self, + lit_length: usize, + buf: &[u8], + literals: usize, + off_base: u32, + match_length: usize, + ) -> Result<()> { + match ( + buf.array_at::<16>(literals), + self.lit_buffer.array_at_mut::<16>(self.lit_len), + ) { + (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, + _ => self + .lit_buffer + .write_at(self.lit_len, buf.span(literals, lit_length)?)?, + } + self.lit_len += lit_length; + self.store_seq_only(lit_length, off_base, match_length); + Ok(()) + } + + /// Appends the literals that follow the last sequence of a block. + #[inline] + pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { + self.lit_buffer.write_at(self.lit_len, literals)?; + self.lit_len += literals.len(); + Ok(()) + } +} + +pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; +pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; +pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; + +/// `OFFSET_TO_OFFBASE`: `offset` is not 0. +#[inline(always)] +pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { + offset.wrapping_add(ZSTD_REP_NUM as u32) +} + +/// `OFFBASE_IS_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { + off_base > ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_IS_REPCODE` +#[inline(always)] +pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { + 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_TO_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { + off_base.wrapping_sub(ZSTD_REP_NUM as u32) +} + +/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. +#[inline(always)] +pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { + let [rep0, rep1, rep2] = *rep; + if offbase_is_offset(off_base) { + *rep = [offbase_to_offset(off_base), rep0, rep1]; + } else { + match off_base - 1 + u32::from(ll0) { + 0 => {} + 1 => *rep = [rep1, rep0, rep2], + 2 => *rep = [rep2, rep0, rep1], + _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], + } + } +} + +/// `ZSTD_newRep` +#[inline(always)] +pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { + let mut new_reps = *rep; + update_rep(&mut new_reps, off_base, ll0); + new_reps +} + +/// `ZSTD_LLcode` +#[inline(always)] +pub(crate) fn ll_code(lit_length: u32) -> u32 { + #[rustfmt::skip] + const LL_CODE: [u8; 64] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + ]; + const LL_DELTA_CODE: u32 = 19; + match LL_CODE.get(lit_length as usize) { + Some(&code) => u32::from(code), + None => highbit32(lit_length) + LL_DELTA_CODE, + } +} + +/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. +#[inline(always)] +pub(crate) fn ml_code(ml_base: u32) -> u32 { + #[rustfmt::skip] + const ML_CODE: [u8; 128] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ]; + const ML_DELTA_CODE: u32 = 36; + match ML_CODE.get(ml_base as usize) { + Some(&code) => u32::from(code), + None => highbit32(ml_base) + ML_DELTA_CODE, + } +} + +/// `ZSTD_noCompressBlock` +pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { + let c_block_header24 = + u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); + if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header24)?; + dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; + Ok(ZSTD_BLOCKHEADERSIZE + src.len()) +} + +/// `ZSTD_rleCompressBlock` +pub(crate) fn rle_compress_block( + dst: &mut [u8], + byte: u8, + src_size: usize, + last_block: bool, +) -> Result { + let c_block_header = + u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); + if dst.len() < 4 { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header)?; + dst.put(3, byte)?; + Ok(4) +} + +/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. +#[inline] +pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { + let minlog = if strat >= Strategy::BtUltra { + strat as u32 - 1 + } else { + 6 + }; + (src_size >> minlog) + 2 +} + +// ── matching ────────────────────────────────────────────────────────────── + +/// How many bytes `a` and `b` have in common at their start. +#[inline(always)] +pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { + let (a_words, _) = a.as_chunks::<8>(); + let (b_words, _) = b.as_chunks::<8>(); + let mut len = 0usize; + for (a_word, b_word) in a_words.iter().zip(b_words) { + let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); + if diff != 0 { + return len + (diff.trailing_zeros() >> 3) as usize; + } + len += 8; + } + let a_tail = a.get(len..).unwrap_or_default(); + let b_tail = b.get(len..).unwrap_or_default(); + len + a_tail + .iter() + .zip(b_tail) + .take_while(|(a_byte, b_byte)| a_byte == b_byte) + .count() +} + +/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. +#[inline(always)] +pub(crate) fn count( + buf: &[u8], + p_in: usize, + match_buf: &[u8], + p_match: usize, + p_in_limit: usize, +) -> Result { + Ok(common_prefix( + buf.sub(p_in..p_in_limit)?, + match_buf.sub(p_match..)?, + )) +} + +/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at +/// `i_start` in `buf`. +#[inline(always)] +pub(crate) fn count_2segments( + buf: &[u8], + ip: usize, + match_buf: &[u8], + match_pos: usize, + i_end: usize, + m_end: usize, + i_start: usize, +) -> Result { + let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); + if match_pos + match_length != m_end { + return Ok(match_length); + } + Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) +} + +const PRIME_3_BYTES: u32 = 506_832_829; +const PRIME_4_BYTES: u32 = 2_654_435_761; +const PRIME_5_BYTES: u64 = 889_523_592_379; +const PRIME_6_BYTES: u64 = 227_718_039_650_203; +const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; +pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; + +/// `ZSTD_hash3` +#[inline(always)] +const fn hash3(u: u32, h: u32, s: u32) -> u32 { + (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash4(u: u32, h: u32, s: u32) -> u32 { + ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash5(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash6(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash7(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +/// `ZSTD_hash8` +#[inline(always)] +pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { + (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} + +/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. +#[inline(always)] +pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { + Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) +} + +/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, +/// else 8. +#[inline(always)] +pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { + hash_ptr_salted(buf, pos, h_bits, mls, 0) +} + +/// `ZSTD_hashPtrSalted` +#[inline(always)] +pub(crate) fn hash_ptr_salted( + buf: &[u8], + pos: usize, + h_bits: u32, + mls: u32, + hash_salt: u64, +) -> Result { + Ok(match mls { + 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), + 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), + 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), + 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), + _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, + }) +} + +// ── the window ──────────────────────────────────────────────────────────── + +/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are +/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the +/// others, and an address is turned into a position in the buffer only to read. +pub(crate) type Addr = usize; + +/// The buffers that a window can have a segment in. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum BufId { + /// Where an empty window points. + None = 0, + /// The context's own buffer of input. + InBuff = 1, + /// What the caller gave to the call in progress. + User = 2, + /// The content of the dictionary, or the prefix. + Dict = 3, +} + +impl BufId { + /// The address of the first byte of the buffer. + #[inline(always)] + pub(crate) const fn start(self) -> Addr { + (self as usize + 1) << 44 + } +} + +/// The buffers themselves, for as long as a call lasts. +#[derive(Clone, Copy)] +pub(crate) struct Buffers<'a> { + pub(crate) in_buff: &'a [u8], + pub(crate) user: &'a [u8], + pub(crate) dict: &'a [u8], +} + +impl<'a> Buffers<'a> { + #[inline] + pub(crate) fn get(&self, id: BufId) -> &'a [u8] { + match id { + BufId::None => &[], + BufId::InBuff => self.in_buff, + BufId::User => self.user, + BufId::Dict => self.dict, + } + } +} + +/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. +pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; + +/// The index past which indices are brought down. +pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; +/// The most input that can be indexed at once. +pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; + +/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the +/// index is less than `dict_limit`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Window { + /// Where input has to continue to be contiguous. + pub(crate) next_src: Addr, + pub(crate) base: Addr, + pub(crate) dict_base: Addr, + /// The buffer of the segment that starts at `dict_limit`. + pub(crate) base_buf: BufId, + /// The buffer of the segment from `low_limit` to `dict_limit`. + pub(crate) dict_buf: BufId, + pub(crate) dict_limit: u32, + /// Below this, there is nothing valid. + pub(crate) low_limit: u32, + pub(crate) nb_overflow_corrections: u32, +} + +impl Window { + /// `ZSTD_window_init` + pub(crate) const fn new() -> Self { + let base = BufId::None.start(); + Self { + next_src: base + ZSTD_WINDOW_START_INDEX as usize, + base, + dict_base: base, + base_buf: BufId::None, + dict_buf: BufId::None, + dict_limit: ZSTD_WINDOW_START_INDEX, + low_limit: ZSTD_WINDOW_START_INDEX, + nb_overflow_corrections: 0, + } + } + + /// The index of `addr`, which is in the current segment or at its end. + #[inline(always)] + pub(crate) const fn index_of(&self, addr: Addr) -> u32 { + addr.wrapping_sub(self.base) as u32 + } + + /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. + pub(crate) fn clear(&mut self) { + let end = self.index_of(self.next_src); + self.low_limit = end; + self.dict_limit = end; + } + + /// `ZSTD_window_isEmpty` + pub(crate) fn is_empty(&self) -> bool { + self.dict_limit == ZSTD_WINDOW_START_INDEX + && self.low_limit == ZSTD_WINDOW_START_INDEX + && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize + } + + /// `ZSTD_window_hasExtDict` + #[inline] + pub(crate) fn has_ext_dict(&self) -> bool { + self.low_limit < self.dict_limit + } + + /// `ZSTD_window_needOverflowCorrection` + #[inline] + pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { + self.index_of(src_end) > ZSTD_CURRENT_MAX + } + + /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns + /// what to subtract from every index that is stored. + pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { + let cycle_size = 1u32 << cycle_log; + let cycle_mask = cycle_size - 1; + let curr = self.index_of(src); + let current_cycle = curr & cycle_mask; + // Indices must not fall below the start index. + let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { + cycle_size.max(ZSTD_WINDOW_START_INDEX) + } else { + 0 + }; + let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); + let correction = curr.wrapping_sub(new_current); + + self.base = self.base.wrapping_add(correction as usize); + self.dict_base = self.dict_base.wrapping_add(correction as usize); + self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.low_limit - correction + }; + self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.dict_limit - correction + }; + self.nb_overflow_corrections += 1; + correction + } + + /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the + /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. + pub(crate) fn enforce_max_dist( + &mut self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { + let new_low_limit = block_end_idx - max_dist; + if self.low_limit < new_low_limit { + self.low_limit = new_low_limit; + } + if self.dict_limit < self.low_limit { + self.dict_limit = self.low_limit; + } + return true; + } + false + } + + /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not + /// right before the current segment, and is to be forgotten. + pub(crate) fn dict_is_invalid( + &self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit + } + + /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not + /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. + pub(crate) fn update( + &mut self, + src: Addr, + src_buf: BufId, + src_size: usize, + force_non_contiguous: bool, + ) -> bool { + let mut contiguous = true; + if src_size == 0 { + return contiguous; + } + if src != self.next_src || force_non_contiguous { + let distance_from_base = self.next_src.wrapping_sub(self.base); + self.low_limit = self.dict_limit; + self.dict_limit = distance_from_base as u32; + self.dict_base = self.base; + self.dict_buf = self.base_buf; + self.base = src.wrapping_sub(distance_from_base); + self.base_buf = src_buf; + if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { + // Too small to be of use. + self.low_limit = self.dict_limit; + } + contiguous = false; + } + self.next_src = src + src_size; + // Where the input overlaps the older segment, it has overwritten it. + if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) + & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) + { + let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); + self.low_limit = if high_input_idx > self.dict_limit as usize { + self.dict_limit + } else { + high_input_idx as u32 + }; + } + contiguous + } +} + +/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes +/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. +#[inline(always)] +pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { + prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 +} + +pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; +pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; + +/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. +#[inline(always)] +pub(crate) fn write_tagged_index( + hash_table: &mut [u32], + hash_and_tag: usize, + index: u32, +) -> Result<()> { + let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; + hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) +} + +/// `ZSTD_comparePackedTags` +#[inline(always)] +pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { + (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) + == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) +} + +// ── the state of the match finders ──────────────────────────────────────── + +/// `ZSTD_match_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Match { + /// As [`SeqDef::off_base`]. + pub(crate) off: u32, + pub(crate) len: u32, +} + +/// `rawSeq`: a sequence found by long distance matching. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct RawSeq { + pub(crate) offset: u32, + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +/// `RawSeqStore_t` +#[derive(Clone, Default)] +pub(crate) struct RawSeqStore { + /// `capacity` of them, of which `size` are in use. + pub(crate) seq: Vec, + /// The sequence that is being read. + pub(crate) pos: usize, + /// How many of its bytes, literals first, are read. + pub(crate) pos_in_sequence: usize, + pub(crate) size: usize, + pub(crate) capacity: usize, +} + +/// `ZSTD_optimal_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Optimal { + /// The cost of getting to this position. + pub(crate) price: i32, + pub(crate) off: u32, + pub(crate) mlen: u32, + pub(crate) litlen: u32, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +/// `ZSTD_OptPrice_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum OptPrice { + Dynamic, + Predef, +} + +/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. +pub(crate) struct OptState { + pub(crate) lit_freq: Vec, + pub(crate) lit_length_freq: Vec, + pub(crate) match_length_freq: Vec, + pub(crate) off_code_freq: Vec, + pub(crate) match_table: Vec, + pub(crate) price_table: Vec, + pub(crate) lit_sum: u32, + pub(crate) lit_length_sum: u32, + pub(crate) match_length_sum: u32, + pub(crate) off_code_sum: u32, + pub(crate) lit_sum_base_price: u32, + pub(crate) lit_length_sum_base_price: u32, + pub(crate) match_length_sum_base_price: u32, + pub(crate) off_code_sum_base_price: u32, + pub(crate) price_type: OptPrice, + pub(crate) literal_compression_mode: ParamSwitch, +} + +impl OptState { + pub(crate) const fn new() -> Self { + Self { + lit_freq: Vec::new(), + lit_length_freq: Vec::new(), + match_length_freq: Vec::new(), + off_code_freq: Vec::new(), + match_table: Vec::new(), + price_table: Vec::new(), + lit_sum: 0, + lit_length_sum: 0, + match_length_sum: 0, + off_code_sum: 0, + lit_sum_base_price: 0, + lit_length_sum_base_price: 0, + match_length_sum_base_price: 0, + off_code_sum_base_price: 0, + price_type: OptPrice::Dynamic, + literal_compression_mode: ParamSwitch::Auto, + } + } +} + +pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; + +/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures +/// own, are in [`BlockSrc`]. +pub(crate) struct MatchState { + pub(crate) window: Window, + /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. + pub(crate) loaded_dict_end: u32, + /// The index from which the tables are still to be filled. + pub(crate) next_to_update: u32, + /// 0 if there is no table of 3-byte hashes. + pub(crate) hash_log3: u32, + pub(crate) row_hash_log: u32, + /// For the row-based match finder: a byte of hash for each entry of `hash_table`. + pub(crate) tag_table: Vec, + pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], + /// Makes what a previous frame left in `tag_table` meaningless. + pub(crate) hash_salt: u64, + pub(crate) hash_salt_entropy: u32, + pub(crate) hash_table: Vec, + pub(crate) hash_table3: Vec, + pub(crate) chain_table: Vec, + pub(crate) force_non_contiguous: bool, + pub(crate) opt: OptState, + /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. + pub(crate) has_dict_match_state: bool, + pub(crate) c_params: CompressionParameters, + pub(crate) lazy_skipping: bool, +} + +impl MatchState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + loaded_dict_end: 0, + next_to_update: 0, + hash_log3: 0, + row_hash_log: 0, + tag_table: Vec::new(), + hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + hash_salt: 0, + hash_salt_entropy: 0, + hash_table: Vec::new(), + hash_table3: Vec::new(), + chain_table: Vec::new(), + force_non_contiguous: false, + opt: OptState::new(), + has_dict_match_state: false, + c_params: CompressionParameters { + window_log: 0, + chain_log: 0, + hash_log: 0, + search_log: 0, + min_match: 0, + target_length: 0, + strategy: None, + }, + lazy_skipping: false, + } + } + + /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. + pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { + if self.window.has_ext_dict() { + return DictMode::ExtDict; + } + if dms.is_some() { + DictMode::DictMatchState + } else { + DictMode::NoDict + } + } + + /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. + #[inline(always)] + pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.low_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + // A dictionary is valid as a whole until it is invalidated as a whole. + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } + + /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. + #[inline(always)] + pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.dict_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } +} + +/// The match state of a dictionary, and where its indices lead. +#[derive(Clone, Copy)] +pub(crate) struct DictMatchState<'a> { + pub(crate) ms: &'a MatchState, + /// The buffer its window is in, which is one segment. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// `window.nextSrc`, as a position in `buf`. + pub(crate) end: usize, +} + +/// The current segment of a window, for the functions that index a dictionary. +#[derive(Clone, Copy)] +pub(crate) struct Prefix<'a> { + /// The buffer that the segment is in. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, +} + +impl Prefix<'_> { + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. +#[derive(Clone, Copy)] +pub(crate) struct BlockSrc<'a> { + /// The buffer that the current segment of the window is in, and so the block. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// The buffer that the older segment of the window is in. + pub(crate) dict_buf: &'a [u8], + /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. + pub(crate) dict_base: usize, + /// Where the block starts in `buf`. + pub(crate) start: usize, + /// Where it ends. + pub(crate) end: usize, + pub(crate) dms: Option>, + /// `opt.symbolCosts`: the tables of the previous block. + pub(crate) symbol_costs: &'a EntropyCTables, + /// `ldmSeqStore`: matches that the optimal parser is to consider. + pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, +} + +impl<'a> BlockSrc<'a> { + #[inline(always)] + pub(crate) const fn prefix(&self) -> Prefix<'a> { + Prefix { + buf: self.buf, + base: self.base, + } + } + + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`, which is in the current segment. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } + + /// The position in `dict_buf` of `index`, which is in the older segment. + #[inline(always)] + pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { + self.dict_base.wrapping_add(index as usize) + } +} + +impl DictMatchState<'_> { + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +// ── long distance matching ──────────────────────────────────────────────── + +/// `ldmEntry_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct LdmEntry { + pub(crate) offset: u32, + pub(crate) checksum: u32, +} + +/// `ldmParams_t` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct LdmParams { + pub(crate) enable_ldm: ParamSwitch, + pub(crate) hash_log: u32, + pub(crate) bucket_size_log: u32, + pub(crate) min_match_length: u32, + pub(crate) hash_rate_log: u32, + pub(crate) window_log: u32, +} + +/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. +pub(crate) struct LdmState { + pub(crate) window: Window, + pub(crate) hash_table: Vec, + pub(crate) loaded_dict_end: u32, + /// The next entry to replace in each bucket. + pub(crate) bucket_offsets: Vec, +} + +impl LdmState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + hash_table: Vec::new(), + loaded_dict_end: 0, + bucket_offsets: Vec::new(), + } + } +} diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs new file mode 100644 index 000000000000..267b0e7c9577 --- /dev/null +++ b/src/zstd/compress/lazy.rs @@ -0,0 +1,1384 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use super::internal::{ + BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, + SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, + hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, + offset_to_offbase, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +const K_LAZY_SKIPPING_STEP: usize = 8; + +// `ZSTD_dictMode_e`, as an argument of a const generic. +const NO_DICT: u8 = DictMode::NoDict as u8; +const EXT_DICT: u8 = DictMode::ExtDict as u8; +const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; + +/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. +#[inline(always)] +const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { + dms.end.wrapping_sub(dms.base) as u32 +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. +#[inline(always)] +fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { + let hash_log = ms.c_params.hash_log; + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let bt = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); + *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. +fn insert_dubt1( + ms: &mut MatchState, + src: &BlockSrc<'_>, + curr: u32, + input_end: usize, + mut nb_compares: u32, + bt_low: u32, +) -> Result<()> { + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let (ip_buf, ip, iend) = if curr >= dict_limit { + (src.buf, src.pos_of(curr), input_end) + } else { + (src.dict_buf, src.dict_pos_of(curr), dict_end) + }; + let window_valid = ms.window.low_limit; + let max_distance = 1u32 << ms.c_params.window_log; + let window_low = if curr.wrapping_sub(window_valid) > max_distance { + curr - max_distance + } else { + window_valid + }; + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + // The next sorted candidate. The other slot has the previous unsorted one. + let mut match_index = bt.at(smaller_ptr)?; + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let in_prefix = + DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; + // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = if in_prefix { + (src.buf, src.pos_of(match_index)) + } else { + (src.dict_buf, src.dict_pos_of(match_index)) + }; + + if in_prefix || curr < dict_limit { + // Both are in the same segment. + match_length += count( + ip_buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + ip_buf, + ip + match_length, + match_buf, + match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if match_index as usize + match_length >= dict_limit as usize { + (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); + } + } + + if ip + match_length == iend { + // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0) +} + +/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. +fn dubt_find_better_dict_match( + ms: &MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mut best_length: usize, + mut nb_compares: u32, + mls: u32, +) -> Result { + let dms = src.dms.ok_or(BUG)?; + let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(h)?; + + let prefix_start = src.pos_of(ms.window.dict_limit); + let curr = src.index_of(ip); + let dict_high_limit = dms_end_index(&dms); + let dict_low_limit = dms.ms.window.low_limit; + let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); + + let dict_bt = dms.ms.chain_table.as_slice(); + let bt_log = dms.ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { + dict_low_limit + } else { + dict_high_limit - bt_mask + }; + + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + + while nb_compares != 0 && dict_match_index > dict_low_limit { + let next_ptr = 2 * (dict_match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let match_index = dict_match_index.wrapping_add(dict_index_delta); + match_length += count_2segments( + src.buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + iend, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(offset_ptr.wrapping_add(1)) as i32 + { + best_length = match_length; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // No way to know which is smaller. + break; + } + } + + if dict_match_index <= bt_low { + // Beyond the tree. + break; + } + let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { + let match_pos = src.pos_of(match_index); + src.buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < src.buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dict_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dict_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + Ok(best_length) +} + +/// `ZSTD_DUBT_findBestMatch` +#[inline(always)] +fn dubt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + off_base_ptr: &mut u32, + mls: u32, +) -> Result { + let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; + let mut match_index = ms.hash_table.at(h)?; + + let curr = src.index_of(ip); + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = curr.saturating_sub(bt_mask); + let unsort_limit = bt_low.max(window_low); + let mut nb_compares = 1u32 << ms.c_params.search_log; + let mut nb_candidates = nb_compares; + let mut previous_candidate = 0u32; + + // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. + let bt = ms.chain_table.as_mut_slice(); + let mut next_candidate = 2 * (match_index & bt_mask) as usize; + while match_index > unsort_limit + && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK + && nb_candidates > 1 + { + bt.put(next_candidate + 1, previous_candidate)?; + previous_candidate = match_index; + match_index = bt.at(next_candidate)?; + next_candidate = 2 * (match_index & bt_mask) as usize; + nb_candidates -= 1; + } + + // If the last candidate is still unsorted, cut the list there. + if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { + *bt.array_at_mut(next_candidate)? = [0, 0]; + } + + // Sort the candidates, the oldest first. + match_index = previous_candidate; + while match_index != 0 { + let next_candidate_idx = ms + .chain_table + .at(2 * (match_index & bt_mask) as usize + 1)?; + insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; + match_index = next_candidate_idx; + nb_candidates += 1; + } + + // Find the longest match. + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + let mut match_end_idx = curr + 8 + 1; + let mut best_length = 0usize; + + match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); + + if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + src.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + let dict_match_pos = src.dict_pos_of(match_index); + match_length += count_2segments( + src.buf, + ip + match_length, + src.dict_buf, + dict_match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + (match_buf, match_pos) = (src.dict_buf, dict_match_pos); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index + match_length as u32; + } + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(*off_base_ptr) as i32 + { + best_length = match_length; + *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // Equal: no way to know which is smaller. + if DICT_MODE == DICT_MATCH_STATE { + // Nothing in the dictionary can be longer. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0)?; + + if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { + best_length = dubt_find_better_dict_match( + ms, + src, + ip, + iend, + off_base_ptr, + best_length, + nb_compares, + mls, + )?; + } + + // Skip repetitive patterns. + ms.next_to_update = match_end_idx - 8; + Ok(best_length) +} + +/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. +/// `off_base_ptr` is written to if there is one. +#[inline(never)] +fn bt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + off_base_ptr: &mut u32, +) -> Result { + if ip < src.pos_of(ms.next_to_update) { + // In an area that is skipped. + return Ok(0); + } + update_dubt(ms, src.prefix(), ip, MLS)?; + dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) +} + +// ── hash chains ─────────────────────────────────────────────────────────── + +/// `ZSTD_insertAndFindFirstIndex_internal` +#[inline(always)] +fn insert_and_find_first_index_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + lazy_skipping: bool, +) -> Result { + let hash_log = ms.c_params.hash_log; + let chain_mask = (1u32 << ms.c_params.chain_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let chain_table = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); + chain_table.put((idx & chain_mask) as usize, next_in_chain)?; + // While skipping, not every position is inserted. + if lazy_skipping { + break; + } + } + + ms.next_to_update = target; + hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) +} + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) +} + +/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn hc_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let chain_size = 1u32 << ms.c_params.chain_log; + let chain_mask = chain_size - 1; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let min_chain = curr.saturating_sub(chain_size); + let mut nb_attempts = 1u32 << ms.c_params.search_log; + let mut ml = 4 - 1; + + let mut match_index = + insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; + let chain_table = ms.chain_table.as_slice(); + + while match_index >= low_limit && nb_attempts > 0 { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + + if match_index <= min_chain { + break; + } + match_index = chain_table.at((match_index & chain_mask) as usize)?; + nb_attempts -= 1; + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_chain_table = dms.ms.chain_table.as_slice(); + let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; + let dms_chain_mask = dms_chain_size - 1; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_size = dms_end_index(&dms); + let dms_index_delta = dict_limit.wrapping_sub(dms_size); + let dms_min_chain = dms_size.saturating_sub(dms_chain_size); + + let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; + match_index = dms.ms.hash_table.at(h)?; + + while match_index >= dms_lowest_index && nb_attempts > 0 { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + + if match_index <= dms_min_chain { + break; + } + match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; + nb_attempts -= 1; + } + } + + Ok(ml) +} + +// ── row-based match finder ──────────────────────────────────────────────── +// +// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits +// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag +// table is the "head": which entry was written last. + +pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; +const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; +const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; + +/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. +#[inline(always)] +fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { + let mut next = u32::from(*head).wrapping_sub(1) & row_mask; + if next == 0 { + next = row_mask; + } + *head = next as u8; + next +} + +#[inline(always)] +fn prefetch_l1(slice: &[T], index: usize) { + core::hint::prefetch_read(slice.as_ptr().wrapping_add(index), core::hint::Locality::L1); +} + +#[inline(always)] +fn row_prefetch(hash_table: &[u32], tag_table: &[u8], rel_row: usize, row_log: u32) { + prefetch_l1(hash_table, rel_row); + if row_log >= 5 { + prefetch_l1(hash_table, rel_row + 16); + } + prefetch_l1(tag_table, rel_row); + if row_log == 6 { + prefetch_l1(tag_table, rel_row + 32); + } +} + +/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. +#[inline(always)] +fn row_fill_hash_cache( + ms: &mut MatchState, + prefix: Prefix<'_>, + mls: u32, + idx: u32, + i_limit: usize, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); + let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; + + for idx in idx..lim { + let hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + ms.hash_salt, + )? as u32; + let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; + ms.hash_cache.put(cached, hash)?; + } + Ok(()) +} + +/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes +/// further takes its place. +#[inline(always)] +fn row_next_cached_hash( + cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], + hash_table: &[u32], + tag_table: &[u8], + row_log: u32, + prefix: Prefix<'_>, + idx: u32, + hash_log: u32, + mls: u32, + hash_salt: u64, +) -> Result { + let new_hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32; + row_prefetch(hash_table, tag_table, ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize, row_log); + let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; + Ok(core::mem::replace(cached, new_hash)) +} + +/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. +#[inline(always)] +fn row_update_internal_impl( + ms: &mut MatchState, + prefix: Prefix<'_>, + update_start_idx: u32, + update_end_idx: u32, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let hash_salt = ms.hash_salt; + let hash_table = ms.hash_table.as_mut_slice(); + let tag_table = ms.tag_table.as_mut_slice(); + + for idx in update_start_idx..update_end_idx { + let hash = if use_cache { + row_next_cached_hash(&mut ms.hash_cache, hash_table, tag_table, row_log, prefix, idx, hash_log, mls, hash_salt)? + } else { + hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32 + }; + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; + tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; + hash_table.put(rel_row + pos, idx)?; + } + Ok(()) +} + +/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is +/// when compressing, not all of them if they are many. +#[inline(always)] +fn row_update_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + const K_SKIP_THRESHOLD: u32 = 384; + const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; + const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; + + let mut idx = ms.next_to_update; + let target = prefix.index_of(ip); + + if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { + let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; + row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; + idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; + row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; + } + row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the +/// row-based match finder. +pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { + let row_log = ms.c_params.search_log.clamp(4, 6); + let row_mask = (1u32 << row_log) - 1; + let mls = ms.c_params.min_match.min(6); + row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) +} + +/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. +#[inline(always)] +fn row_get_match_mask( + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + head: u32, +) -> u64 { + let mut matches = 0u64; + let (chunks, _) = tag_row.as_chunks::<16>(); + for (n, chunk) in chunks.iter().enumerate() { + let mut chunk_matches = 0u16; + for (i, &entry) in chunk.iter().enumerate() { + chunk_matches |= u16::from(entry == tag) << i; + } + matches |= u64::from(chunk_matches) << (16 * n); + } + match ROW_ENTRIES { + 16 => u64::from((matches as u16).rotate_right(head)), + 32 => u64::from((matches as u32).rotate_right(head)), + _ => matches.rotate_right(head), + } +} + +/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, +/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than +/// `low_limit`. Returns how many there are. +#[inline(always)] +fn row_collect_matches( + row: &[u32; ROW_ENTRIES], + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + low_limit: u32, + nb_attempts: &mut u32, + match_buffer: &mut [u32; ROW_ENTRIES], + prefetch: impl Fn(u32), +) -> Result { + let row_mask = ROW_ENTRIES as u32 - 1; + let head = u32::from(tag_row.at(0)?) & row_mask; + let mut num_matches = 0usize; + let mut matches = row_get_match_mask(tag_row, tag, head); + + while matches > 0 && *nb_attempts > 0 { + let match_pos = (head + matches.trailing_zeros()) & row_mask; + matches &= matches - 1; + if match_pos == 0 { + // The head, which happens to be the same as the tag. + continue; + } + let match_index = row.at(match_pos as usize)?; + if match_index < low_limit { + break; + } + prefetch(match_index); + match_buffer.put(num_matches, match_index)?; + num_matches += 1; + *nb_attempts -= 1; + } + Ok(num_matches) +} + +/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn row_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let hash_log = ms.row_hash_log; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let row_log = ROW_ENTRIES.trailing_zeros(); + let row_mask = ROW_ENTRIES as u32 - 1; + // There are no more searches than entries in a row. + let capped_search_log = ms.c_params.search_log.min(row_log); + let hash_salt = ms.hash_salt; + let mut nb_attempts = 1u32 << capped_search_log; + let mut ml = 4 - 1; + let mut match_buffer = [0u32; ROW_ENTRIES]; + + // Update the tables up to, and not including, `ip`. + let hash = if ms.lazy_skipping { + // While skipping, not every position is inserted. + ms.next_to_update = curr; + hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 + } else { + let prefix = src.prefix(); + row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; + row_next_cached_hash(&mut ms.hash_cache, &ms.hash_table, &ms.tag_table, row_log, prefix, curr, hash_log, MLS, hash_salt)? + }; + ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); + + { + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; + let row = ms.hash_table.array_at_mut::(rel_row)?; + let tag_row = ms.tag_table.array_at_mut::(rel_row)?; + let num_matches = row_collect_matches( + row, + tag_row, + tag, + low_limit, + &mut nb_attempts, + &mut match_buffer, + |match_index| { + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + prefetch_l1(buf, src.pos_of(match_index)); + } else { + prefetch_l1(src.dict_buf, src.dict_pos_of(match_index)); + } + }, + )?; + + // Insert `ip`. + let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; + tag_row.put(pos, tag)?; + row.put(pos, ms.next_to_update)?; + ms.next_to_update += 1; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + } + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); + + let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + let dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; + let dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let num_matches = row_collect_matches( + dms.ms.hash_table.array_at::(dms_rel_row)?, + dms.ms.tag_table.array_at::(dms_rel_row)?, + (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, + dms_lowest_index, + &mut nb_attempts, + &mut match_buffer, + |match_index| prefetch_l1(dms.buf, dms.pos_of(match_index)), + )?; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + } + } + + Ok(ml) +} + +// ── the parsers ─────────────────────────────────────────────────────────── + +// `searchMethod_e`, as an argument of a const generic. +const SEARCH_HASH_CHAIN: u8 = 0; +const SEARCH_BINARY_TREE: u8 = 1; +const SEARCH_ROW_HASH: u8 = 2; + +/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. +#[inline(always)] +fn search_max( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mls: u32, + row_log: u32, +) -> Result { + match SEARCH_METHOD { + SEARCH_HASH_CHAIN => match mls { + 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + SEARCH_BINARY_TREE => match mls { + 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + _ => match (mls, row_log) { + (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + } +} + +/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match +/// state is searched. +#[derive(Clone, Copy)] +struct DictSegment<'a> { + buf: &'a [u8], + /// Its sum with an index of the window is a position in `buf`. It wraps around. + base: usize, + /// Where the segment starts in `buf`. + start: usize, + /// Where it ends. + end: usize, +} + +/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. +#[inline(always)] +fn rep_length( + ms: &MatchState, + src: &BlockSrc<'_>, + dict: &DictSegment<'_>, + ip: usize, + offset: u32, +) -> Result { + let buf = src.buf; + let iend = src.end; + if DICT_MODE == NO_DICT { + let rep_match = ip.wrapping_sub(offset as usize); + if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { + return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); + } + return Ok(0); + } + + let dict_limit = ms.window.dict_limit; + let curr = src.index_of(ip); + let rep_index = curr.wrapping_sub(offset); + if !index_overlap_check(dict_limit, rep_index) { + return Ok(0); + } + if DICT_MODE == EXT_DICT { + // The offset may lead to what is no longer in the window. + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + if offset > curr.wrapping_sub(window_low) { + return Ok(0); + } + } + let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { + let rep_match = dict.base.wrapping_add(rep_index as usize); + (dict.buf, rep_match, dict.end) + } else { + (buf, src.pos_of(rep_index), iend) + }; + if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { + return Ok(0); + } + let prefix_start = src.pos_of(dict_limit); + Ok(count_2segments( + buf, + ip + 4, + rep_buf, + rep_match + 4, + iend, + rep_end, + prefix_start, + )? + 4) +} + +/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions +/// further a better match is looked for, once there is one. +#[inline(never)] +fn compress_block_lazy_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let prefix_lowest_index = ms.window.dict_limit; + let prefix_lowest = src.pos_of(prefix_lowest_index); + let mls = ms.c_params.min_match.clamp(4, 6); + let row_log = ms.c_params.search_log.clamp(4, 6); + + let [rep_0, rep_1, _] = rep; + let mut offset_1 = *rep_0; + let mut offset_2 = *rep_1; + let mut offset_saved_1 = 0u32; + let mut offset_saved_2 = 0u32; + + ms.lazy_skipping = false; + + let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { + 8 + ZSTD_ROW_HASH_CACHE_SIZE + } else { + 8 + }; + let Some(ilimit) = iend.checked_sub(margin) else { + // There is no position to search, nor to hash. + return Ok(iend - istart); + }; + + let dict = match DICT_MODE { + NO_DICT => DictSegment { + buf: &[], + base: 0, + start: 0, + end: 0, + }, + EXT_DICT => DictSegment { + buf: src.dict_buf, + base: src.dict_base, + start: src.dict_pos_of(ms.window.low_limit), + end: src.dict_pos_of(prefix_lowest_index), + }, + _ => { + let dms = src.dms.ok_or(BUG)?; + let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); + DictSegment { + buf: dms.buf, + base: dms.base.wrapping_sub(dict_index_delta as usize), + start: dms.pos_of(dms.ms.window.dict_limit), + end: dms.end, + } + } + }; + + if DICT_MODE == EXT_DICT { + ip += usize::from(ip == prefix_lowest); + } else { + let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); + ip += usize::from(dict_and_prefix_length == 0); + } + if DICT_MODE == NO_DICT { + let curr = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); + let max_rep = curr - window_low; + if offset_2 > max_rep { + offset_saved_2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved_1 = offset_1; + offset_1 = 0; + } + } + + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + + 'parse: while ip < ilimit { + let mut match_length; + let mut off_base = REPCODE1_TO_OFFBASE; + let mut start = ip + 1; + + 'found: { + // Check the repeat offset. + match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; + if DEPTH == 0 && match_length != 0 { + break 'found; + } + + // First search. + { + let mut off_base_found = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut off_base_found, + mls, + row_log, + )?; + if ml2 > match_length { + match_length = ml2; + start = ip; + off_base = off_base_found; + } + } + + if match_length < 4 { + // Jump faster over incompressible sections. + let step = (ip - anchor) >> K_SEARCH_STRENGTH; + ip += step + 1; + // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of + // no use out of the tables. As in the C code, it does not start at the same step in all modes. + ms.lazy_skipping = if DICT_MODE == EXT_DICT { + step > K_LAZY_SKIPPING_STEP + } else { + step + 1 > K_LAZY_SKIPPING_STEP + }; + continue 'parse; + } + + // Look for something better at the next positions. + if DEPTH >= 1 { + while ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 3) as i32; + let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + + // Nothing better: try one position further. + if DEPTH == 2 && ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 4) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + } + break; + } + } + + // Catch up: the match may start earlier. + if offbase_is_offset(off_base) { + let offset = offbase_to_offset(off_base); + let match_index = src.index_of(start).wrapping_sub(offset); + let (match_buf, mut match_pos, m_start) = + if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { + let match_pos = dict.base.wrapping_add(match_index as usize); + (dict.buf, match_pos, dict.start) + } else { + (buf, src.pos_of(match_index), prefix_lowest) + }; + while start > anchor + && match_pos > m_start + && buf.at(start - 1)? == match_buf.at(match_pos - 1)? + { + start -= 1; + match_pos -= 1; + match_length += 1; + } + offset_2 = offset_1; + offset_1 = offset; + } + } + + // Store the sequence. + seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; + ip = start + match_length; + anchor = ip; + + if ms.lazy_skipping { + // There was a match: every position is inserted again. + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + ms.lazy_skipping = false; + } + + // Check the second repeat offset right away. + while ip <= ilimit { + match_length = rep_length::(ms, src, &dict, ip, offset_2)?; + if match_length == 0 { + break; + } + (offset_1, offset_2) = (offset_2, offset_1); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; + ip += match_length; + anchor = ip; + } + } + + // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. + if offset_saved_1 != 0 && offset_1 != 0 { + offset_saved_2 = offset_saved_1; + } + + // Save the repeat offsets for the next block. + *rep_0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved_1 + }; + *rep_1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved_2 + }; + + // The size of the last literals. + Ok(iend - anchor) +} + +/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. +#[inline(always)] +fn compress_block_lazy_any_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + match dict_mode { + DictMode::NoDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::ExtDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + compress_block_lazy_generic::( + ms, seq_store, rep, src, + ) + } + } +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row: bool, + dict_mode: DictMode, +) -> Result { + match (strategy, use_row) { + (Strategy::Greedy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Greedy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( + ms, seq_store, rep, src, dict_mode, + ), + _ => Err(BUG), + } +} diff --git a/src/zstd/compress/ldm.rs b/src/zstd/compress/ldm.rs new file mode 100644 index 000000000000..7f060fcda748 --- /dev/null +++ b/src/zstd/compress/ldm.rs @@ -0,0 +1,779 @@ +//! `zstd_ldm.c`: long distance matching. + +use super::internal::{ + BlockSrc, Buffers, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, LdmEntry, + LdmParams, LdmState, MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, + TableFillPurpose, count, count_2segments, offset_to_offbase, +}; +use super::{compress_block, double_fast, fast}; +use crate::common::ZSTD_REP_NUM; +use crate::common::xxhash::Xxh64; +use crate::error::{ErrorCode, Result}; +use crate::util::{BUG, SliceExt}; + +const ZSTD_HASHLOG_MIN: u32 = 6; +const ZSTD_HASHLOG_MAX: u32 = 30; +const ZSTD_LDM_BUCKETSIZELOG_MAX: u32 = 8; + +const LDM_BUCKET_SIZE_LOG: u32 = 4; +const LDM_MIN_MATCH_LENGTH: u32 = 64; +const LDM_BATCH_SIZE: usize = 64; + +/// `ZSTD_ldm_gearTab` +#[rustfmt::skip] +static GEAR_TAB: [u64; 256] = [ + 0xf5b8_f72c_5f77_775c, 0x8493_5f26_6b7a_c412, 0xb647_ada9_ca73_0ccc, 0xb065_bb4b_114f_b1de, + 0x3458_4e7e_8c3a_9fd0, 0x4e97_e17c_6ae2_6b05, 0x3a03_d743_bc99_a604, 0xcecd_0424_22c4_044f, + 0x76de_76c5_8524_259e, 0x9c85_28f6_5bad_eaca, 0x8656_3706_e209_7529, 0x2902_475f_a375_d889, + 0xafb3_2a97_39a5_ebe6, 0xce27_14da_3883_e639, 0x021e_af82_1722_e69e, 0x0003_7b62_8620_b628, + 0x049a_8d45_5d88_caf5, 0x8556_d711_e695_8140, 0x04f7_ae74_fc60_5c1f, 0x829f_0c34_68bd_3a20, + 0x4ffd_c885_c625_179e, 0x8473_de04_8a3d_af1b, 0x5100_8822_b056_46b2, 0x69d7_5d12_b2d1_cc5f, + 0x8c9d_4a19_1591_54bc, 0xc3cc_10f4_abbd_4003, 0xd06d_dc1c_ecb9_7391, 0xbe48_e6e7_ed80_302e, + 0x3481_db31_cee0_3547, 0xacc3_f67c_daa1_d210, 0x65cb_771d_8c7f_96cc, 0x8eb2_7177_0557_23dd, + 0xc789_950d_44cd_94be, 0x934f_eadc_3700_b12b, 0x5e48_5f11_edbd_f182, 0x1e2e_2a46_fd64_767a, + 0x2969_ca71_d82e_fa7c, 0x9d46_e993_5ebb_ba2e, 0xe056_b67e_05e6_822b, 0x94d7_3f55_739d_03a0, + 0xcd70_10bd_b69b_5a03, 0x455e_f9fc_d79b_82f4, 0x869c_b54a_8749_c161, 0x38d1_a4fa_6185_d225, + 0xb475_166f_94bb_e9bb, 0xa414_3548_7209_59f1, 0x7aed_4780_ba6b_26ba, 0xd0ce_2644_39e0_2312, + 0x8436_6d74_6078_d508, 0xa8ce_973c_72ed_17be, 0x21c3_23a2_9a43_0b01, 0x9962_d617_e3af_80ee, + 0xab0c_e91d_9c8c_f75b, 0x530e_8ee6_d19a_4dbc, 0x2ef6_8c0c_f53f_5d72, 0xc03a_6816_40a8_5506, + 0x496e_4e9f_9c31_0967, 0x7858_0472_b59b_14a0, 0x2738_24c2_3b38_8577, 0x66bf_923a_d45c_b553, + 0x47ae_1a5a_2492_ba86, 0x35e3_0456_9e22_9659, 0x4765_182a_4687_0b6f, 0x6cba_b625_e909_9412, + 0xddac_9a2e_5985_22c1, 0x7172_086e_6666_24f2, 0xdf50_03ca_503b_7837, 0x88c0_c1db_7856_3d09, + 0x58d5_1865_acfc_289d, 0x1776_71ae_c652_24f1, 0xfb79_d8a2_41e9_67d7, 0x2be1_e101_cad9_a49a, + 0x6625_682f_6e29_186b, 0x3995_5345_7ac0_6e50, 0x035d_ffb4_c23a_bb74, 0x429d_b259_1f54_aade, + 0xc528_02a8_037d_1009, 0x6acb_2738_1f0b_25f3, 0xf45e_2551_ee4f_823b, 0x8b0e_a2d9_9580_c2f7, + 0x3bed_519c_bcb4_e1e1, 0x0ff4_5282_3dbb_010a, 0x9d42_ed61_4f3d_d267, 0x5b93_13c0_6257_c57b, + 0xa114_b800_8b5e_1442, 0xc1fe_311c_11c1_3d4b, 0x66e8_763e_a34c_5568, 0x8b98_2af1_c262_f05d, + 0xee88_76fa_aa75_fbb7, 0x8a62_a4d0_d172_bb2a, 0xc13d_94a3_b744_9a97, 0x6dbb_ba9d_c15d_037c, + 0xc786_101f_1d92_e0f1, 0xd786_81a9_07a0_b79b, 0xf61a_af29_62c9_abb9, 0x2cfd_16fc_d3cb_7ad9, + 0x868c_5b67_4462_4d21, 0x25e6_5089_9c74_ddd7, 0xba04_2af4_a7c3_7463, 0x4eb1_a539_465a_3eca, + 0xbe09_dbf0_3b05_d5ca, 0x774e_5a36_2b54_72ba, 0x47a1_2212_29d1_83cd, 0x504b_0ca1_8ef5_a2df, + 0xdffb_dfbd_e245_6eb9, 0x46cd_2b2f_bee3_4634, 0xf2ae_f8fe_819d_98c3, 0x357f_5276_d459_9d61, + 0x24a5_4838_79c4_53e3, 0x0880_2688_9192_b4b9, 0x28da_9667_1782_dbec, 0x4ef3_7c40_588e_9aaa, + 0x8837_b906_51bc_9fb3, 0xc164_f741_d3f0_e5d6, 0xbc13_5a0a_704b_70ba, 0x069c_d868_f762_2ada, + 0xbc37_ba89_e0b9_c0ab, 0x47c1_4a01_3235_52f6, 0x4f00_794b_acee_98bb, 0x7107_de7d_637a_69d5, + 0x88af_793b_b6f2_255e, 0xf3c6_466b_8799_b598, 0xc288_c616_aa7f_3b59, 0x81ca_63cf_42fc_a3fd, + 0x88d8_5ace_36a2_674b, 0x0d05_6bd3_7923_89e7, 0xe55c_396c_4e9d_d32d, 0xbefb_5045_71e6_c0a6, + 0x96ab_3211_5e91_e8cc, 0xbf8a_cb18_de8f_38d1, 0x66da_e588_0167_2606, 0x833b_6017_8723_17fb, + 0xb87c_16f2_d1c9_2864, 0xdb76_6a74_e58b_669c, 0x8965_9f85_c614_17be, 0xc8da_ad85_6011_ea0c, + 0x76a4_b565_b6fe_7eae, 0xa469_d085_f623_7312, 0xaaf0_3656_83a3_e96c, 0x4dbb_746f_8424_f7b8, + 0x0638_755a_f4e4_acc1, 0x3d78_07f5_bde6_4486, 0x17be_6d8f_5bbb_7639, 0x0903_f0cd_44dc_35dc, + 0x67b6_72ea_fdf1_196c, 0xa676_ff93_ed4c_82f1, 0x521d_1004_c505_3d9d, 0x37ba_9ad0_9ccc_9202, + 0x84e5_4d29_7aac_fb51, 0x0a0b_4b77_6a14_3445, 0x0820_d471_e20b_348e, 0x1874_383c_b83d_46dc, + 0x97ed_eec7_a1ef_e11c, 0xb330_e50b_1bdc_42aa, 0x1dd9_1955_ce70_e032, 0xa514_cdb8_8f29_39d5, + 0x2791_233f_d90d_b9d3, 0x7b67_0a4c_c50f_7a9b, 0x77c0_7d2a_05c6_dfa5, 0xe377_8b66_46d0_a6fa, + 0xb39c_8eda_47b5_6749, 0x933e_d448_addb_ef28, 0xaf84_6af6_ab7d_0bf4, 0x0e5a_f208_eb66_6e49, + 0x5e66_22f7_3534_cd6a, 0x297d_aeca_42ef_5b6e, 0x862d_aef3_d355_39a6, 0xe687_2249_8f8e_1ea9, + 0x981c_5309_3dc0_d572, 0xfa09_b0bf_bf86_fbf5, 0x30b1_e961_6621_9f15, 0x70e7_d466_bdc4_fb83, + 0x5a66_736e_35f2_a8e9, 0xcddb_59d2_b7c1_baef, 0xd6c7_d247_d26d_8996, 0xea4e_39ea_c8de_1ba3, + 0x539c_8bb1_9fa3_aff2, 0x09f9_0e4c_5fd5_08d8, 0xa34e_5956_fbaf_3385, 0x2e2f_8e15_1d3e_f375, + 0x1736_91e9_b83f_aec1, 0xb85a_8d56_bf01_6379, 0x8382_3812_6740_8ae3, 0xb90f_901b_bdc0_096d, + 0x7c6a_d329_33bc_ec65, 0x76bb_5e2f_2c8a_d595, 0x390f_851a_6cf4_6d28, 0xc3e6_064d_a1c2_da72, + 0xc52a_0c10_1cfa_5389, 0xd78e_af84_a3fb_c530, 0x3781_b9e2_288b_997e, 0x73c2_f6de_a83d_05c4, + 0x0422_8e36_4c5b_5ed7, 0x9d7a_3edf_0da4_3911, 0x8edc_feda_2468_6756, 0x5e76_67a7_b7a9_b3a1, + 0x4c4f_389f_a143_791d, 0xb08b_c102_3da7_cddc, 0x7ab4_be3a_e529_b1cc, 0x754e_6132_dbe7_4ff9, + 0x7163_5442_a839_df45, 0x2f6f_b164_3fbe_52de, 0x961e_0a42_cf7a_8177, 0xf3b4_5d83_d89e_f2ea, + 0xee3d_e4cf_4a6e_3e9b, 0xcd68_4854_2c32_95e7, 0xe4ce_e166_4c78_662f, 0x9947_548b_474c_68c4, + 0x25d7_3777_a5ed_8b0b, 0x00c9_15b1_d636_b7fc, 0x21c2_ba75_d9b0_d2da, 0x5f6b_5dcf_608a_64a1, + 0xdcf3_3325_5ff9_570c, 0x633b_9224_18ce_d4ee, 0xc136_dde0_b004_b34a, 0x58cc_83b0_5d4b_2f5a, + 0x5eb4_24dd_a28e_42d2, 0x62df_4736_9739_cd98, 0xb4e0_b424_85e4_ce17, 0x16e1_f0c1_f9a8_d1e7, + 0x8ec3_9167_0756_0ebf, 0x62ba_6e2d_f2cc_9db3, 0xcbf9_f4ff_77d8_3a16, 0x78d9_d7d0_7d2b_bcc4, + 0xef55_4ce1_e02c_41f4, 0x8d75_8112_7ecc_f94d, 0xa9b5_3336_cb3c_8a05, 0x38c4_2c0b_f45c_4f91, + 0x6408_93cd_f448_8863, 0x80ec_34bc_575e_a568, 0x39f3_24f5_b48e_aa40, 0xe9d9_ed1f_8eff_527f, + 0x9224_fc05_8cc5_a214, 0xbaba_00b0_4cfe_7741, 0x309a_9f12_0fcf_52af, 0xa558_f3ec_6562_6212, + 0x424b_ec8b_7ada_be2f, 0x4162_2513_a6ae_a433, 0xb88d_a2d5_324c_a798, 0xd287_733b_2455_28a4, + 0x9a44_697e_6d68_aec3, 0x7b10_93be_2f49_bb28, 0x50bb_ec63_2e3d_8aad, 0x6cd9_0723_e1ea_8283, + 0x897b_9e74_31b0_2bf3, 0x219e_fdcb_338a_7047, 0x3b03_11f0_a27c_0656, 0xdb17_bf91_c0db_96e7, + 0x8cd4_fd6b_4e85_a5b2, 0xfab0_7105_4ba6_409d, 0x40d6_fe83_1fa9_dfd9, 0xaf35_8deb_ad7d_791e, + 0xeb8d_0e25_a65e_3e58, 0xbbcb_d3df_14e0_8580, 0x0cf7_51f2_7ecd_ab2b, 0x2b4d_a14f_2613_d8f4, +]; + +/// `ldmRollingHashState_t` +struct RollingHashState { + rolling: u64, + stop_mask: u64, +} + +impl RollingHashState { + /// `ZSTD_ldm_gear_init` + fn new(params: &LdmParams) -> Self { + let max_bits_in_mask = params.min_match_length.min(64); + let hash_rate_log = params.hash_rate_log; + let rate_mask = (1u64 << (hash_rate_log & 63)) - 1; + // A split is to happen every `1 << hash_rate_log` bytes on average, and bit n of the hash depends on the last + // n bytes: the mask has the highest bits that depend on no more than `min_match_length` bytes. + let stop_mask = if hash_rate_log > 0 && hash_rate_log <= max_bits_in_mask { + rate_mask << ((max_bits_in_mask - hash_rate_log) & 63) + } else { + rate_mask + }; + Self { + rolling: u64::from(u32::MAX), + stop_mask, + } + } + + /// `ZSTD_ldm_gear_feed`: writes to `splits` the split points of `data`, as positions in it. Stops at the end of + /// `data` or once `splits` is full. Returns how many bytes it went through, and how many splits it found. + #[inline] + fn feed(&mut self, data: &[u8], splits: &mut [usize; LDM_BATCH_SIZE]) -> (usize, usize) { + let mut hash = self.rolling; + let mask = self.stop_mask; + let mut n = 0usize; + let mut num_splits = 0usize; + // Returns whether `splits` is full. + let mut iter_once = |byte: u8| { + let gear = GEAR_TAB.get(usize::from(byte)).copied().unwrap_or(0); + hash = (hash << 1).wrapping_add(gear); + n += 1; + if hash & mask == 0 { + if let Some(split) = splits.get_mut(num_splits) { + *split = n; + } + num_splits += 1; + return num_splits == LDM_BATCH_SIZE; + } + false + }; + let (quads, rest) = data.as_chunks::<4>(); + let full = quads + .iter() + .any(|&[a, b, c, d]| iter_once(a) || iter_once(b) || iter_once(c) || iter_once(d)); + if !full { + for &byte in rest { + if iter_once(byte) { + break; + } + } + } + self.rolling = hash; + (n, num_splits) + } +} + +/// `XXH64`, with a seed of 0. +#[inline] +fn xxh64(data: &[u8]) -> u64 { + let mut state = Xxh64::new(); + state.update(data); + state.digest() +} + +/// `ZSTD_ldm_adjustParameters`: gives a value to the parameters that are 0. `c_params` are those that are applied. +pub(crate) fn adjust_parameters(params: &mut LdmParams, c_params: &CompressionParameters) { + let strategy = c_params.strat(); + params.window_log = c_params.window_log; + if params.hash_rate_log == 0 { + if params.hash_log > 0 { + if params.window_log > params.hash_log { + params.hash_rate_log = params.window_log - params.hash_log; + } + } else { + // From 7 for `fast` to 4 for `btultra2`. + params.hash_rate_log = 7 - (strategy as u32 / 3); + } + } + if params.hash_log == 0 { + params.hash_log = params + .window_log + .wrapping_sub(params.hash_rate_log) + .clamp(ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + } + if params.min_match_length == 0 { + params.min_match_length = LDM_MIN_MATCH_LENGTH; + if strategy >= Strategy::BtUltra { + params.min_match_length /= 2; + } + } + if params.bucket_size_log == 0 { + params.bucket_size_log = + (strategy as u32).clamp(LDM_BUCKET_SIZE_LOG, ZSTD_LDM_BUCKETSIZELOG_MAX); + } + params.bucket_size_log = params.bucket_size_log.min(params.hash_log); +} + +/// `ZSTD_cwksp_alloc_size`, which adds something only under ASAN. +const fn cwksp_alloc_size(size: usize) -> usize { + size +} + +/// `ZSTD_ldm_getTableSize`: how many bytes of workspace the C code takes for the tables. +pub(crate) fn get_table_size(params: LdmParams) -> usize { + let ldm_h_size = 1usize << (params.hash_log & 63); + let ldm_bucket_size_log = params.bucket_size_log.min(params.hash_log); + let ldm_bucket_size = 1usize << ((params.hash_log - ldm_bucket_size_log) & 63); + let total_size = cwksp_alloc_size(ldm_bucket_size) + + cwksp_alloc_size(ldm_h_size.wrapping_mul(size_of::())); + if params.enable_ldm == ParamSwitch::Enable { + total_size + } else { + 0 + } +} + +/// `ZSTD_ldm_getMaxNbSeq`: the most sequences that `max_chunk_size` bytes can give. +pub(crate) fn get_max_nb_seq(params: LdmParams, max_chunk_size: usize) -> usize { + if params.enable_ldm == ParamSwitch::Enable { + max_chunk_size + .checked_div(params.min_match_length as usize) + .unwrap_or(0) + } else { + 0 + } +} + +/// The mask that keeps, of a hash, the number of a bucket. +#[inline] +const fn bucket_mask(params: &LdmParams) -> u32 { + (1u32 << (params.hash_log.wrapping_sub(params.bucket_size_log) & 31)) - 1 +} + +/// `ZSTD_ldm_getBucket`: where the bucket `hash` starts in the hash table. +#[inline(always)] +const fn get_bucket(hash: usize, bucket_size_log: u32) -> usize { + hash << (bucket_size_log & 31) +} + +/// `ZSTD_ldm_insertEntry`: in place of the oldest entry of the bucket `hash`. +#[inline] +fn insert_entry( + ldm_state: &mut LdmState, + hash: usize, + entry: LdmEntry, + bucket_size_log: u32, +) -> Result<()> { + let p_offset = ldm_state.bucket_offsets.at_mut(hash)?; + let offset = u32::from(*p_offset); + *p_offset = ((offset + 1) & ((1u32 << (bucket_size_log & 31)) - 1)) as u8; + ldm_state + .hash_table + .put(get_bucket(hash, bucket_size_log) + offset as usize, entry) +} + +/// `ZSTD_ldm_countBackwardsMatch`: how many bytes before `p_in` in `buf`, down to `p_anchor`, are the same as those +/// before `p_match` in `match_buf`, down to `p_match_base`. +#[inline] +fn count_backwards_match( + buf: &[u8], + p_in: usize, + p_anchor: usize, + match_buf: &[u8], + p_match: usize, + p_match_base: usize, +) -> Result { + Ok(buf + .sub(p_anchor..p_in)? + .iter() + .rev() + .zip(match_buf.sub(p_match_base..p_match)?.iter().rev()) + .take_while(|(in_byte, match_byte)| in_byte == match_byte) + .count()) +} + +/// `ZSTD_ldm_countBackwardsMatch_2segments`, for a match in the current segment, which starts at `p_match_base`: from +/// there, the match goes on at the end of the older segment. A match in the older segment has nowhere to go on. +#[inline] +fn count_backwards_match_2segments( + buf: &[u8], + p_in: usize, + p_anchor: usize, + p_match: usize, + p_match_base: usize, + dict_buf: &[u8], + p_ext_dict_start: usize, + p_ext_dict_end: usize, +) -> Result { + let match_length = count_backwards_match(buf, p_in, p_anchor, buf, p_match, p_match_base)?; + if p_match - match_length != p_match_base { + return Ok(match_length); + } + Ok(match_length + + count_backwards_match( + buf, + p_in - match_length, + p_anchor, + dict_buf, + p_ext_dict_end, + p_ext_dict_start, + )?) +} + +/// `ZSTD_ldm_fillFastTables`: the strategies that index only what they search have to be told about what they skip. +fn fill_fast_tables(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) -> Result<()> { + match ms.c_params.strat() { + Strategy::Fast => fast::fill_hash_table( + ms, + prefix, + end, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + ), + Strategy::DFast => double_fast::fill_double_hash_table( + ms, + prefix, + end, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + ), + Strategy::Greedy + | Strategy::Lazy + | Strategy::Lazy2 + | Strategy::BtLazy2 + | Strategy::BtOpt + | Strategy::BtUltra + | Strategy::BtUltra2 => Ok(()), + } +} + +/// `ZSTD_ldm_fillHashTable`: indexes the content of a dictionary, which is `prefix.buf` from `ip` to `iend`. +pub(crate) fn fill_hash_table( + ldm_state: &mut LdmState, + prefix: Prefix<'_>, + mut ip: usize, + iend: usize, + params: &LdmParams, +) -> Result<()> { + let min_match_length = params.min_match_length as usize; + let bucket_mask = bucket_mask(params); + let istart = ip; + let mut hash_state = RollingHashState::new(params); + let mut splits = [0usize; LDM_BATCH_SIZE]; + + while ip < iend { + let (hashed, num_splits) = hash_state.feed(prefix.buf.sub(ip..iend)?, &mut splits); + for &split in splits.iter().take(num_splits) { + if ip + split >= istart + min_match_length { + let split = ip + split - min_match_length; + let xxhash = xxh64(prefix.buf.span(split, min_match_length)?); + let hash = xxhash as u32 & bucket_mask; + let entry = LdmEntry { + offset: prefix.index_of(split), + checksum: (xxhash >> 32) as u32, + }; + insert_entry(ldm_state, hash as usize, entry, params.bucket_size_log)?; + } + } + ip += hashed; + } + Ok(()) +} + +/// `ZSTD_ldm_limitTableUpdate`: if the tables of `ms` are far behind `anchor`, skips most of the way. +fn limit_table_update(ms: &mut MatchState, src: &BlockSrc<'_>, anchor: usize) { + let curr = src.index_of(anchor); + if curr > ms.next_to_update.wrapping_add(1024) { + let behind = curr.wrapping_sub(ms.next_to_update).wrapping_sub(1024); + ms.next_to_update = curr.wrapping_sub(behind.min(512)); + } +} + +/// `ldmMatchCandidate_t` +#[derive(Clone, Copy, Default)] +struct MatchCandidate { + split: usize, + hash: u32, + checksum: u32, + /// Where the bucket starts in the hash table. + bucket: usize, +} + +/// `ZSTD_ldm_generateSequences_internal`: the input is from `istart` to `iend` in the buffer of the current segment. +/// Returns how many bytes at its end are in no sequence. +fn generate_sequences_internal( + ldm_state: &mut LdmState, + raw_seq_store: &mut RawSeqStore, + params: &LdmParams, + bufs: &Buffers<'_>, + istart: usize, + iend: usize, +) -> Result { + let window = ldm_state.window; + let ext_dict = window.has_ext_dict(); + let min_match_length = params.min_match_length as usize; + let ents_per_bucket = 1usize << (params.bucket_size_log & 31); + let bucket_mask = bucket_mask(params); + let dict_limit = window.dict_limit; + let lowest_index = if ext_dict { + window.low_limit + } else { + dict_limit + }; + let prefix = Prefix { + buf: bufs.get(window.base_buf), + base: window.base.wrapping_sub(window.base_buf.start()), + }; + let buf = prefix.buf; + let dict_buf = bufs.get(window.dict_buf); + let dict_base = window.dict_base.wrapping_sub(window.dict_buf.start()); + let dict_start = dict_base.wrapping_add(lowest_index as usize); + let dict_end = dict_base.wrapping_add(dict_limit as usize); + let low_prefix = prefix.pos_of(dict_limit); + let ilimit = iend.saturating_sub(HASH_READ_SIZE); + let mut anchor = istart; + let mut ip = istart; + let mut splits = [0usize; LDM_BATCH_SIZE]; + let mut candidates = [MatchCandidate::default(); LDM_BATCH_SIZE]; + + if iend.checked_sub(istart).ok_or(BUG)? < min_match_length { + return Ok(iend - anchor); + } + + // Here and where it skips over a match, the C code calls `ZSTD_ldm_gear_reset`, which computes the hash of the + // `min_match_length` bytes before `ip` and drops it: the rolling hash goes on from what it was. + let mut hash_state = RollingHashState::new(params); + ip += min_match_length; + + while ip < ilimit { + let (hashed, num_splits) = hash_state.feed(buf.sub(ip..ilimit)?, &mut splits); + + for (candidate, &split) in candidates.iter_mut().zip(&splits).take(num_splits) { + let split = ip + split - min_match_length; + let xxhash = xxh64(buf.span(split, min_match_length)?); + let hash = xxhash as u32 & bucket_mask; + *candidate = MatchCandidate { + split, + hash, + checksum: (xxhash >> 32) as u32, + bucket: get_bucket(hash as usize, params.bucket_size_log), + }; + } + + for candidate in candidates.iter().take(num_splits) { + let MatchCandidate { + split, + hash, + checksum, + bucket, + } = *candidate; + let hash = hash as usize; + let mut forward_match_length = 0usize; + let mut backward_match_length = 0usize; + let mut best_match_length = 0usize; + let mut best_entry = None; + let new_entry = LdmEntry { + offset: prefix.index_of(split), + checksum, + }; + + // A split inside the previous match is indexed, but not searched. + if split < anchor { + insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; + continue; + } + + for cur in ldm_state.hash_table.span(bucket, ents_per_bucket)? { + if cur.checksum != checksum || cur.offset <= lowest_index { + continue; + } + let (cur_forward_match_length, cur_backward_match_length) = if ext_dict + && cur.offset < dict_limit + { + let p_match = dict_base.wrapping_add(cur.offset as usize); + let forward = + count_2segments(buf, split, dict_buf, p_match, iend, dict_end, low_prefix)?; + if forward < min_match_length { + continue; + } + let backward = + count_backwards_match(buf, split, anchor, dict_buf, p_match, dict_start)?; + (forward, backward) + } else { + let p_match = prefix.pos_of(cur.offset); + let forward = count(buf, split, buf, p_match, iend)?; + if forward < min_match_length { + continue; + } + let backward = if ext_dict { + count_backwards_match_2segments( + buf, split, anchor, p_match, low_prefix, dict_buf, dict_start, dict_end, + )? + } else { + count_backwards_match(buf, split, anchor, buf, p_match, low_prefix)? + }; + (forward, backward) + }; + let cur_total_match_length = cur_forward_match_length + cur_backward_match_length; + + if cur_total_match_length > best_match_length { + best_match_length = cur_total_match_length; + forward_match_length = cur_forward_match_length; + backward_match_length = cur_backward_match_length; + best_entry = Some(*cur); + } + } + + let Some(best_entry) = best_entry else { + insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; + continue; + }; + + let offset = new_entry.offset.wrapping_sub(best_entry.offset); + let m_length = forward_match_length + backward_match_length; + if raw_seq_store.size == raw_seq_store.capacity { + return Err(ErrorCode::DstSizeTooSmall); + } + raw_seq_store.seq.put( + raw_seq_store.size, + RawSeq { + offset, + lit_length: (split - backward_match_length - anchor) as u32, + match_length: m_length as u32, + }, + )?; + raw_seq_store.size += 1; + + // Only now, so that the position does not match itself. + insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; + + anchor = split + forward_match_length; + + // The match goes beyond what was hashed: what is left of it is skipped, and the other candidates dropped. + if anchor > ip + hashed { + ip = anchor - hashed; + break; + } + } + + ip += hashed; + } + + Ok(iend - anchor) +} + +/// `ZSTD_ldm_reduceTable` +fn reduce_table(table: &mut [LdmEntry], reducer_value: u32) { + for entry in table { + entry.offset = entry.offset.saturating_sub(reducer_value); + } +} + +/// `ZSTD_ldm_generateSequences`: adds to `sequences` the long distance matches of the input, which is the bytes from +/// `start` to `end` of the buffer of the current segment of the window, and which the window was updated with. +/// Stops early, without an error, at a chunk boundary where `sequences` is full. +pub(crate) fn generate_sequences( + ldm_state: &mut LdmState, + sequences: &mut RawSeqStore, + params: &LdmParams, + bufs: &Buffers<'_>, + start: usize, + end: usize, +) -> Result<()> { + const K_MAX_CHUNK_SIZE: usize = 1 << 20; + let max_dist = 1u32 << (params.window_log & 31); + let buf_start = ldm_state.window.base_buf.start(); + let mut leftover_size = 0usize; + let mut chunk_start = start; + + while chunk_start < end && sequences.size < sequences.capacity { + let chunk_end = end.min(chunk_start + K_MAX_CHUNK_SIZE); + let chunk_size = chunk_end - chunk_start; + let prev_size = sequences.size; + + if ldm_state + .window + .need_overflow_correction(buf_start + chunk_end) + { + let ldm_h_size = 1usize << (params.hash_log & 31); + let correction = + ldm_state + .window + .correct_overflow(0, max_dist, buf_start + chunk_start); + reduce_table(ldm_state.hash_table.sub_mut(..ldm_h_size)?, correction); + // An overflow correction invalidates the dictionary. + ldm_state.loaded_dict_end = 0; + } + // The whole chunk is searched with the window that its end has, so that no match is too far for any position + // of it, at the cost of some matches at its start. + if ldm_state.window.enforce_max_dist( + buf_start + chunk_end, + max_dist, + ldm_state.loaded_dict_end, + ) { + ldm_state.loaded_dict_end = 0; + } + let new_leftover_size = generate_sequences_internal( + ldm_state, + sequences, + params, + bufs, + chunk_start, + chunk_end, + )?; + // What the previous chunks left goes to the literals of the first sequence of this one. + if prev_size < sequences.size { + let first = sequences.seq.at_mut(prev_size)?; + first.lit_length = first.lit_length.wrapping_add(leftover_size as u32); + leftover_size = new_leftover_size; + } else { + leftover_size += chunk_size; + } + chunk_start = chunk_end; + } + Ok(()) +} + +/// `ZSTD_ldm_skipSequences`: moves `src_size` bytes forward in the sequences. A match of which less than `min_match` +/// is left becomes literals of the next sequence. +pub(crate) fn skip_sequences( + raw_seq_store: &mut RawSeqStore, + mut src_size: usize, + min_match: u32, +) -> Result<()> { + while src_size > 0 && raw_seq_store.pos < raw_seq_store.size { + let seq = raw_seq_store.seq.at_mut(raw_seq_store.pos)?; + if src_size <= seq.lit_length as usize { + seq.lit_length -= src_size as u32; + return Ok(()); + } + src_size -= seq.lit_length as usize; + seq.lit_length = 0; + if src_size < seq.match_length as usize { + seq.match_length -= src_size as u32; + if seq.match_length < min_match { + let match_length = seq.match_length; + if raw_seq_store.pos + 1 < raw_seq_store.size { + let next = raw_seq_store.seq.at_mut(raw_seq_store.pos + 1)?; + next.lit_length = next.lit_length.wrapping_add(match_length); + } + raw_seq_store.pos += 1; + } + return Ok(()); + } + src_size -= seq.match_length as usize; + seq.match_length = 0; + raw_seq_store.pos += 1; + } + Ok(()) +} + +/// `maybeSplitSequence`: the next sequence, cut to the `remaining` bytes of the block. What is cut off stays in the +/// store. If there is no match of `min_match` left in the block, the offset is 0. +fn maybe_split_sequence( + raw_seq_store: &mut RawSeqStore, + remaining: u32, + min_match: u32, +) -> Result { + let mut sequence = raw_seq_store.seq.at(raw_seq_store.pos)?; + let length = sequence.lit_length.wrapping_add(sequence.match_length); + if remaining >= length { + raw_seq_store.pos += 1; + return Ok(sequence); + } + if remaining <= sequence.lit_length { + sequence.offset = 0; + } else { + sequence.match_length = remaining - sequence.lit_length; + if sequence.match_length < min_match { + sequence.offset = 0; + } + } + skip_sequences(raw_seq_store, remaining as usize, min_match)?; + Ok(sequence) +} + +/// `ZSTD_ldm_skipRawSeqStoreBytes`: moves `nb_bytes` forward in the sequences, leaving them as they are. +pub(crate) fn skip_raw_seq_store_bytes( + raw_seq_store: &mut RawSeqStore, + nb_bytes: usize, +) -> Result<()> { + let mut curr_pos = raw_seq_store.pos_in_sequence.wrapping_add(nb_bytes) as u32; + while curr_pos != 0 && raw_seq_store.pos < raw_seq_store.size { + let curr_seq = raw_seq_store.seq.at(raw_seq_store.pos)?; + let length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); + if curr_pos >= length { + curr_pos -= length; + raw_seq_store.pos += 1; + } else { + raw_seq_store.pos_in_sequence = curr_pos as usize; + break; + } + } + if curr_pos == 0 || raw_seq_store.pos == raw_seq_store.size { + raw_seq_store.pos_in_sequence = 0; + } + Ok(()) +} + +/// `ZSTD_ldm_blockCompress`: finds the sequences of the block `src`, given the long distance matches, which it moves +/// forward in by the size of the block. Returns how many bytes at the end of the block are in no sequence. +pub(crate) fn block_compress( + raw_seq_store: &mut RawSeqStore, + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + use_row_match_finder: bool, + src: &BlockSrc<'_>, +) -> Result { + let strategy = ms.c_params.strat(); + let min_match = ms.c_params.min_match; + let dict_mode = ms.dict_mode(src.dms.as_ref()); + let iend = src.end; + let mut ip = src.start; + + // The optimal parser weighs the long distance matches against its own. + if strategy >= Strategy::BtOpt { + let block = BlockSrc { + ldm_seq_store: Some(&*raw_seq_store), + ..*src + }; + let last_ll_size = compress_block( + ms, + seq_store, + rep, + &block, + strategy, + use_row_match_finder, + dict_mode, + )?; + skip_raw_seq_store_bytes(raw_seq_store, iend.wrapping_sub(ip))?; + return Ok(last_ll_size); + } + + while raw_seq_store.pos < raw_seq_store.size && ip < iend { + let sequence = maybe_split_sequence(raw_seq_store, (iend - ip) as u32, min_match)?; + if sequence.offset == 0 { + break; + } + + limit_table_update(ms, src, ip); + fill_fast_tables(ms, src.prefix(), ip)?; + // The block compressor takes care of the literals, and what it leaves goes with the long distance match. + let literals = BlockSrc { + start: ip, + end: ip + sequence.lit_length as usize, + ..*src + }; + let new_lit_length = compress_block( + ms, + seq_store, + rep, + &literals, + strategy, + use_row_match_finder, + dict_mode, + )?; + ip += sequence.lit_length as usize; + let [rep0, rep1, _] = *rep; + *rep = [sequence.offset, rep0, rep1]; + seq_store.store_seq( + new_lit_length, + src.buf, + ip.checked_sub(new_lit_length).ok_or(BUG)?, + offset_to_offbase(sequence.offset), + sequence.match_length as usize, + )?; + ip += sequence.match_length as usize; + } + limit_table_update(ms, src, ip); + fill_fast_tables(ms, src.prefix(), ip)?; + let last = BlockSrc { start: ip, ..*src }; + compress_block( + ms, + seq_store, + rep, + &last, + strategy, + use_row_match_finder, + dict_mode, + ) +} diff --git a/src/zstd/compress/literals.rs b/src/zstd/compress/literals.rs new file mode 100644 index 000000000000..a942f06582f2 --- /dev/null +++ b/src/zstd/compress/literals.rs @@ -0,0 +1,147 @@ +//! `zstd_compress_literals.c`: the literals section of a block. + +use super::huf::{self, HUF_OPTIMAL_DEPTH_THRESHOLD, HufFlags, HufRepeat}; +use super::internal::{HufCTables, Strategy, min_gain}; +use crate::common::{HUF_SYMBOLVALUE_MAX, LIT_HUF_LOG, SymbolEncodingType}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, write_le16, write_le24, write_le32}; + +/// The header of a section of raw or run-length encoded literals. Returns its size. +fn write_short_header(dst: &mut [u8], ty: SymbolEncodingType, src_size: usize) -> Result { + let fl_size = 1 + usize::from(src_size > 31) + usize::from(src_size > 4095); + let ty = ty as u32; + let src_size = src_size as u32; + match fl_size { + 1 => dst.put(0, (ty + (src_size << 3)) as u8)?, + 2 => write_le16(dst, 0, (ty + (1 << 2) + (src_size << 4)) as u16)?, + _ => write_le24(dst, 0, ty + (3 << 2) + (src_size << 4))?, + } + Ok(fl_size) +} + +/// `ZSTD_noCompressLiterals` +pub(crate) fn no_compress_literals(dst: &mut [u8], src: &[u8]) -> Result { + let fl_size = 1 + usize::from(src.len() > 31) + usize::from(src.len() > 4095); + if src.len() + fl_size > dst.len() { + return Err(ErrorCode::DstSizeTooSmall); + } + write_short_header(dst, SymbolEncodingType::Basic, src.len())?; + dst.write_at(fl_size, src)?; + Ok(src.len() + fl_size) +} + +/// `ZSTD_compressRleLiteralsBlock`: `src` is one byte repeated, and `dst` has room for 4 bytes. +pub(crate) fn compress_rle_literals_block(dst: &mut [u8], src: &[u8]) -> Result { + let fl_size = write_short_header(dst, SymbolEncodingType::Rle, src.len())?; + dst.put(fl_size, src.at(0)?)?; + Ok(fl_size + 1) +} + +/// `ZSTD_minLiteralsToCompress`: fewer literals than this are not worth a try. +fn min_literals_to_compress(strategy: Strategy, huf_repeat: HufRepeat) -> usize { + let shift = (9 - strategy as i32).min(3); + if huf_repeat == HufRepeat::Valid { + 6 + } else { + 8 << shift + } +} + +/// `ZSTD_compressLiterals` +pub(crate) fn compress_literals( + dst: &mut [u8], + src: &[u8], + prev_huf: &HufCTables, + next_huf: &mut HufCTables, + strategy: Strategy, + disable_literal_compression: bool, + suspect_uncompressible: bool, +) -> Result { + let src_size = src.len(); + let lh_size = 3 + usize::from(src_size >= 1 << 10) + usize::from(src_size >= 16 << 10); + let mut single_stream = src_size < 256; + let mut h_type = SymbolEncodingType::Compressed; + + next_huf.clone_from(prev_huf); + + if disable_literal_compression { + return no_compress_literals(dst, src); + } + if src_size < min_literals_to_compress(strategy, prev_huf.repeat_mode) { + return no_compress_literals(dst, src); + } + + if dst.len() < lh_size + 1 { + return Err(ErrorCode::DstSizeTooSmall); + } + let c_lit_size = { + let mut repeat = prev_huf.repeat_mode; + let flags = HufFlags { + prefer_repeat: strategy < Strategy::Lazy && src_size <= 1024, + optimal_depth: strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD, + suspect_uncompressible, + }; + if repeat == HufRepeat::Valid && lh_size == 3 { + single_stream = true; + } + let huf_compress = if single_stream { + huf::compress_1x_repeat + } else { + huf::compress_4x_repeat + }; + let c_lit_size = huf_compress( + dst.sub_mut(lh_size..)?, + src, + HUF_SYMBOLVALUE_MAX, + LIT_HUF_LOG, + &mut next_huf.c_table, + &mut repeat, + flags, + ); + if repeat != HufRepeat::None { + h_type = SymbolEncodingType::Repeat; + } + c_lit_size + }; + + let min_gain = min_gain(src_size, strategy); + let c_lit_size = match c_lit_size { + Ok(size) if size != 0 && size < src_size.wrapping_sub(min_gain) => size, + _ => { + next_huf.clone_from(prev_huf); + return no_compress_literals(dst, src); + } + }; + if c_lit_size == 1 { + // With few literals, a size of 1 can also be that of a Huffman stream. + let first = src.at(0)?; + if src_size >= 8 || src.iter().all(|&b| b == first) { + next_huf.clone_from(prev_huf); + return compress_rle_literals_block(dst, src); + } + } + + if h_type == SymbolEncodingType::Compressed { + next_huf.repeat_mode = HufRepeat::Check; + } + + let h_type = h_type as u32; + let src_size = src_size as u32; + let c_lit = c_lit_size as u32; + match lh_size { + 3 => { + let lhc = h_type + (u32::from(!single_stream) << 2) + (src_size << 4) + (c_lit << 14); + write_le24(dst, 0, lhc)?; + } + 4 => { + let lhc = h_type + (2 << 2) + (src_size << 4) + (c_lit << 18); + write_le32(dst, 0, lhc)?; + } + _ => { + let lhc = h_type + (3 << 2) + (src_size << 4) + (c_lit << 22); + write_le32(dst, 0, lhc)?; + dst.put(4, (c_lit_size >> 10) as u8)?; + } + } + Ok(lh_size + c_lit_size) +} diff --git a/src/zstd/compress/mod.rs b/src/zstd/compress/mod.rs new file mode 100644 index 000000000000..fd2986adf53a --- /dev/null +++ b/src/zstd/compress/mod.rs @@ -0,0 +1,76 @@ +//! `lib/compress` + +pub(crate) mod block_entropy; +pub(crate) mod cctx; +pub(crate) mod double_fast; +pub(crate) mod fast; +pub(crate) mod fse; +pub(crate) mod hist; +pub(crate) mod huf; +pub(crate) mod internal; +pub(crate) mod lazy; +pub(crate) mod ldm; +pub(crate) mod literals; +pub(crate) mod mt; +pub(crate) mod opt; +pub(crate) mod params; +pub(crate) mod presplit; +pub(crate) mod sequences; +pub(crate) mod stream; +pub(crate) mod superblock; + +use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; + +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; + +/// `ZSTD_rowMatchFinderSupported` +pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { + strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 +} + +/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to +/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies +/// that support it look at. Returns how many bytes at the end of the block are in no sequence. +pub(crate) fn compress_block( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row_match_finder: bool, + dict_mode: DictMode, +) -> Result { + match strategy { + Strategy::Fast => match dict_mode { + DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), + DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), + DictMode::DictMatchState => { + fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::DFast => match dict_mode { + DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), + DictMode::ExtDict => { + double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { + lazy::compress_block_lazy( + ms, + seq_store, + rep, + src, + strategy, + use_row_match_finder && row_match_finder_supported(strategy), + dict_mode, + ) + } + Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { + opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) + } + } +} diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs new file mode 100644 index 000000000000..66e96566dc27 --- /dev/null +++ b/src/zstd/compress/mt.rs @@ -0,0 +1,829 @@ +//! `zstdmt_compress.c`: a frame made of sections that are compressed in parallel. +//! +//! The input is cut in the same sections as the C code cuts it in, each of which starts from the end of the previous +//! one as a prefix, so the frames are the same. What differs is how memory is shared. The C code has one round buffer +//! for the input, which workers read while it is being filled, one output buffer for each job, which is flushed while +//! it is being written, and a state that jobs lock in turn. Here a job owns its input, sends its output in pieces, and +//! hands that state over to the next job. + +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::thread::JoinHandle; + +use super::cctx::{ + CDict, DictContentType, FrameCompressor, Input, Span, reset_struct_table, reset_table, + write_last_empty_block, +}; +use super::internal::{ + BufId, Buffers, DictTableLoadMethod, LdmParams, LdmState, ParamSwitch, RawSeq, RawSeqStore, + Strategy, Window, +}; +use super::ldm; +use super::params::{ + CCtxParams, CParamMode, ZSTDMT_JOBLOG_MAX, ZSTDMT_JOBSIZE_MAX, ZSTDMT_JOBSIZE_MIN, + ZSTDMT_NBWORKERS_MAX, compress_bound, cycle_log, get_c_params_from_cctx_params, +}; +use super::stream::EndDirective; +use crate::common::xxhash::Xxh64; +use crate::common::{ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_UNKNOWN}; +use crate::error::{ErrorCode, Result}; +use crate::util::SliceExt; +use crate::{InBuffer, OutBuffer}; + +/// `SerialState`: what has to see the input in order. +struct SerialState { + ldm_params: LdmParams, + checksum_flag: bool, + ldm_state: LdmState, + xxh_state: Xxh64, + /// The input, where the C code has it in its round buffer: long distance matches reach further back than the + /// prefix of a job. + round_buff: Vec, +} + +impl SerialState { + fn new() -> Self { + Self { + ldm_params: NO_LDM_PARAMS, + checksum_flag: false, + ldm_state: LdmState::new(), + xxh_state: Xxh64::new(), + round_buff: Vec::new(), + } + } + + /// `ZSTDMT_serialState_reset` + fn reset(&mut self, params: &CCtxParams, round_buff_capacity: usize) -> Result<()> { + let mut ldm_params = params.ldm_params; + if ldm_params.enable_ldm == ParamSwitch::Enable { + ldm::adjust_parameters(&mut ldm_params, ¶ms.c_params); + } else { + ldm_params = NO_LDM_PARAMS; + } + self.checksum_flag = params.f_params.checksum_flag; + self.xxh_state = Xxh64::new(); + if ldm_params.enable_ldm == ParamSwitch::Enable { + let bucket_log = ldm_params.hash_log - ldm_params.bucket_size_log; + self.ldm_state.window = Window::new(); + reset_struct_table( + &mut self.ldm_state.hash_table, + 1 << ldm_params.hash_log, + true, + )?; + reset_table(&mut self.ldm_state.bucket_offsets, 1 << bucket_log, true)?; + self.ldm_state.loaded_dict_end = 0; + reset_table(&mut self.round_buff, round_buff_capacity, false)?; + } else { + self.ldm_state = LdmState::new(); + self.round_buff = Vec::new(); + } + self.ldm_params = ldm_params; + Ok(()) + } + + /// `ZSTDMT_serialState_genSequences`: `src` is the input of the next job, and `round_pos` where it goes in the + /// round buffer. + fn gen_sequences( + &mut self, + seq_store: &mut RawSeqStore, + src: &[u8], + round_pos: usize, + ) -> Result<()> { + if self.ldm_params.enable_ldm == ParamSwitch::Enable { + self.round_buff.write_at(round_pos, src)?; + (self.ldm_state.window).update( + BufId::InBuff.start() + round_pos, + BufId::InBuff, + src.len(), + false, + ); + ldm::generate_sequences( + &mut self.ldm_state, + seq_store, + &self.ldm_params, + &Buffers { + in_buff: &self.round_buff, + user: &[], + dict: &[], + }, + round_pos, + round_pos + src.len(), + )?; + } + if self.checksum_flag && !src.is_empty() { + self.xxh_state.update(src); + } + Ok(()) + } +} + +const NO_LDM_PARAMS: LdmParams = LdmParams { + enable_ldm: ParamSwitch::Auto, + hash_log: 0, + bucket_size_log: 0, + min_match_length: 0, + hash_rate_log: 0, + window_log: 0, +}; + +/// What a job tells about itself. +enum Progress { + /// Some more output, and how much of the input is consumed by now. + Output { data: Vec, consumed: usize }, + /// The job stops here. + Failed(ErrorCode), +} + +/// `ZSTDMT_jobDescription`, as far as the worker is concerned. +struct Job { + /// The prefix, then the input. + data: Vec, + prefix_size: usize, + round_pos: usize, + first_job: bool, + last_job: bool, + params: CCtxParams, + cdict: Option>, + full_frame_size: u64, + /// How many long distance matches to make room for. + max_nb_seq: usize, + /// From the previous job. + serial_in: Receiver>, + /// To the next one. + serial_out: Sender>, + progress: Sender, +} + +/// What a worker keeps from one job to the next. +struct WorkerState { + cctx: FrameCompressor, + seq: Vec, + dst: Vec, +} + +const CHUNK_SIZE: usize = 4 * ZSTD_BLOCKSIZE_MAX; + +/// `ZSTDMT_compressionJob` +fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) -> Result<()> { + let mut job_params = job.params; + let src_size = job.data.len() - job.prefix_size; + let src = job.data.sub(job.prefix_size..)?; + + // Whatever happens, the next job waits for this. + let mut raw_seq_store = RawSeqStore { + seq: core::mem::take(&mut state.seq), + pos: 0, + pos_in_sequence: 0, + size: 0, + capacity: job.max_nb_seq, + }; + let mut serial = job.serial_in.recv().map_err(|_| ErrorCode::Generic)?; + let generated = if cancelled.load(Ordering::Relaxed) { + Err(ErrorCode::Generic) + } else { + reset_struct_table(&mut raw_seq_store.seq, job.max_nb_seq, false) + .and_then(|()| serial.gen_sequences(&mut raw_seq_store, src, job.round_pos)) + }; + // The receiver is gone if the frame is given up. + let _ = job.serial_out.send(serial); + generated?; + + reset_table(&mut state.dst, compress_bound(CHUNK_SIZE) + 1024, false)?; + + // Only the first job writes the frame header, and the checksum is not that of one job. + if !job.first_job { + job_params.f_params.checksum_flag = false; + } + job_params.ldm_params.enable_ldm = ParamSwitch::Disable; + job_params.nb_workers = 0; + + let cdict = job.cdict.as_deref(); + let input = Input { + bufs: Buffers { + in_buff: &[], + user: &job.data, + dict: cdict.map_or(&[], CDict::dict_content), + }, + cdict, + }; + let cctx = &mut state.cctx; + if cdict.is_some() { + cctx.begin( + &input, + None, + DictContentType::Auto, + DictTableLoadMethod::Fast, + &job_params, + job.full_frame_size, + false, + )?; + } else { + let pledged_src_size = if job.first_job { + job.full_frame_size + } else { + src_size as u64 + }; + job_params.force_window = !job.first_job; + cctx.begin( + &input, + Some(Span { + buf: BufId::User, + start: 0, + end: job.prefix_size, + }), + DictContentType::RawContent, + DictTableLoadMethod::Fast, + &job_params, + pledged_src_size, + false, + )?; + } + + raw_seq_store.seq.truncate(raw_seq_store.size); + cctx.reference_external_sequences(raw_seq_store.seq); + + if !job.first_job { + // This frame header is not for the output. + let nothing = Span { + buf: BufId::User, + start: job.prefix_size, + end: job.prefix_size, + }; + cctx.compress_continue(&mut state.dst, &input, nothing)?; + cctx.invalidate_rep_codes(); + } + + let nb_chunks = src_size.div_ceil(CHUNK_SIZE); + let mut ip = job.prefix_size; + let send = |dst: &[u8], consumed: usize| -> Result<()> { + let mut data = Vec::new(); + data.try_reserve_exact(dst.len()) + .map_err(|_| ErrorCode::MemoryAllocation)?; + data.extend_from_slice(dst); + let _ = job.progress.send(Progress::Output { data, consumed }); + Ok(()) + }; + for chunk_nb in 1..nb_chunks { + if cancelled.load(Ordering::Relaxed) { + return Err(ErrorCode::Generic); + } + let chunk = Span { + buf: BufId::User, + start: ip, + end: ip + CHUNK_SIZE, + }; + let c_size = cctx.compress_continue(&mut state.dst, &input, chunk)?; + ip += CHUNK_SIZE; + send(state.dst.sub(..c_size)?, CHUNK_SIZE * chunk_nb)?; + } + let mut last_c_block_size = 0; + if nb_chunks > 0 || job.last_job { + let chunk = Span { + buf: BufId::User, + start: ip, + end: job.data.len(), + }; + last_c_block_size = if job.last_job { + cctx.compress_end(&mut state.dst, &input, chunk)? + } else { + cctx.compress_continue(&mut state.dst, &input, chunk)? + }; + } + send(state.dst.sub(..last_c_block_size)?, src_size) +} + +fn worker_main(jobs: &Receiver, cancelled: &AtomicBool) { + let mut state = WorkerState { + cctx: FrameCompressor::new(), + seq: Vec::new(), + dst: Vec::new(), + }; + while let Ok(job) = jobs.recv() { + if let Err(code) = compression_job(&mut state, &job, cancelled) { + let _ = job.progress.send(Progress::Failed(code)); + } + state.seq = state.cctx.take_external_sequences(); + } +} + +struct Worker { + jobs: Sender, + thread: JoinHandle<()>, +} + +/// `ZSTDMT_jobDescription`, as far as flushing is concerned. +struct PendingJob { + /// `None` for a job that needs no worker. + progress: Option>, + /// Output that is received and not flushed. The first piece is flushed up to `flushed`. + output: VecDeque>, + flushed: usize, + /// How many bytes there are in `output` beyond `flushed`. + to_flush: usize, + /// How many bytes were received in all. + c_size: usize, + consumed: usize, + src_size: usize, + frame_checksum_needed: bool, + error: Option, +} + +impl PendingJob { + fn take(&mut self, progress: Progress) { + match progress { + Progress::Output { data, consumed } => { + self.c_size += data.len(); + self.to_flush += data.len(); + self.consumed = consumed; + if !data.is_empty() { + self.output.push_back(data); + } + } + Progress::Failed(code) => { + self.error = Some(code); + self.consumed = self.src_size; + self.progress = None; + } + } + } + + /// `consumed == src.size` is how the C code tells, which does not do for a job that has no input. + fn is_complete(&self) -> bool { + self.progress.is_none() + } + + /// Takes what the worker has sent. If `wait` is set and there is nothing to flush, waits for some. + fn poll(&mut self, wait: bool) { + while let Some(receiver) = &self.progress { + let received = if wait && self.to_flush == 0 { + receiver.recv().ok() + } else { + match receiver.try_recv() { + Ok(progress) => Some(progress), + Err(std::sync::mpsc::TryRecvError::Empty) => return, + Err(std::sync::mpsc::TryRecvError::Disconnected) => None, + } + }; + match received { + Some(progress) => { + let last = matches!(&progress, Progress::Output { consumed, .. } if *consumed == self.src_size); + self.take(progress); + if last { + self.progress = None; + } + } + // The worker is gone. + None => self.take(Progress::Failed(ErrorCode::Generic)), + } + } + } +} + +/// `ZSTDMT_CCtx` +pub(crate) struct MtCtx { + workers: Vec, + cancelled: Arc, + /// The jobs from `doneJobID` to `nextJobID`. + jobs: VecDeque, + params: CCtxParams, + target_section_size: usize, + target_prefix_size: usize, + /// `inBuff`: the prefix of the next job, then as much of its input as there is. + in_buff: Vec, + in_buff_prefix_size: usize, + /// `inBuff.buffer.start != NULL`: there is room for the input of the next job. + in_buff_ready: bool, + round_buff_capacity: usize, + round_buff_pos: usize, + /// Where the last job that was created leaves the state. + serial: Option>>, + next_job_id: u32, + frame_ended: bool, + all_jobs_completed: bool, + frame_content_size: u64, + cdict: Option>, +} + +/// `ZSTDMT_computeTargetJobLog` +fn compute_target_job_log(params: &CCtxParams) -> u32 { + let job_log = if params.ldm_params.enable_ldm == ParamSwitch::Enable { + // Long distance matching makes up for the matches that cutting loses, so sections can be smaller. + 21.max(cycle_log(params.c_params.chain_log, params.c_params.strat()) + 3) + } else { + 20.max(params.c_params.window_log + 2) + }; + job_log.min(ZSTDMT_JOBLOG_MAX) +} + +/// `ZSTDMT_overlapLog` +fn overlap_log(ovlog: i32, strat: Strategy) -> i32 { + if ovlog != 0 { + return ovlog; + } + match strat { + Strategy::BtUltra2 => 9, + Strategy::BtUltra | Strategy::BtOpt => 8, + Strategy::BtLazy2 | Strategy::Lazy2 => 7, + Strategy::Lazy | Strategy::Greedy | Strategy::DFast | Strategy::Fast => 6, + } +} + +/// `ZSTDMT_computeOverlapSize` +fn compute_overlap_size(params: &CCtxParams) -> usize { + let overlap_r_log = 9 - overlap_log(params.overlap_log, params.c_params.strat()); + let mut ov_log = if overlap_r_log >= 8 { + 0 + } else { + params.c_params.window_log as i32 - overlap_r_log + }; + if params.ldm_params.enable_ldm == ParamSwitch::Enable { + ov_log = (params.c_params.window_log).min(compute_target_job_log(params) - 2) as i32 + - overlap_r_log; + } + if ov_log == 0 { 0 } else { 1 << ov_log } +} + +impl MtCtx { + /// `ZSTDMT_createCCtx_advanced` + pub(crate) fn new(nb_workers: u32) -> Result { + let mut mtctx = Self { + workers: Vec::new(), + cancelled: Arc::new(AtomicBool::new(false)), + jobs: VecDeque::new(), + params: CCtxParams::new(0), + target_section_size: 0, + target_prefix_size: 0, + in_buff: Vec::new(), + in_buff_prefix_size: 0, + in_buff_ready: false, + round_buff_capacity: 0, + round_buff_pos: 0, + serial: None, + next_job_id: 0, + frame_ended: false, + all_jobs_completed: true, + frame_content_size: 0, + cdict: None, + }; + mtctx.resize(nb_workers.clamp(1, ZSTDMT_NBWORKERS_MAX as u32) as usize)?; + Ok(mtctx) + } + + /// `ZSTDMT_resize`: no job is under way. + fn resize(&mut self, nb_workers: usize) -> Result<()> { + for worker in self.workers.drain(nb_workers.min(self.workers.len())..) { + drop(worker.jobs); + let _ = worker.thread.join(); + } + while self.workers.len() < nb_workers { + let (jobs, receiver) = channel(); + let cancelled = Arc::clone(&self.cancelled); + let thread = std::thread::Builder::new() + .name("zstd".into()) + .stack_size(2 << 20) + .spawn(move || worker_main(&receiver, &cancelled)) + .map_err(|_| ErrorCode::MemoryAllocation)?; + self.workers.push(Worker { jobs, thread }); + } + self.params.nb_workers = nb_workers as i32; + Ok(()) + } + + /// `ZSTDMT_waitForAllJobsCompleted` and `ZSTDMT_releaseAllJobResources`, for a frame that is given up. + fn abandon_jobs(&mut self) { + self.cancelled.store(true, Ordering::Relaxed); + for job in &mut self.jobs { + while !job.is_complete() { + job.to_flush = 0; + job.output.clear(); + job.poll(true); + } + } + self.cancelled.store(false, Ordering::Relaxed); + self.jobs.clear(); + self.in_buff = Vec::new(); + self.in_buff_prefix_size = 0; + self.in_buff_ready = false; + self.all_jobs_completed = true; + } + + /// The state, once all the jobs that were created have had it. + fn take_serial(&mut self) -> Box { + (self.serial.take()) + .and_then(|serial| serial.recv().ok()) + .unwrap_or_else(|| Box::new(SerialState::new())) + } + + /// `ZSTDMT_updateCParams_whileCompressing`: for the jobs to come. + pub(crate) fn update_c_params_while_compressing(&mut self, cctx_params: &CCtxParams) { + // The window is in the frame header. + let saved_wlog = self.params.c_params.window_log; + self.params.compression_level = cctx_params.compression_level; + let mut c_params = get_c_params_from_cctx_params( + cctx_params, + ZSTD_CONTENTSIZE_UNKNOWN, + 0, + CParamMode::NoAttachDict, + ); + c_params.window_log = saved_wlog; + self.params.c_params = c_params; + } + + /// `ZSTDMT_initCStream_internal` + pub(crate) fn init_c_stream_internal( + &mut self, + cdict: Option>, + mut params: CCtxParams, + pledged_src_size: u64, + ) -> Result<()> { + if !self.all_jobs_completed { + self.abandon_jobs(); + } + if params.nb_workers != self.params.nb_workers { + self.resize(params.nb_workers as usize)?; + } + + if params.job_size != 0 && params.job_size < ZSTDMT_JOBSIZE_MIN as usize { + params.job_size = ZSTDMT_JOBSIZE_MIN as usize; + } + params.job_size = params.job_size.min(ZSTDMT_JOBSIZE_MAX as usize); + + self.params = params; + self.frame_content_size = pledged_src_size; + self.target_prefix_size = compute_overlap_size(¶ms); + self.target_section_size = params.job_size; + if self.target_section_size == 0 { + self.target_section_size = 1 << compute_target_job_log(¶ms); + } + self.target_section_size = self.target_section_size.max(self.target_prefix_size); + { + let window_size = if params.ldm_params.enable_ldm == ParamSwitch::Enable { + 1usize << params.c_params.window_log + } else { + 0 + }; + let nb_slack_buffers = 2 + usize::from(self.target_prefix_size > 0); + let slack_size = self.target_section_size * nb_slack_buffers; + let nb_workers = params.nb_workers.max(1) as usize; + let sections_size = self.target_section_size * nb_workers; + let capacity = window_size.max(sections_size) + slack_size; + self.round_buff_capacity = self.round_buff_capacity.max(capacity); + } + self.round_buff_pos = 0; + self.in_buff = Vec::new(); + self.in_buff_prefix_size = 0; + self.in_buff_ready = false; + self.next_job_id = 0; + self.frame_ended = false; + self.all_jobs_completed = false; + self.cdict = cdict; + + let mut serial = self.take_serial(); + serial.reset(¶ms, self.round_buff_capacity)?; + let (sender, receiver) = channel(); + let _ = sender.send(serial); + self.serial = Some(receiver); + Ok(()) + } + + /// The most jobs at a time, which bounds memory as the size of the round buffer does in the C code. + fn max_jobs(&self) -> usize { + self.workers.len() + 2 + } + + /// `ZSTDMT_createCompressionJob`: of the input there is in `in_buff`. + fn create_compression_job(&mut self, end_op: EndDirective) -> Result<()> { + let end_frame = end_op == EndDirective::End; + if self.jobs.len() >= self.max_jobs() { + // Later. + return Ok(()); + } + + let data = core::mem::take(&mut self.in_buff); + let prefix_size = self.in_buff_prefix_size; + let src_size = data.len() - prefix_size; + let round_pos = self.round_buff_pos; + let first_job = self.next_job_id == 0; + let params = self.params; + let frame_checksum_needed = params.f_params.checksum_flag && end_frame && !first_job; + + self.round_buff_pos += src_size; + self.in_buff_ready = false; + self.in_buff_prefix_size = 0; + if !end_frame { + let new_prefix_size = src_size.min(self.target_prefix_size); + self.in_buff + .try_reserve_exact(new_prefix_size + self.target_section_size) + .map_err(|_| ErrorCode::MemoryAllocation)?; + self.in_buff + .extend_from_slice(data.sub(data.len() - new_prefix_size..)?); + self.in_buff_prefix_size = new_prefix_size; + } else { + self.frame_ended = true; + if first_job { + // The only job writes the checksum itself. + self.params.f_params.checksum_flag = false; + } + } + + let mut pending = PendingJob { + progress: None, + output: VecDeque::new(), + flushed: 0, + to_flush: 0, + c_size: 0, + consumed: 0, + src_size, + frame_checksum_needed, + error: None, + }; + + if src_size == 0 && !first_job { + // `ZSTDMT_writeLastEmptyBlock`: all there is left to do is to end the frame. + let mut block = [0u8; 3]; + let size = write_last_empty_block(&mut block)?; + pending.take(Progress::Output { + data: block.sub(..size)?.to_vec(), + consumed: 0, + }); + self.jobs.push_back(pending); + self.next_job_id += 1; + return Ok(()); + } + + let (progress, progress_receiver) = channel(); + let (serial_out, serial_next) = channel(); + let serial_in = self.serial.replace(serial_next).ok_or(ErrorCode::Generic)?; + pending.progress = Some(progress_receiver); + let mut ldm_params = params.ldm_params; + if ldm_params.enable_ldm == ParamSwitch::Enable { + ldm::adjust_parameters(&mut ldm_params, ¶ms.c_params); + } + let job = Job { + data, + prefix_size, + round_pos, + first_job, + last_job: end_frame, + params, + cdict: if first_job { self.cdict.clone() } else { None }, + full_frame_size: self.frame_content_size, + max_nb_seq: ldm::get_max_nb_seq(ldm_params, self.target_section_size), + serial_in, + serial_out, + progress, + }; + let worker = (self.workers) + .get(self.next_job_id as usize % self.workers.len().max(1)) + .ok_or(ErrorCode::Generic)?; + worker.jobs.send(job).map_err(|_| ErrorCode::Generic)?; + self.jobs.push_back(pending); + self.next_job_id += 1; + Ok(()) + } + + /// `ZSTDMT_flushProduced`: returns the least number of bytes that are left to flush. + fn flush_produced( + &mut self, + output: &mut OutBuffer<'_>, + block_to_flush: bool, + end: EndDirective, + ) -> Result { + if let Some(job) = self.jobs.front_mut() { + job.poll(block_to_flush); + if let Some(code) = job.error { + self.abandon_jobs(); + return Err(code); + } + } + if self + .jobs + .front() + .is_some_and(|job| job.is_complete() && job.frame_checksum_needed) + { + let serial = self.take_serial(); + let checksum = serial.xxh_state.digest() as u32; + let (sender, receiver) = channel(); + let _ = sender.send(serial); + self.serial = Some(receiver); + if let Some(job) = self.jobs.front_mut() { + job.frame_checksum_needed = false; + let consumed = job.consumed; + job.take(Progress::Output { + data: checksum.to_le_bytes().to_vec(), + consumed, + }); + } + } + if let Some(job) = self.jobs.front_mut() { + while let Some(piece) = job.output.front() { + let room = output.dst.sub_mut(output.pos..)?; + let rest = piece.sub(job.flushed..)?; + let n = rest.len().min(room.len()); + room.write_at(0, rest.sub(..n)?)?; + output.pos += n; + job.to_flush -= n; + if n < rest.len() { + job.flushed += n; + break; + } + job.flushed = 0; + job.output.pop_front(); + } + let (to_flush, complete) = (job.to_flush, job.is_complete()); + if complete && to_flush == 0 { + self.jobs.pop_front(); + } + if to_flush > 0 { + return Ok(to_flush); + } + if !complete { + return Ok(1); + } + } + if !self.jobs.is_empty() { + return Ok(1); + } + if self.in_buff.len() > self.in_buff_prefix_size { + return Ok(1); + } + self.all_jobs_completed = self.frame_ended; + if end == EndDirective::End { + return Ok(usize::from(!self.frame_ended)); + } + Ok(0) + } + + /// `ZSTDMT_tryGetInputRange`: whether the next job can start taking input. + fn try_get_input_range(&mut self) -> Result { + if self.jobs.len() >= self.max_jobs() { + return Ok(false); + } + let space_left = self.round_buff_capacity - self.round_buff_pos; + if space_left < self.target_section_size { + // The C code moves the prefix to the start of the round buffer. + self.round_buff_pos = self.in_buff_prefix_size; + } + self.in_buff + .try_reserve_exact(self.target_section_size) + .map_err(|_| ErrorCode::MemoryAllocation)?; + self.in_buff_ready = true; + Ok(true) + } + + /// `ZSTDMT_compressStream_generic` + pub(crate) fn compress_stream_generic( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + mut end_op: EndDirective, + ) -> Result { + let mut forward_input_progress = false; + + if self.frame_ended && end_op == EndDirective::Continue { + // The frame is complete and still to be flushed. + return Err(ErrorCode::StageWrong); + } + + if input.src.len() > input.pos && (self.in_buff_ready || self.try_get_input_range()?) { + let filled = self.in_buff.len() - self.in_buff_prefix_size; + let to_load = (input.src.len() - input.pos).min(self.target_section_size - filled); + self.in_buff + .extend_from_slice(input.src.span(input.pos, to_load)?); + input.pos += to_load; + forward_input_progress = to_load > 0; + } + + if input.pos < input.src.len() && end_op == EndDirective::End { + // Not before all the input is consumed. + end_op = EndDirective::Flush; + } + + let filled = self.in_buff.len() - self.in_buff_prefix_size; + if filled >= self.target_section_size + || (end_op != EndDirective::Continue && filled > 0) + || (end_op == EndDirective::End && !self.frame_ended) + { + self.create_compression_job(end_op)?; + } + + // Waits for a job to output something if this call did nothing else. + let remaining_to_flush = self.flush_produced(output, !forward_input_progress, end_op)?; + if input.pos < input.src.len() { + return Ok(remaining_to_flush.max(1)); + } + Ok(remaining_to_flush) + } +} + +impl Drop for MtCtx { + fn drop(&mut self) { + self.cancelled.store(true, Ordering::Relaxed); + for worker in self.workers.drain(..) { + drop(worker.jobs); + let _ = worker.thread.join(); + } + } +} diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs new file mode 100644 index 000000000000..a7a6022a3a45 --- /dev/null +++ b/src/zstd/compress/opt.rs @@ -0,0 +1,1721 @@ +//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. + +use core::mem; + +use super::hist; +use super::huf::HufRepeat; +use super::internal::{ + BlockSrc, DictMode, EntropyCTables, Match, MatchState, OptPrice, OptState, Optimal, + ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, ZSTD_OPT_NUM, count, + count_2segments, hash_ptr, hash3_ptr, index_overlap_check, ll_code, ml_code, new_rep, + offset_to_offbase, +}; +use crate::common::{ + LL_BITS, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, +}; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +/// How much a literal adds to its frequency, so that frequencies adapt fast to new statistics. +const ZSTD_LITFREQ_ADD: u32 = 2; +const ZSTD_MAX_PRICE: i32 = 1 << 30; +/// A first block of at most this size is priced by fixed rules, not by statistics. +const ZSTD_PREDEF_THRESHOLD: usize = 8; + +const MAX_LIT: u32 = 255; + +// ── prices ──────────────────────────────────────────────────────────────── + +/// Prices are in units of `1 / BITCOST_MULTIPLIER` of a bit. +const BITCOST_ACCURACY: u32 = 8; +const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY; + +/// `WEIGHT`: approximate for `btopt`, accurate for `btultra`. +#[inline(always)] +const fn weight(stat: u32, opt_level: i32) -> u32 { + if opt_level != 0 { + frac_weight(stat) + } else { + bit_weight(stat) + } +} + +/// `ZSTD_bitWeight`: the cost of a statistic, in whole bits. +#[inline(always)] +const fn bit_weight(stat: u32) -> u32 { + highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER +} + +/// `ZSTD_fracWeight`: the cost of a statistic, with a fraction of a bit that is interpolated linearly. +#[inline(always)] +const fn frac_weight(raw_stat: u32) -> u32 { + let stat = raw_stat.wrapping_add(1); + let hb = highbit32(stat); + let b_weight = hb * BITCOST_MULTIPLIER; + // From 1 to 2, in fixed point. + let f_weight = (stat << BITCOST_ACCURACY) >> hb; + b_weight.wrapping_add(f_weight) +} + +/// `ZSTD_compressedLiterals` +#[inline(always)] +fn compressed_literals(opt_ptr: &OptState) -> bool { + opt_ptr.literal_compression_mode != ParamSwitch::Disable +} + +/// `ZSTD_setBasePrices` +#[inline(always)] +fn set_base_prices(opt_ptr: &mut OptState, opt_level: i32) { + if compressed_literals(opt_ptr) { + opt_ptr.lit_sum_base_price = weight(opt_ptr.lit_sum, opt_level); + } + opt_ptr.lit_length_sum_base_price = weight(opt_ptr.lit_length_sum, opt_level); + opt_ptr.match_length_sum_base_price = weight(opt_ptr.match_length_sum, opt_level); + opt_ptr.off_code_sum_base_price = weight(opt_ptr.off_code_sum, opt_level); +} + +/// `sum_u32` +fn sum_u32(table: &[u32]) -> u32 { + table.iter().fold(0, |total, &elt| total.wrapping_add(elt)) +} + +/// `base_directive_e`: the least a scaled down statistic can be. +#[derive(Clone, Copy, PartialEq, Eq)] +enum BaseDirective { + /// 0, if it is 0 already. + Base0Possible, + Base1Guaranteed, +} + +/// `ZSTD_downscaleStats`: returns the new sum of the elements. +fn downscale_stats( + table: &mut [u32], + last_elt_index: u32, + shift: u32, + base1: BaseDirective, +) -> Result { + let mut sum = 0u32; + for elt in table.sub_mut(..=last_elt_index as usize)? { + let base = u32::from(base1 == BaseDirective::Base1Guaranteed || *elt > 0); + let new_stat = base.wrapping_add(*elt >> (shift & 31)); + sum = sum.wrapping_add(new_stat); + *elt = new_stat; + } + Ok(sum) +} + +/// `ZSTD_scaleStats`: scales all the elements down if their sum is too large. Returns the sum that results. +fn scale_stats(table: &mut [u32], last_elt_index: u32, log_target: u32) -> Result { + let prevsum = sum_u32(table.sub(..=last_elt_index as usize)?); + let factor = prevsum >> log_target; + if factor <= 1 { + return Ok(prevsum); + } + downscale_stats( + table, + last_elt_index, + highbit32(factor), + BaseDirective::Base1Guaranteed, + ) +} + +/// Frequencies, out of about `1 << scale_log`, that make each symbol cost the bits `bit_cost` tells. Returns their sum. +fn freqs_from_bit_costs( + freqs: &mut [u32], + last_elt_index: u32, + scale_log: u32, + bit_cost: impl Fn(u32) -> u32, +) -> Result { + let mut sum = 0u32; + for (symbol, freq) in (0u32..).zip(freqs.sub_mut(..=last_elt_index as usize)?) { + let bit_cost = bit_cost(symbol); + // At least 1, to have a cost. + *freq = if bit_cost != 0 { + 1 << (scale_log.wrapping_sub(bit_cost) & 31) + } else { + 1 + }; + sum = sum.wrapping_add(*freq); + } + Ok(sum) +} + +/// `ZSTD_rescaleFreqs`: for the first block, which no statistics tell, starts them from the tables of the dictionary +/// if there is one, else from the literals of `src` and from fixed values for the sequences. For the next ones, scales +/// down what the blocks before have accumulated. +fn rescale_freqs( + opt_ptr: &mut OptState, + symbol_costs: &EntropyCTables, + src: &[u8], + opt_level: i32, +) -> Result<()> { + #[rustfmt::skip] + const BASE_LL_FREQS: [u32; MAX_LL as usize + 1] = [ + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, + ]; + #[rustfmt::skip] + const BASE_OFC_FREQS: [u32; MAX_OFF as usize + 1] = [ + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + ]; + + let compressed_literals = compressed_literals(opt_ptr); + opt_ptr.price_type = OptPrice::Dynamic; + + if opt_ptr.lit_length_sum == 0 { + if src.len() <= ZSTD_PREDEF_THRESHOLD { + opt_ptr.price_type = OptPrice::Predef; + } + + if symbol_costs.huf.repeat_mode == HufRepeat::Valid { + // A Huffman table that has all the values is presumed to come from a dictionary. + opt_ptr.price_type = OptPrice::Dynamic; + + if compressed_literals { + let huf = &symbol_costs.huf.c_table; + opt_ptr.lit_sum = + freqs_from_bit_costs(&mut opt_ptr.lit_freq, MAX_LIT, 11, |lit| { + huf.get_nb_bits(lit) + })?; + } + let fse = &symbol_costs.fse; + opt_ptr.lit_length_sum = + freqs_from_bit_costs(&mut opt_ptr.lit_length_freq, MAX_LL, 10, |ll| { + fse.litlength_c_table.get_max_nb_bits(ll) + })?; + opt_ptr.match_length_sum = + freqs_from_bit_costs(&mut opt_ptr.match_length_freq, MAX_ML, 10, |ml| { + fse.matchlength_c_table.get_max_nb_bits(ml) + })?; + opt_ptr.off_code_sum = + freqs_from_bit_costs(&mut opt_ptr.off_code_freq, MAX_OFF, 10, |of| { + fse.offcode_c_table.get_max_nb_bits(of) + })?; + } else { + if compressed_literals { + let mut lit = MAX_LIT; + hist::count_simple(&mut opt_ptr.lit_freq, &mut lit, src)?; + opt_ptr.lit_sum = downscale_stats( + &mut opt_ptr.lit_freq, + MAX_LIT, + 8, + BaseDirective::Base0Possible, + )?; + } + + opt_ptr.lit_length_freq.write_at(0, &BASE_LL_FREQS)?; + opt_ptr.lit_length_sum = sum_u32(&BASE_LL_FREQS); + + opt_ptr + .match_length_freq + .sub_mut(..=MAX_ML as usize)? + .fill(1); + opt_ptr.match_length_sum = MAX_ML + 1; + + opt_ptr.off_code_freq.write_at(0, &BASE_OFC_FREQS)?; + opt_ptr.off_code_sum = sum_u32(&BASE_OFC_FREQS); + } + } else { + if compressed_literals { + opt_ptr.lit_sum = scale_stats(&mut opt_ptr.lit_freq, MAX_LIT, 12)?; + } + opt_ptr.lit_length_sum = scale_stats(&mut opt_ptr.lit_length_freq, MAX_LL, 11)?; + opt_ptr.match_length_sum = scale_stats(&mut opt_ptr.match_length_freq, MAX_ML, 11)?; + opt_ptr.off_code_sum = scale_stats(&mut opt_ptr.off_code_freq, MAX_OFF, 11)?; + } + + set_base_prices(opt_ptr, opt_level); + Ok(()) +} + +/// `ZSTD_rawLiteralsCost`: the price of `literals`, without that of their length. +#[inline(always)] +fn raw_literals_cost(literals: &[u8], opt_ptr: &OptState, opt_level: i32) -> Result { + let lit_length = literals.len() as u32; + if lit_length == 0 { + return Ok(0); + } + + if !compressed_literals(opt_ptr) { + // 8 bits per literal. + return Ok((lit_length << 3).wrapping_mul(BITCOST_MULTIPLIER)); + } + + if opt_ptr.price_type == OptPrice::Predef { + // 6 bits per literal. + return Ok(lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER)); + } + + let mut price = opt_ptr.lit_sum_base_price.wrapping_mul(lit_length); + let lit_price_max = opt_ptr.lit_sum_base_price.wrapping_sub(BITCOST_MULTIPLIER); + for &literal in literals { + let lit_price = weight(opt_ptr.lit_freq.at(usize::from(literal))?, opt_level); + price = price.wrapping_sub(lit_price.min(lit_price_max)); + } + Ok(price) +} + +/// `ZSTD_litLengthPrice`: the price of the symbol for a length of literals. +#[inline(always)] +fn lit_length_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + if opt_ptr.price_type == OptPrice::Predef { + return Ok(weight(lit_length, opt_level)); + } + + // The format has no code for the length of a whole block of the largest size, which is then all literals. It is + // made to cost 1 bit more than the length before it. + let (lit_length, extra) = if lit_length == ZSTD_BLOCKSIZE_MAX as u32 { + (ZSTD_BLOCKSIZE_MAX as u32 - 1, BITCOST_MULTIPLIER) + } else { + (lit_length, 0) + }; + + let ll_code = ll_code(lit_length) as usize; + Ok(extra + .wrapping_add(u32::from(LL_BITS.at(ll_code)?) * BITCOST_MULTIPLIER) + .wrapping_add(opt_ptr.lit_length_sum_base_price) + .wrapping_sub(weight(opt_ptr.lit_length_freq.at(ll_code)?, opt_level))) +} + +/// `ZSTD_getMatchPrice`: the price of the offset and of the match length of a sequence. `off_base` is as in +/// [`SeqStore::store_seq`]. Below level 2, it favours small offsets, which are faster to decompress. +#[inline(always)] +fn get_match_price( + off_base: u32, + match_length: u32, + opt_ptr: &OptState, + opt_level: i32, +) -> Result { + let off_code = highbit32(off_base); + let ml_base = match_length.wrapping_sub(MINMATCH); + + if opt_ptr.price_type == OptPrice::Predef { + return Ok(weight(ml_base, opt_level).wrapping_add((16 + off_code) * BITCOST_MULTIPLIER)); + } + + let off_code_weight = weight(opt_ptr.off_code_freq.at(off_code as usize)?, opt_level); + let mut price = (off_code * BITCOST_MULTIPLIER).wrapping_add( + opt_ptr + .off_code_sum_base_price + .wrapping_sub(off_code_weight), + ); + if opt_level < 2 && off_code >= 20 { + price = price.wrapping_add((off_code - 19) * 2 * BITCOST_MULTIPLIER); + } + + let ml_code = ml_code(ml_base) as usize; + let ml_code_weight = weight(opt_ptr.match_length_freq.at(ml_code)?, opt_level); + price = price + .wrapping_add(u32::from(ML_BITS.at(ml_code)?) * BITCOST_MULTIPLIER) + .wrapping_add( + opt_ptr + .match_length_sum_base_price + .wrapping_sub(ml_code_weight), + ); + + // Matches cost a bit more, to have fewer sequences, which is faster to decompress. + Ok(price.wrapping_add(BITCOST_MULTIPLIER / 5)) +} + +/// `ZSTD_updateStats` +#[inline(always)] +fn update_stats( + opt_ptr: &mut OptState, + literals: &[u8], + off_base: u32, + match_length: u32, +) -> Result<()> { + #[inline(always)] + fn increment(freqs: &mut [u32], code: u32, sum: &mut u32) -> Result<()> { + let freq = freqs.at_mut(code as usize)?; + *freq = freq.wrapping_add(1); + *sum = sum.wrapping_add(1); + Ok(()) + } + + let lit_length = literals.len() as u32; + if compressed_literals(opt_ptr) { + for &literal in literals { + let freq = opt_ptr.lit_freq.at_mut(usize::from(literal))?; + *freq = freq.wrapping_add(ZSTD_LITFREQ_ADD); + } + opt_ptr.lit_sum = opt_ptr + .lit_sum + .wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD)); + } + + increment( + &mut opt_ptr.lit_length_freq, + ll_code(lit_length), + &mut opt_ptr.lit_length_sum, + )?; + increment( + &mut opt_ptr.off_code_freq, + highbit32(off_base), + &mut opt_ptr.off_code_sum, + )?; + increment( + &mut opt_ptr.match_length_freq, + ml_code(match_length.wrapping_sub(MINMATCH)), + &mut opt_ptr.match_length_sum, + ) +} + +/// `ZSTD_readMINMATCH`: `length` bytes, 3 or 4, to compare with others. Reads 4. +#[inline(always)] +fn read_minmatch(buf: &[u8], pos: usize, length: u32) -> Result { + let value = read_le32(buf, pos)?; + Ok(if length == 3 { value << 8 } else { value }) +} + +/// `ZSTD_insertAndFindFirstIndexHash3`: indexes the 3-byte hashes up to `ip`, which is not included, and looks it up. +/// All of it is in the current segment. +fn insert_and_find_first_index_hash3( + ms: &mut MatchState, + next_to_update3: &mut u32, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + let hash_table3 = ms.hash_table3.as_mut_slice(); + let hash_log3 = ms.hash_log3; + let target = prefix.index_of(ip); + let hash3 = hash3_ptr(prefix.buf, ip, hash_log3)?; + + for idx in *next_to_update3..target { + hash_table3.put(hash3_ptr(prefix.buf, prefix.pos_of(idx), hash_log3)?, idx)?; + } + + *next_to_update3 = target; + hash_table3.at(hash3) +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `*ptr = value`, where `ptr` is an element of `bt`, or `None` for a place that nothing reads. +#[inline(always)] +fn store(bt: &mut [u32], ptr: Option, value: u32) -> Result<()> { + match ptr { + Some(ptr) => bt.put(ptr, value), + None => Ok(()), + } +} + +/// `ZSTD_insertBt1`: adds the position `ip`, which is at most `iend - 8`, to the tree. `target` is up to where the +/// caller is filling it, and `dict` the older segment, which is looked at if `ext_dict`. Returns how many positions +/// to move forward. +#[inline(always)] +fn insert_bt1( + ms: &mut MatchState, + prefix: Prefix<'_>, + dict: Prefix<'_>, + ip: usize, + iend: usize, + target: u32, + mls: u32, + ext_dict: bool, +) -> Result { + let c_params = ms.c_params; + let h = hash_ptr(prefix.buf, ip, c_params.hash_log, mls)?; + let bt_log = c_params.chain_log.wrapping_sub(1); + let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = dict.pos_of(dict_limit); + let prefix_start = prefix.pos_of(dict_limit); + let curr = prefix.index_of(ip); + let bt_low = curr.saturating_sub(bt_mask); + let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); + let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); + // Only the positions that are still in the window once the tree is filled are of use. + let window_low = ms.get_lowest_match_index(target, c_params.window_log); + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut best_length = 8usize; + let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); + + let bt = ms.chain_table.as_mut_slice(); + let hash_entry = ms.hash_table.at_mut(h)?; + let mut match_index = *hash_entry; + *hash_entry = curr; + + while nb_compares != 0 && match_index >= window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + // What is sure to be common. + let mut match_length = common_length_smaller.min(common_length_larger); + + // A position of the older segment, as one of the current segment, only makes sense with a length that + // brings it there. + let mut match_buf = prefix.buf; + let mut match_pos = prefix.pos_of(match_index); + if !ext_dict || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + prefix.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + prefix.buf, + ip + match_length, + dict.buf, + dict.pos_of(match_index) + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + match_buf = dict.buf; + match_pos = dict.pos_of(match_index); + } + } + + if match_length > best_length { + best_length = match_length; + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + } + + if ip + match_length == iend { + // They are equal: there is no telling which is smaller. Dropping it keeps the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < prefix.buf.at(ip + match_length)? { + // All the smaller ones will have at least this much in common. + store(bt, smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. + smaller_ptr = None; + break; + } + smaller_ptr = Some(next_ptr + 1); + match_index = bt.at(next_ptr + 1)?; + } else { + store(bt, larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = None; + break; + } + larger_ptr = Some(next_ptr); + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + store(bt, smaller_ptr, 0)?; + store(bt, larger_ptr, 0)?; + + let positions = if best_length > 384 { + 192.min((best_length - 384) as u32) + } else { + 0 + }; + Ok(positions.max(match_end_idx.wrapping_sub(curr.wrapping_add(8)))) +} + +/// `ZSTD_updateTree_internal` +#[inline(always)] +fn update_tree_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + dict: Prefix<'_>, + ip: usize, + iend: usize, + mls: u32, + dict_mode: DictMode, +) -> Result<()> { + let target = prefix.index_of(ip); + let mut idx = ms.next_to_update; + + while idx < target { + let forward = insert_bt1( + ms, + prefix, + dict, + prefix.pos_of(idx), + iend, + target, + mls, + dict_mode == DictMode::ExtDict, + )?; + idx = idx.wrapping_add(forward); + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. +/// Matches are not compared past the position `iend`. +pub(crate) fn update_tree( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + iend: usize, +) -> Result<()> { + let no_dict = Prefix { buf: &[], base: 0 }; + let mls = ms.c_params.min_match; + update_tree_internal(ms, prefix, no_dict, ip, iend, mls, DictMode::NoDict) +} + +/// `ZSTD_insertBtAndGetAllMatches`: adds the position `ip` to the tree, and stores in `matches` the matches found on +/// the way, each longer than the one before and all at least `length_to_beat` long. `ll0` is whether no literals +/// come before `ip`. Returns how many there are. +#[inline(always)] +fn insert_bt_and_get_all_matches( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + dict_mode: DictMode, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, + mls: u32, +) -> Result { + let buf = src.buf; + let c_params = ms.c_params; + let sufficient_len = c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); + let curr = src.index_of(ip); + let min_match = if mls == 3 { 3 } else { 4 }; + let h = hash_ptr(buf, ip, c_params.hash_log, mls)?; + let bt_log = c_params.chain_log.wrapping_sub(1); + let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt_low = curr.saturating_sub(bt_mask); + let window_low = ms.get_lowest_match_index(curr, c_params.window_log); + let match_low = window_low.max(1); + let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); + let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); + // The farthest position any match refers to, which detects repetitive patterns. + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut mnum = 0u32; + let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); + + let dms = if dict_mode == DictMode::DictMatchState { + Some(src.dms.ok_or(BUG)?) + } else { + None + }; + let dms_high_limit = dms.map_or(0, |dms| dms.end.wrapping_sub(dms.base) as u32); + let dms_low_limit = dms.map_or(0, |dms| dms.ms.window.low_limit); + let dms_index_delta = dms.map_or(0, |_| window_low.wrapping_sub(dms_high_limit)); + + let mut best_length = length_to_beat.wrapping_sub(1) as usize; + + // The repeat offsets. + let ll0 = u32::from(ll0); + let [rep0, _, _] = *rep; + for rep_code in ll0..ZSTD_REP_NUM as u32 + ll0 { + let rep_offset = match rep.get(rep_code as usize) { + Some(&offset) => offset, + None => rep0.wrapping_sub(1), + }; + let rep_index = curr.wrapping_sub(rep_offset); + let mut rep_len = 0u32; + // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. + if rep_offset.wrapping_sub(1) < curr.wrapping_sub(dict_limit) { + // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. + let rep_match = ip.wrapping_sub(rep_offset as usize); + if rep_index >= window_low + && read_minmatch(buf, ip, min_match)? == read_minmatch(buf, rep_match, min_match)? + { + rep_len = count( + buf, + ip + min_match as usize, + buf, + rep_match + min_match as usize, + i_limit, + )? as u32 + + min_match; + } + } else if dict_mode == DictMode::ExtDict { + // `curr > rep_index >= window_low` + if rep_offset.wrapping_sub(1) < curr.wrapping_sub(window_low) + && index_overlap_check(dict_limit, rep_index) + { + let rep_match = src.dict_pos_of(rep_index); + if read_minmatch(buf, ip, min_match)? + == read_minmatch(src.dict_buf, rep_match, min_match)? + { + rep_len = count_2segments( + buf, + ip + min_match as usize, + src.dict_buf, + rep_match + min_match as usize, + i_limit, + dict_end, + prefix_start, + )? as u32 + + min_match; + } + } + } else if let Some(dms) = dms { + // `curr > rep_index >= dms_low_limit` + if rep_offset.wrapping_sub(1) + < curr.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta)) + && index_overlap_check(dict_limit, rep_index) + { + let rep_match = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); + if read_minmatch(buf, ip, min_match)? + == read_minmatch(dms.buf, rep_match, min_match)? + { + rep_len = count_2segments( + buf, + ip + min_match as usize, + dms.buf, + rep_match + min_match as usize, + i_limit, + dms.end, + prefix_start, + )? as u32 + + min_match; + } + } + } + if rep_len as usize > best_length { + best_length = rep_len as usize; + matches.put( + mnum as usize, + Match { + off: rep_code - ll0 + 1, + len: rep_len, + }, + )?; + mnum += 1; + if rep_len > sufficient_len || ip + rep_len as usize == i_limit { + // The best there can be. + return Ok(mnum); + } + } + } + + // The table of 3-byte hashes. That of a dictionary is not filled. + if mls == 3 && best_length < mls as usize { + let match_index3 = + insert_and_find_first_index_hash3(ms, next_to_update3, src.prefix(), ip)?; + // Farther than that is likely to cost too much. + if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) { + let mlen = if dict_mode != DictMode::ExtDict || match_index3 >= dict_limit { + count(buf, ip, buf, src.pos_of(match_index3), i_limit)? + } else { + count_2segments( + buf, + ip, + src.dict_buf, + src.dict_pos_of(match_index3), + i_limit, + dict_end, + prefix_start, + )? + }; + + if mlen >= mls as usize { + best_length = mlen; + matches.put( + 0, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index3)), + len: mlen as u32, + }, + )?; + mnum = 1; + if mlen > sufficient_len as usize || ip + mlen == i_limit { + // The best there can be: `ip` is not inserted. + ms.next_to_update = curr.wrapping_add(1); + return Ok(1); + } + } + } + } + + let bt = ms.chain_table.as_mut_slice(); + let hash_entry = ms.hash_table.at_mut(h)?; + let mut match_index = *hash_entry; + *hash_entry = curr; + + while nb_compares != 0 && match_index >= match_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + // What is sure to be common. + let mut match_length = common_length_smaller.min(common_length_larger); + + // A position of the older segment, as one of the current segment, only makes sense with a length that + // brings it there. + let mut match_buf = buf; + let mut match_pos = src.pos_of(match_index); + if dict_mode != DictMode::ExtDict + || match_index as usize + match_length >= dict_limit as usize + { + match_length += count( + buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + i_limit, + )?; + } else { + match_length += count_2segments( + buf, + ip + match_length, + src.dict_buf, + src.dict_pos_of(match_index) + match_length, + i_limit, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + match_buf = src.dict_buf; + match_pos = src.dict_pos_of(match_index); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + best_length = match_length; + matches.put( + mnum as usize, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index)), + len: match_length as u32, + }, + )?; + mnum += 1; + if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { + // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. + if dict_mode == DictMode::DictMatchState { + // The dictionary is not to be searched either. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < buf.at(ip + match_length)? { + // All the smaller ones will have at least this much in common. + store(bt, smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. + smaller_ptr = None; + break; + } + smaller_ptr = Some(next_ptr + 1); + match_index = bt.at(next_ptr + 1)?; + } else { + store(bt, larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = None; + break; + } + larger_ptr = Some(next_ptr); + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + store(bt, smaller_ptr, 0)?; + store(bt, larger_ptr, 0)?; + + if let Some(dms) = dms + && nb_compares != 0 + { + let dms_c_params = &dms.ms.c_params; + let dms_bt_log = dms_c_params.chain_log.wrapping_sub(1); + let dms_bt_mask = 1u32.wrapping_shl(dms_bt_log).wrapping_sub(1); + let dms_bt_low = if dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit) { + dms_high_limit.wrapping_sub(dms_bt_mask) + } else { + dms_low_limit + }; + let dms_h = hash_ptr(buf, ip, dms_c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(dms_h)?; + let dms_bt = dms.ms.chain_table.as_slice(); + common_length_smaller = 0; + common_length_larger = 0; + while nb_compares != 0 && dict_match_index > dms_low_limit { + let next_ptr = 2 * (dict_match_index & dms_bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + match_length += count_2segments( + buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + i_limit, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + match_index = dict_match_index.wrapping_add(dms_index_delta); + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + best_length = match_length; + matches.put( + mnum as usize, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index)), + len: match_length as u32, + }, + )?; + mnum += 1; + if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { + break; + } + } + + if dict_match_index <= dms_bt_low { + // Beyond the tree. + break; + } + // A match that reaches the end of the dictionary goes on at the start of the current segment. + let match_byte = if dict_match_index as usize + match_length >= dms_high_limit as usize + { + let match_pos = src.pos_of(dict_match_index.wrapping_add(dms_index_delta)); + buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dms_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dms_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + } + + // Repetitive patterns are skipped. + ms.next_to_update = match_end_idx.wrapping_sub(8); + Ok(mnum) +} + +/// `ZSTD_getAllMatchesFn`: the arguments are `matches`, `ms`, `next_to_update3`, `src`, `ip`, `i_high_limit`, `rep`, +/// `ll0` and `length_to_beat`. +type GetAllMatchesFn = fn( + &mut [Match], + &mut MatchState, + &mut u32, + &BlockSrc<'_>, + usize, + usize, + &[u32; ZSTD_REP_NUM], + bool, + u32, +) -> Result; + +/// `ZSTD_btGetAllMatches_internal` +#[inline(always)] +fn bt_get_all_matches_internal( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, + dict_mode: DictMode, + mls: u32, +) -> Result { + if src.index_of(ip) < ms.next_to_update { + // An area that is skipped. + return Ok(0); + } + let dict = Prefix { + buf: src.dict_buf, + base: src.dict_base, + }; + update_tree_internal(ms, src.prefix(), dict, ip, i_high_limit, mls, dict_mode)?; + insert_bt_and_get_all_matches( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + dict_mode, + rep, + ll0, + length_to_beat, + mls, + ) +} + +/// `ZSTD_btGetAllMatches_noDict_3` to `ZSTD_btGetAllMatches_noDict_6` +fn bt_get_all_matches_no_dict( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::NoDict, + MLS, + ) +} + +/// `ZSTD_btGetAllMatches_extDict_3` to `ZSTD_btGetAllMatches_extDict_6` +fn bt_get_all_matches_ext_dict( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::ExtDict, + MLS, + ) +} + +/// `ZSTD_btGetAllMatches_dictMatchState_3` to `ZSTD_btGetAllMatches_dictMatchState_6` +fn bt_get_all_matches_dict_match_state( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::DictMatchState, + MLS, + ) +} + +/// `ZSTD_selectBtGetAllMatches` +fn select_bt_get_all_matches(ms: &MatchState, dict_mode: DictMode) -> GetAllMatchesFn { + match (dict_mode, ms.c_params.min_match) { + (DictMode::NoDict, ..=3) => bt_get_all_matches_no_dict::<3>, + (DictMode::NoDict, 4) => bt_get_all_matches_no_dict::<4>, + (DictMode::NoDict, 5) => bt_get_all_matches_no_dict::<5>, + (DictMode::NoDict, 6..) => bt_get_all_matches_no_dict::<6>, + (DictMode::ExtDict, ..=3) => bt_get_all_matches_ext_dict::<3>, + (DictMode::ExtDict, 4) => bt_get_all_matches_ext_dict::<4>, + (DictMode::ExtDict, 5) => bt_get_all_matches_ext_dict::<5>, + (DictMode::ExtDict, 6..) => bt_get_all_matches_ext_dict::<6>, + (DictMode::DictMatchState, ..=3) => bt_get_all_matches_dict_match_state::<3>, + (DictMode::DictMatchState, 4) => bt_get_all_matches_dict_match_state::<4>, + (DictMode::DictMatchState, 5) => bt_get_all_matches_dict_match_state::<5>, + (DictMode::DictMatchState, 6..) => bt_get_all_matches_dict_match_state::<6>, + } +} + +// ── candidates from long distance matching ──────────────────────────────── + +/// `ZSTD_optLdm_t`: what it takes to decide whether to use a long distance match. +struct OptLdm<'a> { + /// `seqStore.seq`: the `seqStore.size` candidates for this block and the next ones. + seq: &'a [RawSeq], + /// `seqStore.pos` + pos: usize, + /// `seqStore.posInSequence` + pos_in_sequence: usize, + /// Where the current candidate starts. + start_pos_in_block: u32, + /// Where it ends. + end_pos_in_block: u32, + offset: u32, +} + +impl<'a> OptLdm<'a> { + fn new(seq_store: Option<&'a RawSeqStore>) -> Self { + Self { + seq: seq_store + .and_then(|store| store.seq.get(..store.size)) + .unwrap_or_default(), + pos: seq_store.map_or(0, |store| store.pos), + pos_in_sequence: seq_store.map_or(0, |store| store.pos_in_sequence), + start_pos_in_block: 0, + end_pos_in_block: 0, + offset: 0, + } + } + + /// `ZSTD_optLdm_skipRawSeqStoreBytes`: moves forward by `nb_bytes`. + fn skip_raw_seq_store_bytes(&mut self, nb_bytes: usize) { + let mut curr_pos = self.pos_in_sequence.wrapping_add(nb_bytes) as u32; + while curr_pos != 0 { + let Some(curr_seq) = self.seq.get(self.pos) else { + break; + }; + let seq_length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); + if curr_pos >= seq_length { + curr_pos -= seq_length; + self.pos += 1; + } else { + self.pos_in_sequence = curr_pos as usize; + break; + } + } + if curr_pos == 0 || self.pos == self.seq.len() { + self.pos_in_sequence = 0; + } + } + + /// `ZSTD_opt_getNextMatchAndUpdateSeqStore`: finds where the next candidate starts and ends in the block, and + /// moves past it. + fn get_next_match_and_update_seq_store( + &mut self, + curr_pos_in_block: u32, + block_bytes_remaining: u32, + ) { + let Some(&curr_seq) = self.seq.get(self.pos) else { + // No position gets there, so that there is no candidate in this block. + self.start_pos_in_block = u32::MAX; + self.end_pos_in_block = u32::MAX; + return; + }; + // What is left of the literals and of the match. + let pos_in_sequence = self.pos_in_sequence as u32; + let curr_block_end_pos = curr_pos_in_block.wrapping_add(block_bytes_remaining); + let literals_bytes_remaining = curr_seq.lit_length.saturating_sub(pos_in_sequence); + let match_bytes_remaining = if literals_bytes_remaining == 0 { + curr_seq + .match_length + .wrapping_sub(pos_in_sequence.wrapping_sub(curr_seq.lit_length)) + } else { + curr_seq.match_length + }; + + if literals_bytes_remaining >= block_bytes_remaining { + // The match starts after the block. + self.start_pos_in_block = u32::MAX; + self.end_pos_in_block = u32::MAX; + self.skip_raw_seq_store_bytes(block_bytes_remaining as usize); + return; + } + + // What is left of a match can be too short to use, which is checked later. + self.start_pos_in_block = curr_pos_in_block.wrapping_add(literals_bytes_remaining); + self.end_pos_in_block = self.start_pos_in_block.wrapping_add(match_bytes_remaining); + self.offset = curr_seq.offset; + + if self.end_pos_in_block > curr_block_end_pos { + // The match ends after the block, which can only use its start. + self.end_pos_in_block = curr_block_end_pos; + self.skip_raw_seq_store_bytes( + curr_block_end_pos.wrapping_sub(curr_pos_in_block) as usize + ); + } else { + self.skip_raw_seq_store_bytes( + literals_bytes_remaining.wrapping_add(match_bytes_remaining) as usize, + ); + } + } + + /// `ZSTD_optLdm_maybeAddMatch`: adds what is left of the candidate at `curr_pos_in_block` to `matches`, if it is + /// long enough and longer than all of them, since they are in order of length. + fn maybe_add_match( + &self, + matches: &mut [Match], + nb_matches: &mut u32, + curr_pos_in_block: u32, + min_match: u32, + ) -> Result<()> { + let pos_diff = curr_pos_in_block.wrapping_sub(self.start_pos_in_block); + let candidate_match_length = self + .end_pos_in_block + .wrapping_sub(self.start_pos_in_block) + .wrapping_sub(pos_diff); + + if curr_pos_in_block < self.start_pos_in_block + || curr_pos_in_block >= self.end_pos_in_block + || candidate_match_length < min_match + { + return Ok(()); + } + + if *nb_matches == 0 + || (candidate_match_length > matches.at(*nb_matches as usize - 1)?.len + && (*nb_matches as usize) < ZSTD_OPT_NUM) + { + matches.put( + *nb_matches as usize, + Match { + off: offset_to_offbase(self.offset), + len: candidate_match_length, + }, + )?; + *nb_matches += 1; + } + Ok(()) + } + + /// `ZSTD_optLdm_processMatchCandidate` + #[inline(always)] + fn process_match_candidate( + &mut self, + matches: &mut [Match], + nb_matches: &mut u32, + curr_pos_in_block: u32, + remaining_bytes: u32, + min_match: u32, + ) -> Result<()> { + if self.pos >= self.seq.len() { + return Ok(()); + } + + if curr_pos_in_block >= self.end_pos_in_block { + if curr_pos_in_block > self.end_pos_in_block { + // The parser does not stop at the end of each candidate: it is often some bytes past it. + let pos_overshoot = curr_pos_in_block - self.end_pos_in_block; + self.skip_raw_seq_store_bytes(pos_overshoot as usize); + } + self.get_next_match_and_update_seq_store(curr_pos_in_block, remaining_bytes); + } + self.maybe_add_match(matches, nb_matches, curr_pos_in_block, min_match) + } +} + +// ── the optimal parser ──────────────────────────────────────────────────── + +/// `LIT_PRICE`: the price of the literal at `pos`. +#[inline(always)] +fn lit_price(buf: &[u8], pos: usize, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok(raw_literals_cost(buf.span(pos, 1)?, opt_ptr, opt_level)? as i32) +} + +/// `LL_PRICE` +#[inline(always)] +fn ll_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok(lit_length_price(lit_length, opt_ptr, opt_level)? as i32) +} + +/// `LL_INCPRICE`: what one more literal, which makes `lit_length` of them, adds to the price of their length. +#[inline(always)] +fn ll_inc_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok( + ll_price(lit_length, opt_ptr, opt_level)?.wrapping_sub(ll_price( + lit_length.wrapping_sub(1), + opt_ptr, + opt_level, + )?), + ) +} + +/// `ZSTD_compressBlock_opt_generic` +#[inline(always)] +fn compress_block_opt_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + opt_level: i32, + dict_mode: DictMode, +) -> Result { + // The match finders borrow the whole of `ms`. + let mut price_table = mem::take(&mut ms.opt.price_table); + let mut match_table = mem::take(&mut ms.opt.match_table); + let result = parse_block( + ms, + seq_store, + rep, + src, + &mut price_table, + &mut match_table, + opt_level, + dict_mode, + ); + ms.opt.price_table = price_table; + ms.opt.match_table = match_table; + result +} + +/// The body of `ZSTD_compressBlock_opt_generic`. `opt` is `ms.opt.price_table` and `matches` is `ms.opt.match_table`. +#[inline(always)] +fn parse_block( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + opt: &mut [Optimal], + matches: &mut [Match], + opt_level: i32, + dict_mode: DictMode, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let ilimit = iend.saturating_sub(8); + let prefix_start = src.pos_of(ms.window.dict_limit); + + let get_all_matches = select_bt_get_all_matches(ms, dict_mode); + + let sufficient_len = ms.c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); + let min_match = if ms.c_params.min_match == 3 { 3 } else { 4 }; + let mut next_to_update3 = ms.next_to_update; + + let mut last_stretch = Optimal::default(); + let mut opt_ldm = OptLdm::new(src.ldm_seq_store); + opt_ldm.get_next_match_and_update_seq_store(0, (iend - ip) as u32); + + rescale_freqs( + &mut ms.opt, + src.symbol_costs, + buf.sub(istart..iend)?, + opt_level, + )?; + ip += usize::from(ip == prefix_start); + + 'series: while ip < ilimit { + let mut last_pos: u32; + + // The forward pass. `opt` holds stretches, each a match followed by literals, unlike a sequence, which is + // literals followed by a match: that way each position of a run of literals can have another match before it. + // It ends with the last stretch in `last_stretch`, which starts at `cur` and ends at `last_pos`. + let mut cur = 'forward: { + // Find a first match. + { + let litlen = (ip - anchor) as u32; + let mut nb_matches = get_all_matches( + matches, + ms, + &mut next_to_update3, + src, + ip, + iend, + rep, + litlen == 0, + min_match, + )?; + opt_ldm.process_match_candidate( + matches, + &mut nb_matches, + (ip - istart) as u32, + (iend - ip) as u32, + min_match, + )?; + if nb_matches == 0 { + ip += 1; + continue 'series; + } + let matches = matches.sub(..nb_matches as usize)?; + + // The price of the literals before the first match is the same whatever comes next, so it is left + // out. But how the price of their length changes depends on their length. + let first_price = ll_price(litlen, &ms.opt, opt_level)?; + { + let first = opt.at_mut(0)?; + first.mlen = 0; + first.litlen = litlen; + first.price = first_price; + first.rep = *rep; + } + + // A large match is encoded at once. + let longest = matches.at(matches.len() - 1)?; + if longest.len > sufficient_len { + last_stretch.litlen = 0; + last_stretch.mlen = longest.len; + last_stretch.off = longest.off; + last_pos = longest.len; + break 'forward 0; + } + + // Set the prices of the first matches. + let ll0_price = ll_price(0, &ms.opt, opt_level)?; + let mut pos = 1u32; + while pos < min_match { + let stretch = opt.at_mut(pos as usize)?; + stretch.price = ZSTD_MAX_PRICE; + stretch.mlen = 0; + stretch.litlen = litlen.wrapping_add(pos); + pos += 1; + } + for candidate in matches { + while pos <= candidate.len { + let match_price = + get_match_price(candidate.off, pos, &ms.opt, opt_level)? as i32; + let sequence_price = first_price.wrapping_add(match_price); + let stretch = opt.at_mut(pos as usize)?; + stretch.mlen = pos; + stretch.off = candidate.off; + stretch.litlen = 0; + stretch.price = sequence_price.wrapping_add(ll0_price); + pos += 1; + } + } + last_pos = pos - 1; + opt.at_mut(pos as usize)?.price = ZSTD_MAX_PRICE; + } + + // Check the positions that follow. + let mut cur = 0u32; + while cur < last_pos { + cur += 1; + let inr = ip + cur as usize; + + // Get to this position by a literal, if it is cheaper. + { + let before = opt.at(cur as usize - 1)?; + let litlen = before.litlen.wrapping_add(1); + let price = before + .price + .wrapping_add(lit_price(buf, inr - 1, &ms.opt, opt_level)?) + .wrapping_add(ll_inc_price(litlen, &ms.opt, opt_level)?); + let prev_match = opt.at(cur as usize)?; + if price <= prev_match.price { + opt.put( + cur as usize, + Optimal { + litlen, + price, + ..before + }, + )?; + // If this replaces a match, and 1 literal is cheaper than none, the match followed by a + // literal may be a cheaper way to the next position. + if opt_level >= 1 + && prev_match.litlen == 0 + && ll_inc_price(1, &ms.opt, opt_level)? < 0 + && inr < iend + { + let next_lit_price = lit_price(buf, inr, &ms.opt, opt_level)?; + let with_1_literal = prev_match + .price + .wrapping_add(next_lit_price) + .wrapping_add(ll_inc_price(1, &ms.opt, opt_level)?); + let with_more_literals = price + .wrapping_add(next_lit_price) + .wrapping_add(ll_inc_price( + litlen.wrapping_add(1), + &ms.opt, + opt_level, + )?); + if with_1_literal < with_more_literals + && with_1_literal < opt.at(cur as usize + 1)?.price + { + let prev = opt.at(cur.wrapping_sub(prev_match.mlen) as usize)?; + opt.put( + cur as usize + 1, + Optimal { + rep: new_rep(&prev.rep, prev_match.off, prev.litlen == 0), + litlen: 1, + price: with_1_literal, + ..prev_match + }, + )?; + last_pos = last_pos.max(cur + 1); + } + } + } + } + + // The repeat offsets are not updated while matches are compared, but here, now that one is selected. + let mut here = opt.at(cur as usize)?; + if here.litlen == 0 { + let prev = opt.at(cur.wrapping_sub(here.mlen) as usize)?; + here.rep = new_rep(&prev.rep, here.off, prev.litlen == 0); + opt.at_mut(cur as usize)?.rep = here.rep; + } + + // The last match starts at least 8 bytes before the end. + if inr > ilimit { + continue; + } + + if cur == last_pos { + break; + } + + if opt_level == 0 + && opt.at(cur as usize + 1)?.price + <= here.price.wrapping_add(BITCOST_MULTIPLIER as i32 / 2) + { + // The next position is cheaper to get to: this one is not promising. + continue; + } + + { + let base_price = here.price.wrapping_add(ll_price(0, &ms.opt, opt_level)?); + let mut nb_matches = get_all_matches( + matches, + ms, + &mut next_to_update3, + src, + inr, + iend, + &here.rep, + here.litlen == 0, + min_match, + )?; + opt_ldm.process_match_candidate( + matches, + &mut nb_matches, + (inr - istart) as u32, + (iend - inr) as u32, + min_match, + )?; + if nb_matches == 0 { + continue; + } + let matches = matches.sub(..nb_matches as usize)?; + + let longest = matches.at(matches.len() - 1)?; + if longest.len > sufficient_len + || cur.wrapping_add(longest.len) as usize >= ZSTD_OPT_NUM + || inr + longest.len as usize >= iend + { + last_stretch.mlen = longest.len; + last_stretch.off = longest.off; + last_stretch.litlen = 0; + last_pos = cur.wrapping_add(longest.len); + break 'forward cur; + } + + // Set the prices that the matches found here give. + let mut start_ml = min_match; + for candidate in matches { + let mut mlen = candidate.len; + while mlen >= start_ml { + let pos = cur + mlen; + let price = base_price.wrapping_add(get_match_price( + candidate.off, + mlen, + &ms.opt, + opt_level, + )? + as i32); + + if pos > last_pos || price < opt.at(pos as usize)?.price { + while last_pos < pos { + // Positions that nothing leads to yet. + last_pos += 1; + let stretch = opt.at_mut(last_pos as usize)?; + stretch.price = ZSTD_MAX_PRICE; + // Not 0, which would be the end of a match. + stretch.litlen = 1; + } + let stretch = opt.at_mut(pos as usize)?; + stretch.mlen = mlen; + stretch.off = candidate.off; + stretch.litlen = 0; + stretch.price = price; + } else if opt_level == 0 { + // Shorter lengths are not tried. + break; + } + mlen -= 1; + } + start_ml = candidate.len + 1; + } + } + opt.at_mut(last_pos as usize + 1)?.price = ZSTD_MAX_PRICE; + } + + last_stretch = opt.at(last_pos as usize)?; + last_pos.wrapping_sub(last_stretch.mlen) + }; + + if last_stretch.mlen == 0 { + // All the matches have been turned into literals. + ip += last_pos as usize; + continue; + } + + if last_stretch.litlen == 0 { + // It ends with a match. + let prev = opt.at(cur as usize)?; + *rep = new_rep(&prev.rep, last_stretch.off, prev.litlen == 0); + } else { + *rep = last_stretch.rep; + cur = cur.wrapping_sub(last_stretch.litlen); + } + + // Write the cheapest path in `opt`, backwards from `store_end`, over the stretches that are not on it, and as + // sequences. The literals that end the last stretch are in no sequence: they are parsed again. (The C code + // starts by making them a last sequence without a match, and overwrites it at once.) + let store_end = cur.wrapping_add(2) as usize; + let mut store_start = store_end; + let mut stretch_pos = cur; + opt.put(store_end, last_stretch)?; + loop { + let next_stretch = opt.at(stretch_pos as usize)?; + opt.at_mut(store_start)?.litlen = next_stretch.litlen; + if next_stretch.mlen == 0 { + // The start of the series. + break; + } + store_start = store_start.wrapping_sub(1); + opt.put(store_start, next_stretch)?; + stretch_pos = + stretch_pos.wrapping_sub(next_stretch.litlen.wrapping_add(next_stretch.mlen)); + } + + for sequence in opt.sub(store_start..=store_end)? { + let llen = sequence.litlen as usize; + let mlen = sequence.mlen as usize; + update_stats( + &mut ms.opt, + buf.span(anchor, llen)?, + sequence.off, + sequence.mlen, + )?; + seq_store.store_seq(llen, buf, anchor, sequence.off, mlen)?; + anchor += llen + mlen; + ip = anchor; + } + + set_base_prices(&mut ms.opt, opt_level); + } + + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_opt0` +fn compress_block_opt0( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode) +} + +/// `ZSTD_compressBlock_opt2` +fn compress_block_opt2( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode) +} + +/// `ZSTD_initStats_ultra`: a first pass over the block, only to have better statistics to start from. It has to be the +/// first block, with no dictionary and no long distance matching. Returns the block as the window then sees it, since +/// its indices change. +fn init_stats_ultra<'a>( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &[u32; ZSTD_REP_NUM], + src: &BlockSrc<'a>, +) -> Result> { + // The repeat offsets it leaves are of no use. + let mut tmp_rep = *rep; + compress_block_opt2(ms, seq_store, &mut tmp_rep, src, DictMode::NoDict)?; + + // Of the first pass, only the statistics are kept: what the tables have is made out of reach. + let src_size = src.end - src.start; + seq_store.reset(); + ms.window.base = ms.window.base.wrapping_sub(src_size); + ms.window.dict_limit = ms.window.dict_limit.wrapping_add(src_size as u32); + ms.window.low_limit = ms.window.dict_limit; + ms.next_to_update = ms.window.dict_limit; + Ok(BlockSrc { + base: src.base.wrapping_sub(src_size), + ..*src + }) +} + +/// `ZSTD_compressBlock_btultra2` +fn compress_block_btultra2( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let curr = src.index_of(src.start); + + // The first block is parsed twice, the first time to collect statistics for the second. Since the first pass is + // then forgotten, nothing must be in the tables before it: no dictionary, no prefix, no long distance matching. + if ms.opt.lit_length_sum == 0 + && seq_store.sequences.is_empty() + && ms.window.dict_limit == ms.window.low_limit + && curr == ms.window.dict_limit + && src.end - src.start > ZSTD_PREDEF_THRESHOLD + { + let src = init_stats_ultra(ms, seq_store, rep, src)?; + return compress_block_opt2(ms, seq_store, rep, &src, DictMode::NoDict); + } + + compress_block_opt2(ms, seq_store, rep, src, DictMode::NoDict) +} + +/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` +/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. +/// +/// `btultra2` can move `ms.window.base`, which `src.base` comes from. +pub(crate) fn compress_block_opt( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + dict_mode: DictMode, +) -> Result { + match (strategy, dict_mode) { + (Strategy::BtUltra2, DictMode::NoDict) => compress_block_btultra2(ms, seq_store, rep, src), + (Strategy::BtUltra | Strategy::BtUltra2, _) => { + compress_block_opt2(ms, seq_store, rep, src, dict_mode) + } + _ => compress_block_opt0(ms, seq_store, rep, src, dict_mode), + } +} diff --git a/src/zstd/compress/params.rs b/src/zstd/compress/params.rs new file mode 100644 index 000000000000..44d9fe2ae0b4 --- /dev/null +++ b/src/zstd/compress/params.rs @@ -0,0 +1,911 @@ +//! The parameters of `zstd_compress.c`: what can be asked for, and how it turns into what is applied. + +use super::internal::{ + CompressionParameters, LdmParams, ParamSwitch, Strategy, ZSTD_OPT_SIZE, + ZSTD_SHORT_CACHE_TAG_BITS, +}; +use super::{ldm, row_match_finder_supported}; +use crate::common::{ + LITBITS, MAX_LL, MAX_ML, MAX_OFF, WILDCOPY_OVERLENGTH, ZSTD_BLOCKSIZE_MAX, + ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WINDOWLOG_LIMIT_DEFAULT, + ZSTD_WINDOWLOG_MAX, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::highbit32; + +pub const ZSTD_CLEVEL_DEFAULT: i32 = 3; +pub const ZSTD_MAX_CLEVEL: i32 = 22; +pub const ZSTD_MIN_CLEVEL: i32 = -(ZSTD_TARGETLENGTH_MAX); + +const ZSTD_WINDOWLOG_MIN: i32 = 10; +const ZSTD_HASHLOG_MAX: i32 = 30; +const ZSTD_HASHLOG_MIN: i32 = 6; +const ZSTD_CHAINLOG_MAX: i32 = 30; +const ZSTD_CHAINLOG_MIN: i32 = ZSTD_HASHLOG_MIN; +const ZSTD_SEARCHLOG_MAX: i32 = ZSTD_WINDOWLOG_MAX as i32 - 1; +const ZSTD_SEARCHLOG_MIN: i32 = 1; +const ZSTD_MINMATCH_MAX: i32 = 7; +const ZSTD_MINMATCH_MIN: i32 = 3; +const ZSTD_TARGETLENGTH_MAX: i32 = ZSTD_BLOCKSIZE_MAX as i32; +const ZSTD_TARGETLENGTH_MIN: i32 = 0; +const ZSTD_OVERLAPLOG_MIN: i32 = 0; +const ZSTD_OVERLAPLOG_MAX: i32 = 9; +const ZSTD_LDM_HASHLOG_MIN: i32 = ZSTD_HASHLOG_MIN; +const ZSTD_LDM_HASHLOG_MAX: i32 = ZSTD_HASHLOG_MAX; +const ZSTD_LDM_MINMATCH_MIN: i32 = 4; +const ZSTD_LDM_MINMATCH_MAX: i32 = 4096; +const ZSTD_LDM_BUCKETSIZELOG_MIN: i32 = 1; +const ZSTD_LDM_BUCKETSIZELOG_MAX: i32 = 8; +const ZSTD_LDM_HASHRATELOG_MIN: i32 = 0; +const ZSTD_LDM_HASHRATELOG_MAX: i32 = ZSTD_WINDOWLOG_MAX as i32 - ZSTD_HASHLOG_MIN; +const ZSTD_TARGETCBLOCKSIZE_MIN: i32 = 1340; +const ZSTD_TARGETCBLOCKSIZE_MAX: i32 = ZSTD_BLOCKSIZE_MAX as i32; +const ZSTD_LDM_DEFAULT_WINDOW_LOG: u32 = ZSTD_WINDOWLOG_LIMIT_DEFAULT; +pub(crate) const ZSTD_HASHLOG3_MAX: u32 = 17; +pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; + +pub(crate) const ZSTDMT_NBWORKERS_MAX: i32 = 256; +pub(crate) const ZSTDMT_JOBSIZE_MIN: i32 = 512 << 10; +pub(crate) const ZSTDMT_JOBLOG_MAX: u32 = 30; +pub(crate) const ZSTDMT_JOBSIZE_MAX: i32 = 1024 << 20; + +/// `ZSTD_cParameter`, as far as `node:zlib` lets a parameter through. +pub mod c_param { + pub const FORMAT: i32 = 10; + pub const COMPRESSION_LEVEL: i32 = 100; + pub const WINDOW_LOG: i32 = 101; + pub const HASH_LOG: i32 = 102; + pub const CHAIN_LOG: i32 = 103; + pub const SEARCH_LOG: i32 = 104; + pub const MIN_MATCH: i32 = 105; + pub const TARGET_LENGTH: i32 = 106; + pub const STRATEGY: i32 = 107; + pub const TARGET_C_BLOCK_SIZE: i32 = 130; + pub const ENABLE_LONG_DISTANCE_MATCHING: i32 = 160; + pub const LDM_HASH_LOG: i32 = 161; + pub const LDM_MIN_MATCH: i32 = 162; + pub const LDM_BUCKET_SIZE_LOG: i32 = 163; + pub const LDM_HASH_RATE_LOG: i32 = 164; + pub const CONTENT_SIZE_FLAG: i32 = 200; + pub const CHECKSUM_FLAG: i32 = 201; + pub const DICT_ID_FLAG: i32 = 202; + pub const NB_WORKERS: i32 = 400; + pub const JOB_SIZE: i32 = 401; + pub const OVERLAP_LOG: i32 = 402; +} + +/// `ZSTD_format_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Format { + Zstd1, + Zstd1Magicless, +} + +/// `ZSTD_frameParameters` +#[derive(Clone, Copy)] +pub(crate) struct FrameParameters { + pub(crate) content_size_flag: bool, + pub(crate) checksum_flag: bool, + pub(crate) no_dict_id_flag: bool, +} + +/// `ZSTD_parameters` +#[derive(Clone, Copy)] +pub(crate) struct Parameters { + pub(crate) c_params: CompressionParameters, + pub(crate) f_params: FrameParameters, +} + +/// `ZSTD_CParamMode_e`: what the parameters are selected for. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum CParamMode { + /// No dictionary, or one that is copied or loaded into the context's own tables. + NoAttachDict, + /// The dictionary keeps its own tables, so its size does not count. + AttachDict, + CreateCDict, + Unknown, +} + +/// `ZSTD_CCtx_params`, without the fields that nothing can change from their default. +#[derive(Clone, Copy)] +pub(crate) struct CCtxParams { + pub(crate) format: Format, + pub(crate) c_params: CompressionParameters, + pub(crate) f_params: FrameParameters, + pub(crate) compression_level: i32, + /// Only the jobs of a multithreaded compression set it. + pub(crate) force_window: bool, + pub(crate) target_c_block_size: usize, + pub(crate) nb_workers: i32, + pub(crate) job_size: usize, + pub(crate) overlap_log: i32, + pub(crate) ldm_params: LdmParams, + pub(crate) post_block_splitter: ParamSwitch, + pub(crate) use_row_match_finder: ParamSwitch, +} + +const NO_C_PARAMS: CompressionParameters = CompressionParameters { + window_log: 0, + chain_log: 0, + hash_log: 0, + search_log: 0, + min_match: 0, + target_length: 0, + strategy: None, +}; + +const NO_LDM_PARAMS: LdmParams = LdmParams { + enable_ldm: ParamSwitch::Auto, + hash_log: 0, + bucket_size_log: 0, + min_match_length: 0, + hash_rate_log: 0, + window_log: 0, +}; + +/// `ZSTD_cParam_getBounds` +fn bounds(param: i32) -> Result<(i32, i32)> { + Ok(match param { + c_param::COMPRESSION_LEVEL => (ZSTD_MIN_CLEVEL, ZSTD_MAX_CLEVEL), + c_param::WINDOW_LOG => (ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX as i32), + c_param::HASH_LOG => (ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX), + c_param::CHAIN_LOG => (ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX), + c_param::SEARCH_LOG => (ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX), + c_param::MIN_MATCH => (ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX), + c_param::TARGET_LENGTH => (ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX), + c_param::STRATEGY => (Strategy::Fast as i32, Strategy::BtUltra2 as i32), + c_param::CONTENT_SIZE_FLAG | c_param::CHECKSUM_FLAG | c_param::DICT_ID_FLAG => (0, 1), + c_param::NB_WORKERS => (0, ZSTDMT_NBWORKERS_MAX), + c_param::JOB_SIZE => (0, ZSTDMT_JOBSIZE_MAX), + c_param::OVERLAP_LOG => (ZSTD_OVERLAPLOG_MIN, ZSTD_OVERLAPLOG_MAX), + c_param::ENABLE_LONG_DISTANCE_MATCHING => { + (ParamSwitch::Auto as i32, ParamSwitch::Disable as i32) + } + c_param::LDM_HASH_LOG => (ZSTD_LDM_HASHLOG_MIN, ZSTD_LDM_HASHLOG_MAX), + c_param::LDM_MIN_MATCH => (ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), + c_param::LDM_BUCKET_SIZE_LOG => (ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), + c_param::LDM_HASH_RATE_LOG => (ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), + c_param::FORMAT => (0, 1), + c_param::TARGET_C_BLOCK_SIZE => (ZSTD_TARGETCBLOCKSIZE_MIN, ZSTD_TARGETCBLOCKSIZE_MAX), + _ => return Err(ErrorCode::ParameterUnsupported), + }) +} + +/// `ZSTD_cParam_clampBounds` +fn clamp_bounds(param: i32, value: i32) -> Result { + let (lower, upper) = bounds(param)?; + Ok(value.clamp(lower, upper)) +} + +/// `BOUNDCHECK` +fn bound_check(param: i32, value: i32) -> Result<()> { + let (lower, upper) = bounds(param)?; + if value < lower || value > upper { + return Err(ErrorCode::ParameterOutOfBound); + } + Ok(()) +} + +/// `BOUNDCHECK`, for the parameters of which 0 means "use the default". +fn bound_check_unless_0(param: i32, value: i32) -> Result { + if value != 0 { + bound_check(param, value)?; + } + Ok(value as u32) +} + +/// `ZSTD_isUpdateAuthorized`: whether the parameter can change while a frame is being written. +pub(crate) fn is_update_authorized(param: i32) -> bool { + matches!( + param, + c_param::COMPRESSION_LEVEL + | c_param::HASH_LOG + | c_param::CHAIN_LOG + | c_param::SEARCH_LOG + | c_param::MIN_MATCH + | c_param::TARGET_LENGTH + | c_param::STRATEGY + ) +} + +impl CCtxParams { + /// `ZSTD_CCtxParams_init` + pub(crate) const fn new(compression_level: i32) -> Self { + Self { + format: Format::Zstd1, + c_params: NO_C_PARAMS, + f_params: FrameParameters { + content_size_flag: true, + checksum_flag: false, + no_dict_id_flag: false, + }, + compression_level, + force_window: false, + target_c_block_size: 0, + nb_workers: 0, + job_size: 0, + overlap_log: 0, + ldm_params: NO_LDM_PARAMS, + post_block_splitter: ParamSwitch::Auto, + use_row_match_finder: ParamSwitch::Auto, + } + } + + /// `ZSTD_CCtxParams_init_internal` + pub(crate) fn from_params(params: &Parameters, compression_level: i32) -> Self { + let mut this = Self::new(compression_level); + this.c_params = params.c_params; + this.f_params = params.f_params; + this.use_row_match_finder = + resolve_row_match_finder_mode(this.use_row_match_finder, ¶ms.c_params); + this.post_block_splitter = + resolve_block_splitter_mode(this.post_block_splitter, ¶ms.c_params); + this.ldm_params.enable_ldm = + resolve_enable_ldm(this.ldm_params.enable_ldm, ¶ms.c_params); + this + } + + /// `ZSTD_CCtxParams_setParameter` + pub(crate) fn set_parameter(&mut self, param: i32, value: i32) -> Result { + match param { + c_param::FORMAT => { + bound_check(param, value)?; + self.format = if value == 0 { + Format::Zstd1 + } else { + Format::Zstd1Magicless + }; + Ok(value as usize) + } + c_param::COMPRESSION_LEVEL => { + let value = clamp_bounds(param, value)?; + self.compression_level = if value == 0 { + ZSTD_CLEVEL_DEFAULT + } else { + value + }; + Ok(self.compression_level.max(0) as usize) + } + c_param::WINDOW_LOG => { + self.c_params.window_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::HASH_LOG => { + self.c_params.hash_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::CHAIN_LOG => { + self.c_params.chain_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::SEARCH_LOG => { + self.c_params.search_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::MIN_MATCH => { + self.c_params.min_match = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::TARGET_LENGTH => { + bound_check(param, value)?; + self.c_params.target_length = value as u32; + Ok(value as usize) + } + c_param::STRATEGY => { + bound_check_unless_0(param, value)?; + self.c_params.strategy = Strategy::from_int(value); + Ok(value as usize) + } + c_param::CONTENT_SIZE_FLAG => { + self.f_params.content_size_flag = value != 0; + Ok(usize::from(value != 0)) + } + c_param::CHECKSUM_FLAG => { + self.f_params.checksum_flag = value != 0; + Ok(usize::from(value != 0)) + } + c_param::DICT_ID_FLAG => { + self.f_params.no_dict_id_flag = value == 0; + Ok(usize::from(value != 0)) + } + c_param::NB_WORKERS => { + self.nb_workers = clamp_bounds(param, value)?; + Ok(self.nb_workers as usize) + } + c_param::JOB_SIZE => { + let value = if value != 0 && value < ZSTDMT_JOBSIZE_MIN { + ZSTDMT_JOBSIZE_MIN + } else { + value + }; + self.job_size = clamp_bounds(param, value)? as usize; + Ok(self.job_size) + } + c_param::OVERLAP_LOG => { + self.overlap_log = clamp_bounds(param, value)?; + Ok(self.overlap_log as usize) + } + c_param::ENABLE_LONG_DISTANCE_MATCHING => { + bound_check(param, value)?; + self.ldm_params.enable_ldm = match value { + 0 => ParamSwitch::Auto, + 1 => ParamSwitch::Enable, + _ => ParamSwitch::Disable, + }; + Ok(value as usize) + } + c_param::LDM_HASH_LOG => { + self.ldm_params.hash_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::LDM_MIN_MATCH => { + self.ldm_params.min_match_length = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::LDM_BUCKET_SIZE_LOG => { + self.ldm_params.bucket_size_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::LDM_HASH_RATE_LOG => { + self.ldm_params.hash_rate_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::TARGET_C_BLOCK_SIZE => { + let value = if value != 0 { + value.max(ZSTD_TARGETCBLOCKSIZE_MIN) + } else { + value + }; + self.target_c_block_size = bound_check_unless_0(param, value)? as usize; + Ok(self.target_c_block_size) + } + _ => Err(ErrorCode::ParameterUnsupported), + } + } +} + +/// `ZSTD_rowMatchFinderUsed` +pub(crate) fn row_match_finder_used(strategy: Strategy, mode: ParamSwitch) -> bool { + row_match_finder_supported(strategy) && mode == ParamSwitch::Enable +} + +/// `ZSTD_resolveRowMatchFinderMode` +pub(crate) fn resolve_row_match_finder_mode( + mode: ParamSwitch, + c_params: &CompressionParameters, +) -> ParamSwitch { + if mode != ParamSwitch::Auto { + return mode; + } + if row_match_finder_supported(c_params.strat()) && c_params.window_log > 14 { + ParamSwitch::Enable + } else { + ParamSwitch::Disable + } +} + +/// `ZSTD_resolveBlockSplitterMode` +pub(crate) fn resolve_block_splitter_mode( + mode: ParamSwitch, + c_params: &CompressionParameters, +) -> ParamSwitch { + if mode != ParamSwitch::Auto { + return mode; + } + if c_params.strat() >= Strategy::BtOpt && c_params.window_log >= 17 { + ParamSwitch::Enable + } else { + ParamSwitch::Disable + } +} + +/// `ZSTD_allocateChainTable` +pub(crate) fn allocate_chain_table(strategy: Strategy, use_row_match_finder: ParamSwitch) -> bool { + strategy != Strategy::Fast && !row_match_finder_used(strategy, use_row_match_finder) +} + +/// `ZSTD_resolveEnableLdm` +pub(crate) fn resolve_enable_ldm( + mode: ParamSwitch, + c_params: &CompressionParameters, +) -> ParamSwitch { + if mode != ParamSwitch::Auto { + return mode; + } + if c_params.strat() >= Strategy::BtOpt && c_params.window_log >= 27 { + ParamSwitch::Enable + } else { + ParamSwitch::Disable + } +} + +/// `ZSTD_CDictIndicesAreTagged` +pub(crate) fn cdict_indices_are_tagged(c_params: &CompressionParameters) -> bool { + matches!(c_params.strat(), Strategy::Fast | Strategy::DFast) +} + +/// `ZSTD_cycleLog` +pub(crate) fn cycle_log(hash_log: u32, strat: Strategy) -> u32 { + hash_log - u32::from(strat >= Strategy::BtLazy2) +} + +/// `ZSTD_dictAndWindowLog`: the log of a window that also covers the dictionary. +fn dict_and_window_log(window_log: u32, src_size: u64, dict_size: u64) -> u32 { + let max_window_size = 1u64 << ZSTD_WINDOWLOG_MAX; + if dict_size == 0 { + return window_log; + } + let window_size = 1u64 << window_log; + let dict_and_window_size = dict_size + window_size; + if window_size >= dict_size.wrapping_add(src_size) { + window_log + } else if dict_and_window_size >= max_window_size { + ZSTD_WINDOWLOG_MAX + } else { + highbit32(dict_and_window_size as u32 - 1) + 1 + } +} + +/// `ZSTD_adjustCParams_internal` +pub(crate) fn adjust_c_params_internal( + mut c_par: CompressionParameters, + mut src_size: u64, + mut dict_size: usize, + mode: CParamMode, + mut use_row_match_finder: ParamSwitch, +) -> CompressionParameters { + const MIN_SRC_SIZE: u64 = 513; + const MAX_WINDOW_RESIZE: u64 = 1 << (ZSTD_WINDOWLOG_MAX - 1); + + match mode { + CParamMode::Unknown | CParamMode::NoAttachDict => {} + CParamMode::CreateCDict => { + if dict_size != 0 && src_size == ZSTD_CONTENTSIZE_UNKNOWN { + src_size = MIN_SRC_SIZE; + } + } + CParamMode::AttachDict => dict_size = 0, + } + + if src_size <= MAX_WINDOW_RESIZE && dict_size as u64 <= MAX_WINDOW_RESIZE { + let t_size = (src_size + dict_size as u64) as u32; + const HASH_SIZE_MIN: u32 = 1 << ZSTD_HASHLOG_MIN; + let src_log = if t_size < HASH_SIZE_MIN { + ZSTD_HASHLOG_MIN as u32 + } else { + highbit32(t_size - 1) + 1 + }; + c_par.window_log = c_par.window_log.min(src_log); + } + if src_size != ZSTD_CONTENTSIZE_UNKNOWN { + let dict_and_window_log = dict_and_window_log(c_par.window_log, src_size, dict_size as u64); + let cycle_log = cycle_log(c_par.chain_log, c_par.strat()); + c_par.hash_log = c_par.hash_log.min(dict_and_window_log + 1); + if cycle_log > dict_and_window_log { + c_par.chain_log -= cycle_log - dict_and_window_log; + } + } + + c_par.window_log = c_par.window_log.max(ZSTD_WINDOWLOG_ABSOLUTEMIN); + + if mode == CParamMode::CreateCDict && cdict_indices_are_tagged(&c_par) { + let max_short_cache_hash_log = 32 - ZSTD_SHORT_CACHE_TAG_BITS; + c_par.hash_log = c_par.hash_log.min(max_short_cache_hash_log); + c_par.chain_log = c_par.chain_log.min(max_short_cache_hash_log); + } + + if use_row_match_finder == ParamSwitch::Auto { + use_row_match_finder = ParamSwitch::Enable; + } + if row_match_finder_used(c_par.strat(), use_row_match_finder) { + // The rows' hashes are 32 bits, some of which are the tag. + let row_log = c_par.search_log.clamp(4, 6); + let max_row_hash_log = 32 - ZSTD_ROW_HASH_TAG_BITS; + c_par.hash_log = c_par.hash_log.min(max_row_hash_log + row_log); + } + + c_par +} + +/// `ZSTD_overrideCParams` +fn override_c_params(c_params: &mut CompressionParameters, overrides: &CompressionParameters) { + if overrides.window_log != 0 { + c_params.window_log = overrides.window_log; + } + if overrides.hash_log != 0 { + c_params.hash_log = overrides.hash_log; + } + if overrides.chain_log != 0 { + c_params.chain_log = overrides.chain_log; + } + if overrides.search_log != 0 { + c_params.search_log = overrides.search_log; + } + if overrides.min_match != 0 { + c_params.min_match = overrides.min_match; + } + if overrides.target_length != 0 { + c_params.target_length = overrides.target_length; + } + if overrides.strategy.is_some() { + c_params.strategy = overrides.strategy; + } +} + +/// `ZSTD_getCParamsFromCCtxParams` +pub(crate) fn get_c_params_from_cctx_params( + cctx_params: &CCtxParams, + src_size_hint: u64, + dict_size: usize, + mode: CParamMode, +) -> CompressionParameters { + let mut c_params = + get_c_params_internal(cctx_params.compression_level, src_size_hint, dict_size, mode); + if cctx_params.ldm_params.enable_ldm == ParamSwitch::Enable { + c_params.window_log = ZSTD_LDM_DEFAULT_WINDOW_LOG; + } + override_c_params(&mut c_params, &cctx_params.c_params); + adjust_c_params_internal( + c_params, + src_size_hint, + dict_size, + mode, + cctx_params.use_row_match_finder, + ) +} + +/// `ZSTD_getCParamRowSize`: the size that selects a table of [`DEFAULT_C_PARAMETERS`]. +fn get_c_param_row_size(src_size_hint: u64, mut dict_size: usize, mode: CParamMode) -> u64 { + if mode == CParamMode::AttachDict { + dict_size = 0; + } + let unknown = src_size_hint == ZSTD_CONTENTSIZE_UNKNOWN; + let added_size: u64 = if unknown && dict_size > 0 { 500 } else { 0 }; + if unknown && dict_size == 0 { + ZSTD_CONTENTSIZE_UNKNOWN + } else { + src_size_hint + .wrapping_add(dict_size as u64) + .wrapping_add(added_size) + } +} + +/// `ZSTD_getCParams_internal` +pub(crate) fn get_c_params_internal( + compression_level: i32, + src_size_hint: u64, + dict_size: usize, + mode: CParamMode, +) -> CompressionParameters { + let r_size = get_c_param_row_size(src_size_hint, dict_size, mode); + let table_id = usize::from(r_size <= 256 << 10) + + usize::from(r_size <= 128 << 10) + + usize::from(r_size <= 16 << 10); + let row = match compression_level { + 0 => ZSTD_CLEVEL_DEFAULT, + ..0 => 0, + _ => compression_level.min(ZSTD_MAX_CLEVEL), + }; + let mut cp = DEFAULT_C_PARAMETERS + .get(table_id) + .and_then(|table| table.get(row as usize)) + .copied() + .unwrap_or(NO_C_PARAMS); + if compression_level < 0 { + // Negative levels go faster by skipping more. + cp.target_length = compression_level.max(ZSTD_MIN_CLEVEL).unsigned_abs(); + } + adjust_c_params_internal(cp, src_size_hint, dict_size, mode, ParamSwitch::Auto) +} + +/// `ZSTD_getParams_internal` +pub(crate) fn get_params_internal( + compression_level: i32, + src_size_hint: u64, + dict_size: usize, + mode: CParamMode, +) -> Parameters { + Parameters { + c_params: get_c_params_internal(compression_level, src_size_hint, dict_size, mode), + f_params: FrameParameters { + content_size_flag: true, + checksum_flag: false, + no_dict_id_flag: false, + }, + } +} + +/// `ZSTD_compressBound`: the most that `src_size` bytes can take once compressed. No slice is of a size that is too +/// large for it, which gives more than can be allocated, as the error code of the C function is. +pub const fn compress_bound(src_size: usize) -> usize { + const ZSTD_MAX_INPUT_SIZE: usize = 0xFF00_FF00_FF00_FF00; + if src_size >= ZSTD_MAX_INPUT_SIZE { + return usize::MAX; + } + // A margin for small inputs, so that the bound of two inputs is at most that of both together. + let margin = if src_size < (128 << 10) { + ((128 << 10) - src_size) >> 11 + } else { + 0 + }; + src_size + (src_size >> 8) + margin +} + +// ── the size of the workspace ──────────────────────────────────────────────── +// +// The C code carves all the tables and buffers of a context out of one allocation. A context only goes on counting +// indexes from where the previous frame left them as long as that allocation is kept, so this keeps its books. + +const SIZEOF_COMPRESSED_BLOCK_STATE: usize = 5632; +const TMP_WORKSPACE_SIZE: usize = 8920; +const SIZEOF_SEQ_DEF: usize = 8; +const SIZEOF_MATCH: usize = 8; +const SIZEOF_OPTIMAL: usize = 28; +const SIZEOF_RAW_SEQ: usize = 12; +const SIZEOF_LDM_ENTRY: usize = 8; +const ZSTD_CWKSP_ALIGNMENT_BYTES: usize = 64; +const ZSTD_WORKSPACETOOLARGE_FACTOR: usize = 3; +const ZSTD_WORKSPACETOOLARGE_MAXDURATION: i32 = 128; + +/// `ZSTD_cwksp_aligned64_alloc_size` +const fn aligned64(size: usize) -> usize { + size.next_multiple_of(ZSTD_CWKSP_ALIGNMENT_BYTES) +} + +/// `ZSTD_maxNbSeq` +pub(crate) const fn max_nb_seq(block_size: usize, min_match: u32) -> usize { + block_size / if min_match == 3 { 3 } else { 4 } +} + +/// The sizes of the tables of a context's match state. +pub(crate) struct MatchStateSizes { + pub(crate) chain_size: usize, + pub(crate) h_size: usize, + pub(crate) hash_log3: u32, + pub(crate) h3_size: usize, +} + +impl MatchStateSizes { + pub(crate) fn new( + c_params: &CompressionParameters, + use_row_match_finder: ParamSwitch, + for_cctx: bool, + ) -> Self { + let hash_log3 = if for_cctx && c_params.min_match == 3 { + ZSTD_HASHLOG3_MAX.min(c_params.window_log) + } else { + 0 + }; + Self { + chain_size: if allocate_chain_table(c_params.strat(), use_row_match_finder) { + 1 << c_params.chain_log + } else { + 0 + }, + h_size: 1 << c_params.hash_log, + hash_log3, + h3_size: if hash_log3 != 0 { 1 << hash_log3 } else { 0 }, + } + } +} + +/// The books of a `ZSTD_cwksp`, as offsets from its start, which is taken to be aligned. +pub(crate) struct Workspace { + size: usize, + object_end: usize, + table_end: usize, + alloc_start: usize, + oversized_duration: i32, +} + +/// What a context needs room for. +pub(crate) struct WorkspaceNeeds<'a> { + pub(crate) c_params: &'a CompressionParameters, + pub(crate) ldm_params: &'a LdmParams, + pub(crate) use_row_match_finder: ParamSwitch, + pub(crate) buff_in_size: usize, + pub(crate) buff_out_size: usize, + pub(crate) block_size: usize, +} + +impl WorkspaceNeeds<'_> { + /// The tables, which the C code allocates from the bottom. + fn tables(&self) -> usize { + let sizes = MatchStateSizes::new(self.c_params, self.use_row_match_finder, true); + (sizes.chain_size + sizes.h_size + sizes.h3_size) * size_of::() + } + + /// Everything else, which the C code allocates from the top. + fn allocs(&self) -> usize { + let max_nb_seq = max_nb_seq(self.block_size, self.c_params.min_match); + let mut total = 0; + if row_match_finder_used(self.c_params.strat(), self.use_row_match_finder) { + total += aligned64(1 << self.c_params.hash_log); + } + if self.c_params.strat() >= Strategy::BtOpt { + total += aligned64((MAX_ML as usize + 1) * size_of::()) + + aligned64((MAX_LL as usize + 1) * size_of::()) + + aligned64((MAX_OFF as usize + 1) * size_of::()) + + aligned64((1 << LITBITS) * size_of::()) + + aligned64(ZSTD_OPT_SIZE * SIZEOF_MATCH) + + aligned64(ZSTD_OPT_SIZE * SIZEOF_OPTIMAL); + } + total += aligned64(max_nb_seq * SIZEOF_SEQ_DEF); + if self.ldm_params.enable_ldm == ParamSwitch::Enable { + total += aligned64((1usize << self.ldm_params.hash_log) * SIZEOF_LDM_ENTRY) + + aligned64(ldm::get_max_nb_seq(*self.ldm_params, self.block_size) * SIZEOF_RAW_SEQ) + + (1usize << (self.ldm_params.hash_log - self.ldm_params.bucket_size_log)); + } + total + + (self.block_size + WILDCOPY_OVERLENGTH) + + self.buff_in_size + + self.buff_out_size + + 3 * max_nb_seq + } + + /// `ZSTD_estimateCCtxSize_usingCCtxParams_internal` + fn needed_space(&self) -> usize { + let objects = TMP_WORKSPACE_SIZE + 2 * SIZEOF_COMPRESSED_BLOCK_STATE; + let slack_space = ZSTD_CWKSP_ALIGNMENT_BYTES * 2; + objects + self.tables() + self.allocs() + slack_space + } +} + +impl Workspace { + pub(crate) const fn new() -> Self { + Self { + size: 0, + object_end: 0, + table_end: 0, + alloc_start: 0, + oversized_duration: 0, + } + } + + /// What `ZSTD_resetCCtx_internal` does to the workspace. Returns whether it made a new one. + pub(crate) fn reset(&mut self, needs: &WorkspaceNeeds<'_>) -> bool { + let needed_space = needs.needed_space(); + self.oversized_duration += 1; + let too_small = self.size < needed_space; + let wasteful = self.alloc_start - self.table_end + >= needed_space.saturating_mul(ZSTD_WORKSPACETOOLARGE_FACTOR) + && self.oversized_duration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; + let resize = too_small || wasteful; + if resize { + self.size = needed_space; + self.object_end = aligned64(TMP_WORKSPACE_SIZE + 2 * SIZEOF_COMPRESSED_BLOCK_STATE); + self.oversized_duration = 0; + } + self.table_end = self.object_end + needs.tables(); + self.alloc_start = + (self.size - self.size % ZSTD_CWKSP_ALIGNMENT_BYTES).saturating_sub(needs.allocs()); + self.alloc_start = self.alloc_start.max(self.table_end); + resize + } +} + +const fn cp( + window_log: u32, + chain_log: u32, + hash_log: u32, + search_log: u32, + min_match: u32, + target_length: u32, + strategy: Strategy, +) -> CompressionParameters { + CompressionParameters { + window_log, + chain_log, + hash_log, + search_log, + min_match, + target_length, + strategy: Some(strategy), + } +} + +/// `ZSTD_defaultCParameters`: for any size of input, then for at most 256 KB, 128 KB and 16 KB. +#[rustfmt::skip] +static DEFAULT_C_PARAMETERS: [[CompressionParameters; ZSTD_MAX_CLEVEL as usize + 1]; 4] = [ + [ + cp(19, 12, 13, 1, 6, 1, Strategy::Fast), + cp(19, 13, 14, 1, 7, 0, Strategy::Fast), + cp(20, 15, 16, 1, 6, 0, Strategy::Fast), + cp(21, 16, 17, 1, 5, 0, Strategy::DFast), + cp(21, 18, 18, 1, 5, 0, Strategy::DFast), + cp(21, 18, 19, 3, 5, 2, Strategy::Greedy), + cp(21, 18, 19, 3, 5, 4, Strategy::Lazy), + cp(21, 19, 20, 4, 5, 8, Strategy::Lazy), + cp(21, 19, 20, 4, 5, 16, Strategy::Lazy2), + cp(22, 20, 21, 4, 5, 16, Strategy::Lazy2), + cp(22, 21, 22, 5, 5, 16, Strategy::Lazy2), + cp(22, 21, 22, 6, 5, 16, Strategy::Lazy2), + cp(22, 22, 23, 6, 5, 32, Strategy::Lazy2), + cp(22, 22, 22, 4, 5, 32, Strategy::BtLazy2), + cp(22, 22, 23, 5, 5, 32, Strategy::BtLazy2), + cp(22, 23, 23, 6, 5, 32, Strategy::BtLazy2), + cp(22, 22, 22, 5, 5, 48, Strategy::BtOpt), + cp(23, 23, 22, 5, 4, 64, Strategy::BtOpt), + cp(23, 23, 22, 6, 3, 64, Strategy::BtUltra), + cp(23, 24, 22, 7, 3, 256, Strategy::BtUltra2), + cp(25, 25, 23, 7, 3, 256, Strategy::BtUltra2), + cp(26, 26, 24, 7, 3, 512, Strategy::BtUltra2), + cp(27, 27, 25, 9, 3, 999, Strategy::BtUltra2), + ], + [ + cp(18, 12, 13, 1, 5, 1, Strategy::Fast), + cp(18, 13, 14, 1, 6, 0, Strategy::Fast), + cp(18, 14, 14, 1, 5, 0, Strategy::DFast), + cp(18, 16, 16, 1, 4, 0, Strategy::DFast), + cp(18, 16, 17, 3, 5, 2, Strategy::Greedy), + cp(18, 17, 18, 5, 5, 2, Strategy::Greedy), + cp(18, 18, 19, 3, 5, 4, Strategy::Lazy), + cp(18, 18, 19, 4, 4, 4, Strategy::Lazy), + cp(18, 18, 19, 4, 4, 8, Strategy::Lazy2), + cp(18, 18, 19, 5, 4, 8, Strategy::Lazy2), + cp(18, 18, 19, 6, 4, 8, Strategy::Lazy2), + cp(18, 18, 19, 5, 4, 12, Strategy::BtLazy2), + cp(18, 19, 19, 7, 4, 12, Strategy::BtLazy2), + cp(18, 18, 19, 4, 4, 16, Strategy::BtOpt), + cp(18, 18, 19, 4, 3, 32, Strategy::BtOpt), + cp(18, 18, 19, 6, 3, 128, Strategy::BtOpt), + cp(18, 19, 19, 6, 3, 128, Strategy::BtUltra), + cp(18, 19, 19, 8, 3, 256, Strategy::BtUltra), + cp(18, 19, 19, 6, 3, 128, Strategy::BtUltra2), + cp(18, 19, 19, 8, 3, 256, Strategy::BtUltra2), + cp(18, 19, 19, 10, 3, 512, Strategy::BtUltra2), + cp(18, 19, 19, 12, 3, 512, Strategy::BtUltra2), + cp(18, 19, 19, 13, 3, 999, Strategy::BtUltra2), + ], + [ + cp(17, 12, 12, 1, 5, 1, Strategy::Fast), + cp(17, 12, 13, 1, 6, 0, Strategy::Fast), + cp(17, 13, 15, 1, 5, 0, Strategy::Fast), + cp(17, 15, 16, 2, 5, 0, Strategy::DFast), + cp(17, 17, 17, 2, 4, 0, Strategy::DFast), + cp(17, 16, 17, 3, 4, 2, Strategy::Greedy), + cp(17, 16, 17, 3, 4, 4, Strategy::Lazy), + cp(17, 16, 17, 3, 4, 8, Strategy::Lazy2), + cp(17, 16, 17, 4, 4, 8, Strategy::Lazy2), + cp(17, 16, 17, 5, 4, 8, Strategy::Lazy2), + cp(17, 16, 17, 6, 4, 8, Strategy::Lazy2), + cp(17, 17, 17, 5, 4, 8, Strategy::BtLazy2), + cp(17, 18, 17, 7, 4, 12, Strategy::BtLazy2), + cp(17, 18, 17, 3, 4, 12, Strategy::BtOpt), + cp(17, 18, 17, 4, 3, 32, Strategy::BtOpt), + cp(17, 18, 17, 6, 3, 256, Strategy::BtOpt), + cp(17, 18, 17, 6, 3, 128, Strategy::BtUltra), + cp(17, 18, 17, 8, 3, 256, Strategy::BtUltra), + cp(17, 18, 17, 10, 3, 512, Strategy::BtUltra), + cp(17, 18, 17, 5, 3, 256, Strategy::BtUltra2), + cp(17, 18, 17, 7, 3, 512, Strategy::BtUltra2), + cp(17, 18, 17, 9, 3, 512, Strategy::BtUltra2), + cp(17, 18, 17, 11, 3, 999, Strategy::BtUltra2), + ], + [ + cp(14, 12, 13, 1, 5, 1, Strategy::Fast), + cp(14, 14, 15, 1, 5, 0, Strategy::Fast), + cp(14, 14, 15, 1, 4, 0, Strategy::Fast), + cp(14, 14, 15, 2, 4, 0, Strategy::DFast), + cp(14, 14, 14, 4, 4, 2, Strategy::Greedy), + cp(14, 14, 14, 3, 4, 4, Strategy::Lazy), + cp(14, 14, 14, 4, 4, 8, Strategy::Lazy2), + cp(14, 14, 14, 6, 4, 8, Strategy::Lazy2), + cp(14, 14, 14, 8, 4, 8, Strategy::Lazy2), + cp(14, 15, 14, 5, 4, 8, Strategy::BtLazy2), + cp(14, 15, 14, 9, 4, 8, Strategy::BtLazy2), + cp(14, 15, 14, 3, 4, 12, Strategy::BtOpt), + cp(14, 15, 14, 4, 3, 24, Strategy::BtOpt), + cp(14, 15, 14, 5, 3, 32, Strategy::BtUltra), + cp(14, 15, 15, 6, 3, 64, Strategy::BtUltra), + cp(14, 15, 15, 7, 3, 256, Strategy::BtUltra), + cp(14, 15, 15, 5, 3, 48, Strategy::BtUltra2), + cp(14, 15, 15, 6, 3, 128, Strategy::BtUltra2), + cp(14, 15, 15, 7, 3, 256, Strategy::BtUltra2), + cp(14, 15, 15, 8, 3, 256, Strategy::BtUltra2), + cp(14, 15, 15, 8, 3, 512, Strategy::BtUltra2), + cp(14, 15, 15, 9, 3, 512, Strategy::BtUltra2), + cp(14, 15, 15, 10, 3, 999, Strategy::BtUltra2), + ], +]; diff --git a/src/zstd/compress/presplit.rs b/src/zstd/compress/presplit.rs new file mode 100644 index 000000000000..6676175f0c1f --- /dev/null +++ b/src/zstd/compress/presplit.rs @@ -0,0 +1,184 @@ +//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. + +use super::hist; +use crate::error::Result; +use crate::util::SliceExt; + +/// What the C code needs of workspace, which counts in the size of a context. +pub(crate) const ZSTD_SLIPBLOCK_WORKSPACESIZE: usize = 8208; + +const THRESHOLD_PENALTY_RATE: u64 = 16; +const THRESHOLD_BASE: i32 = THRESHOLD_PENALTY_RATE as i32 - 2; +const THRESHOLD_PENALTY: i32 = 3; + +const HASHLENGTH: usize = 2; +const KNUTH: u32 = 0x9e37_79b9; + +const CHUNKSIZE: usize = 8 << 10; +const SEGMENT_SIZE: usize = 512; + +/// `Fingerprint`, with a table of `SIZE` events, which is `1 << hashLog`: the C code leaves the rest of its table at 0. +struct Fingerprint { + events: [u32; SIZE], + nb_events: usize, +} + +impl Fingerprint { + const fn new() -> Self { + Self { + events: [0; SIZE], + nb_events: 0, + } + } + + /// `hash2`: a table of 256 takes the first byte as it is, a larger one a hash of the 2 bytes. + #[inline(always)] + const fn hash2(pair: [u8; HASHLENGTH]) -> usize { + if SIZE == 256 { + let [first, _] = pair; + return first as usize; + } + ((u16::from_le_bytes(pair) as u32).wrapping_mul(KNUTH) >> (32 - SIZE.trailing_zeros())) + as usize + } + + /// `addEvents_generic`: `src` has at least 2 bytes. + #[inline(always)] + fn add_events(&mut self, src: &[u8]) { + let limit = src.len().saturating_sub(HASHLENGTH - 1); + let mut n = 0; + while n < limit { + if let Some(pair) = src.get(n..).and_then(<[u8]>::first_chunk) + && let Some(event) = self.events.get_mut(Self::hash2(*pair) & (SIZE - 1)) + { + *event += 1; + } + n += SAMPLING_RATE; + } + self.nb_events += limit / SAMPLING_RATE; + } + + /// `recordFingerprint_generic` + #[inline(always)] + fn record(&mut self, src: &[u8]) { + self.events.fill(0); + self.nb_events = 0; + self.add_events::(src); + } + + /// `fpDistance` + fn distance(&self, other: &Self) -> u64 { + let nb_events1 = self.nb_events as i64; + let nb_events2 = other.nb_events as i64; + self.events + .iter() + .zip(&other.events) + .map(|(&event1, &event2)| { + (i64::from(event1) * nb_events2 - i64::from(event2) * nb_events1).unsigned_abs() + }) + .sum() + } + + /// `compareFingerprints`: whether `newfp` is too different from this one. + fn differs_from(&self, newfp: &Self, penalty: i32) -> bool { + let p50 = self.nb_events as u64 * newfp.nb_events as u64; + let deviation = self.distance(newfp); + let threshold = p50 * (THRESHOLD_BASE + penalty) as u64 / THRESHOLD_PENALTY_RATE; + deviation >= threshold + } + + /// `mergeEvents` + fn merge(&mut self, newfp: &Self) { + for (acc, &event) in self.events.iter_mut().zip(&newfp.events) { + *acc += event; + } + self.nb_events += newfp.nb_events; + } +} + +/// `ZSTD_splitBlock_byChunks`: cuts before the first chunk that is too different from all there is before it. +#[inline(always)] +fn split_block_by_chunks(block: &[u8]) -> usize { + let mut past_events = Fingerprint::::new(); + let mut new_events = Fingerprint::::new(); + let mut penalty = THRESHOLD_PENALTY; + let (chunks, _) = block.as_chunks::(); + let Some((first, chunks)) = chunks.split_first() else { + return block.len(); + }; + + past_events.record::(first); + let mut pos = CHUNKSIZE; + for chunk in chunks { + new_events.record::(chunk); + if past_events.differs_from(&new_events, penalty) { + return pos; + } + past_events.merge(&new_events); + if penalty > 0 { + penalty -= 1; + } + pos += CHUNKSIZE; + } + block.len() +} + +fn split_block_by_chunks_43(block: &[u8]) -> usize { + split_block_by_chunks::<43, { 1 << 8 }>(block) +} + +fn split_block_by_chunks_11(block: &[u8]) -> usize { + split_block_by_chunks::<11, { 1 << 9 }>(block) +} + +fn split_block_by_chunks_5(block: &[u8]) -> usize { + split_block_by_chunks::<5, { 1 << 10 }>(block) +} + +fn split_block_by_chunks_1(block: &[u8]) -> usize { + split_block_by_chunks::<1, { 1 << 10 }>(block) +} + +/// `ZSTD_splitBlock_fromBorders`: cuts if the start and the end of the block are too different. In the middle, or at a +/// quarter from the border that the middle is clearly the most different from. +fn split_block_from_borders(block: &[u8]) -> Result { + fn segment(block: &[u8], start: usize) -> Result> { + let mut fingerprint = Fingerprint::new(); + hist::add(&mut fingerprint.events, block.span(start, SEGMENT_SIZE)?)?; + fingerprint.nb_events = SEGMENT_SIZE; + Ok(fingerprint) + } + + let block_size = block.len(); + let past_events = segment(block, 0)?; + let new_events = segment(block, block_size.wrapping_sub(SEGMENT_SIZE))?; + if !past_events.differs_from(&new_events, 0) { + return Ok(block_size); + } + + let middle_events = segment(block, (block_size / 2).wrapping_sub(SEGMENT_SIZE / 2))?; + let dist_from_begin = past_events.distance(&middle_events); + let dist_from_end = new_events.distance(&middle_events); + let min_distance = (SEGMENT_SIZE * SEGMENT_SIZE / 3) as u64; + if dist_from_begin.abs_diff(dist_from_end) < min_distance { + return Ok(64 << 10); + } + Ok(if dist_from_begin > dist_from_end { + 32 << 10 + } else { + 96 << 10 + }) +} + +/// `ZSTD_splitBlock`: `block` is 128 KB, and `level` is from 0 to 4: the higher, the more work is put in. Returns the +/// size of the first part, which is the whole if the block is better left in one piece. +pub(crate) fn split_block(block: &[u8], level: i32) -> Result { + debug_assert!(block.len() == 128 << 10); + Ok(match level { + 0 => split_block_from_borders(block)?, + 1 => split_block_by_chunks_43(block), + 2 => split_block_by_chunks_11(block), + 3 => split_block_by_chunks_5(block), + _ => split_block_by_chunks_1(block), + }) +} diff --git a/src/zstd/compress/sequences.rs b/src/zstd/compress/sequences.rs new file mode 100644 index 000000000000..d05a6abf85b4 --- /dev/null +++ b/src/zstd/compress/sequences.rs @@ -0,0 +1,308 @@ +//! `zstd_compress_sequences.c`: how the sequences of a block are encoded. + +use super::fse::{self, FSE_NCOUNTBOUND, FseCState, FseCTable, FseRepeat}; +use super::internal::{SeqDef, Strategy}; +use crate::common::bitstream::BitCStream; +use crate::common::{ + LL_BITS, LL_FSE_LOG, MAX_SEQ, ML_BITS, ML_FSE_LOG, OFF_FSE_LOG, SymbolEncodingType, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::SliceExt; + +/// `-log2(x / 256)` in units of 1/256 of a bit, for `x` from 1 to 255. The entry for 0 is not used. +#[rustfmt::skip] +static K_INVERSE_PROBABILITY_LOG256: [u32; 256] = [ + 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, 1130, 1100, 1073, 1047, + 1024, 1001, 980, 960, 941, 923, 906, 889, 874, 859, 844, 830, 817, 804, 791, 779, + 768, 756, 745, 734, 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, + 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, 535, 529, 523, 517, + 512, 506, 500, 495, 489, 484, 478, 473, 468, 463, 458, 453, 448, 443, 438, 434, + 429, 424, 420, 415, 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, + 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, 318, 315, 311, 308, + 305, 302, 298, 295, 292, 289, 286, 282, 279, 276, 273, 270, 267, 264, 261, 258, + 256, 253, 250, 247, 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, + 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, 182, 180, 178, 175, + 173, 171, 168, 166, 164, 162, 159, 157, 155, 153, 151, 149, 146, 144, 142, 140, + 138, 136, 134, 132, 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, + 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, 83, 82, 80, 78, + 76, 74, 73, 71, 69, 67, 66, 64, 62, 61, 59, 57, 55, 54, 52, 50, + 49, 47, 46, 44, 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, + 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, 5, 4, 2, 1, +]; + +/// What the C code has for a cost that could not be computed: an error code, which is more than any cost. +fn cost_or_error(cost: Result) -> usize { + cost.unwrap_or_else(|code| 0usize.wrapping_sub(code as usize)) +} + +/// `ZSTD_useLowProbCount`: with few sequences, counts of -1 hurt more than they help. +const fn use_low_prob_count(nb_seq: usize) -> bool { + nb_seq >= 2048 +} + +/// `ZSTD_NCountCost`: how many bytes the header of a table for `count` takes. +fn ncount_cost(count: &[u32], max: u32, nb_seq: usize, fse_log: u32) -> Result { + let mut wksp = [0u8; FSE_NCOUNTBOUND]; + let mut norm = [0i16; MAX_SEQ as usize + 1]; + let table_log = fse::optimal_table_log(fse_log, nb_seq, max); + fse::normalize_count( + &mut norm, + table_log, + count, + nb_seq, + max, + use_low_prob_count(nb_seq), + )?; + fse::write_ncount(&mut wksp, &norm, max, table_log) +} + +/// `ZSTD_entropyCost`: how many bits it takes to encode the distribution `count`, which adds up to `total`, with a +/// table made for it. +fn entropy_cost(count: &[u32], max: u32, total: usize) -> Result { + let mut cost: u32 = 0; + for &c in count.sub(..=max as usize)? { + let mut norm = 256u32.wrapping_mul(c) as usize / total.max(1); + if c != 0 && norm == 0 { + norm = 1; + } + cost = cost.wrapping_add(c.wrapping_mul(K_INVERSE_PROBABILITY_LOG256.at(norm)?)); + } + Ok((cost >> 8) as usize) +} + +/// `ZSTD_fseBitCost`: how many bits it takes to encode the distribution `count` with `ctable`, which is an error if +/// the table lacks one of the symbols. +pub(crate) fn fse_bit_cost(ctable: &FseCTable, count: &[u32], max: u32) -> Result { + const K_ACCURACY_LOG: u32 = 8; + let mut cost: usize = 0; + if ctable.max_symbol_value < max { + return Err(ErrorCode::Generic); + } + let bad_cost = (ctable.table_log + 1) << K_ACCURACY_LOG; + for (s, &c) in count.sub(..=max as usize)?.iter().enumerate() { + if c == 0 { + continue; + } + let bit_cost = ctable.bit_cost(s as u32, K_ACCURACY_LOG); + if bit_cost >= bad_cost { + return Err(ErrorCode::Generic); + } + cost += c as usize * bit_cost as usize; + } + Ok(cost >> K_ACCURACY_LOG) +} + +/// `ZSTD_crossEntropyCost`: how many bits it takes to encode the distribution `count` with a table made from `norm`. +pub(crate) fn cross_entropy_cost( + norm: &[i16], + accuracy_log: u32, + count: &[u32], + max: u32, +) -> Result { + let shift = 8 - accuracy_log; + let mut cost: usize = 0; + let len = max as usize + 1; + for (&n, &c) in norm.sub(..len)?.iter().zip(count.sub(..len)?) { + let norm_acc = if n != -1 { n as u32 } else { 1 }; + let norm256 = norm_acc << shift; + cost += c.wrapping_mul(K_INVERSE_PROBABILITY_LOG256.at(norm256 as usize)?) as usize; + } + Ok(cost >> 8) +} + +/// `ZSTD_selectEncodingType` +pub(crate) fn select_encoding_type( + repeat_mode: &mut FseRepeat, + count: &[u32], + max: u32, + most_frequent: usize, + nb_seq: usize, + fse_log: u32, + prev_ctable: &FseCTable, + default_norm: &[i16], + default_norm_log: u32, + is_default_allowed: bool, + strategy: Strategy, +) -> Result { + if most_frequent == nb_seq { + *repeat_mode = FseRepeat::None; + if is_default_allowed && nb_seq <= 2 { + // A run of 1 or 2 costs more than the predefined table does. + return Ok(SymbolEncodingType::Basic); + } + return Ok(SymbolEncodingType::Rle); + } + if strategy < Strategy::Lazy { + if is_default_allowed { + const STATIC_FSE_NB_SEQ_MAX: usize = 1000; + let mult = 10 - strategy as usize; + const BASE_LOG: usize = 3; + let dynamic_fse_nb_seq_min = ((1usize << default_norm_log) * mult) >> BASE_LOG; + if *repeat_mode == FseRepeat::Valid && nb_seq < STATIC_FSE_NB_SEQ_MAX { + return Ok(SymbolEncodingType::Repeat); + } + if nb_seq < dynamic_fse_nb_seq_min + || most_frequent < (nb_seq >> (default_norm_log - 1)) + { + *repeat_mode = FseRepeat::None; + return Ok(SymbolEncodingType::Basic); + } + } + } else { + let basic_cost = cost_or_error(if is_default_allowed { + cross_entropy_cost(default_norm, default_norm_log, count, max) + } else { + Err(ErrorCode::Generic) + }); + let repeat_cost = cost_or_error(if *repeat_mode != FseRepeat::None { + fse_bit_cost(prev_ctable, count, max) + } else { + Err(ErrorCode::Generic) + }); + let ncount_cost = cost_or_error(ncount_cost(count, max, nb_seq, fse_log)); + let compressed_cost = (ncount_cost << 3).wrapping_add(entropy_cost(count, max, nb_seq)?); + + if basic_cost <= repeat_cost && basic_cost <= compressed_cost { + *repeat_mode = FseRepeat::None; + return Ok(SymbolEncodingType::Basic); + } + if repeat_cost <= compressed_cost { + return Ok(SymbolEncodingType::Repeat); + } + } + *repeat_mode = FseRepeat::Check; + Ok(SymbolEncodingType::Compressed) +} + +/// `ZSTD_buildCTable`: makes the table for the codes `code_table`, of which `count` is the distribution, and writes +/// what the decoder needs to make the same one. Returns how many bytes that takes. +pub(crate) fn build_ctable( + dst: &mut [u8], + next_ctable: &mut FseCTable, + fse_log: u32, + ty: SymbolEncodingType, + count: &mut [u32], + max: u32, + code_table: &[u8], + default_norm: &[i16], + default_norm_log: u32, + default_max: u32, + prev_ctable: &FseCTable, +) -> Result { + match ty { + SymbolEncodingType::Rle => { + next_ctable.build_rle(max as u8)?; + let Some(first) = dst.first_mut() else { + return Err(ErrorCode::DstSizeTooSmall); + }; + *first = code_table.at(0)?; + Ok(1) + } + SymbolEncodingType::Repeat => { + next_ctable.clone_from(prev_ctable); + Ok(0) + } + SymbolEncodingType::Basic => { + next_ctable.build(default_norm, default_max, default_norm_log)?; + Ok(0) + } + SymbolEncodingType::Compressed => { + let mut norm = [0i16; MAX_SEQ as usize + 1]; + let nb_seq = code_table.len(); + let mut nb_seq_1 = nb_seq; + let table_log = fse::optimal_table_log(fse_log, nb_seq, max); + // The last code is the first state, which takes no transition. + let last_count = count.at_mut(usize::from(code_table.at(nb_seq.wrapping_sub(1))?))?; + if *last_count > 1 { + *last_count -= 1; + nb_seq_1 -= 1; + } + fse::normalize_count( + &mut norm, + table_log, + count, + nb_seq_1, + max, + use_low_prob_count(nb_seq_1), + )?; + let ncount_size = fse::write_ncount(dst, &norm, max, table_log)?; + next_ctable.build(&norm, max, table_log)?; + Ok(ncount_size) + } + } +} + +#[inline(always)] +fn bits_of(table: &[u8; N], code: u8) -> u32 { + u32::from(table.get(usize::from(code)).copied().unwrap_or(0)) +} + +/// `ZSTD_encodeSequences`: the four slices are of the same length, which is not 0. +pub(crate) fn encode_sequences( + dst: &mut [u8], + ctable_match_length: &FseCTable, + ml_code_table: &[u8], + ctable_offset_bits: &FseCTable, + of_code_table: &[u8], + ctable_lit_length: &FseCTable, + ll_code_table: &[u8], + sequences: &[SeqDef], +) -> Result { + let mut block_stream = BitCStream::new(dst).map_err(|_| ErrorCode::DstSizeTooSmall)?; + let ( + Some((&ml_last, ml_codes)), + Some((&of_last, of_codes)), + Some((&ll_last, ll_codes)), + Some((seq_last, sequences)), + ) = ( + ml_code_table.split_last(), + of_code_table.split_last(), + ll_code_table.split_last(), + sequences.split_last(), + ) + else { + return Err(ErrorCode::Generic); + }; + + let mut state_match_length = FseCState::new2(ctable_match_length, u32::from(ml_last)); + let mut state_offset_bits = FseCState::new2(ctable_offset_bits, u32::from(of_last)); + let mut state_lit_length = FseCState::new2(ctable_lit_length, u32::from(ll_last)); + block_stream.add_bits(u64::from(seq_last.lit_length), bits_of(&LL_BITS, ll_last)); + block_stream.add_bits(u64::from(seq_last.ml_base), bits_of(&ML_BITS, ml_last)); + block_stream.add_bits(u64::from(seq_last.off_base), u32::from(of_last)); + block_stream.flush_bits(); + + for (((seq, &ll_code), &of_code), &ml_code) in sequences + .iter() + .zip(ll_codes) + .zip(of_codes) + .zip(ml_codes) + .rev() + { + let ll_bits = bits_of(&LL_BITS, ll_code); + let of_bits = u32::from(of_code); + let ml_bits = bits_of(&ML_BITS, ml_code); + state_offset_bits.encode_symbol(&mut block_stream, u32::from(of_code)); + state_match_length.encode_symbol(&mut block_stream, u32::from(ml_code)); + state_lit_length.encode_symbol(&mut block_stream, u32::from(ll_code)); + if of_bits + ml_bits + ll_bits >= 64 - 7 - (LL_FSE_LOG + ML_FSE_LOG + OFF_FSE_LOG) { + block_stream.flush_bits(); + } + block_stream.add_bits(u64::from(seq.lit_length), ll_bits); + block_stream.add_bits(u64::from(seq.ml_base), ml_bits); + if of_bits + ml_bits + ll_bits > 56 { + block_stream.flush_bits(); + } + block_stream.add_bits(u64::from(seq.off_base), of_bits); + block_stream.flush_bits(); + } + + state_match_length.flush(&mut block_stream); + state_offset_bits.flush(&mut block_stream); + state_lit_length.flush(&mut block_stream); + + match block_stream.close() { + 0 => Err(ErrorCode::DstSizeTooSmall), + stream_size => Ok(stream_size), + } +} diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs new file mode 100644 index 000000000000..e5c1cfe24b9c --- /dev/null +++ b/src/zstd/compress/stream.rs @@ -0,0 +1,522 @@ +//! The public side of `zstd_compress.c`: `ZSTD_CCtx`, its parameters, `ZSTD_compressStream2` and `ZSTD_compress`. + +use std::sync::Arc; + +use super::cctx::{CDict, DictContentType, FrameCompressor, Input, Span}; +use super::internal::{BufId, Buffers, DictTableLoadMethod}; +use super::mt::MtCtx; +use super::params::{ + CCtxParams, CParamMode, ZSTD_CLEVEL_DEFAULT, ZSTDMT_JOBSIZE_MIN, compress_bound, + get_c_params_from_cctx_params, get_params_internal, is_update_authorized, + resolve_block_splitter_mode, resolve_enable_ldm, resolve_row_match_finder_mode, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, try_zeroed}; +use crate::{InBuffer, OutBuffer, ResetDirective}; + +/// `ZSTD_EndDirective` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum EndDirective { + /// Collect more input: the encoder decides when to output. + Continue = 0, + /// Output all that was consumed, so that a decoder can get it. + Flush = 1, + /// Flush, and close the frame. + End = 2, +} + +/// `ZSTD_cStreamStage` +#[derive(Clone, Copy, PartialEq, Eq)] +enum StreamStage { + Init, + Load, + Flush, +} + +/// `ZSTD_CCtx` +pub struct CCtx { + frame: FrameCompressor, + c_params_changed: bool, + requested_params: CCtxParams, + in_buff: Vec, + in_to_compress: usize, + in_buff_pos: usize, + in_buff_target: usize, + out_buff: Vec, + out_buff_content_size: usize, + out_buff_flushed_size: usize, + stream_stage: StreamStage, + frame_ended: bool, + /// `localDict.dict` + local_dict: Option>, + /// `localDict.cdict`, which is made when the first frame starts. + cdict: Option>, + mtctx: Option>, +} + +impl Default for CCtx { + fn default() -> Self { + Self::new() + } +} + +/// `ZSTD_limitCopy` +fn limit_copy(dst: &mut [u8], src: &[u8]) -> usize { + let length = dst.len().min(src.len()); + for (to, from) in dst.iter_mut().zip(src) { + *to = *from; + } + length +} + +fn resize_buffer(buffer: &mut Vec, len: usize) -> Result<()> { + if buffer.len() != len { + *buffer = Vec::new(); + *buffer = try_zeroed(len)?; + } + Ok(()) +} + +impl CCtx { + /// `ZSTD_createCCtx` + pub fn new() -> Self { + Self { + frame: FrameCompressor::new(), + c_params_changed: false, + requested_params: CCtxParams::new(ZSTD_CLEVEL_DEFAULT), + in_buff: Vec::new(), + in_to_compress: 0, + in_buff_pos: 0, + in_buff_target: 0, + out_buff: Vec::new(), + out_buff_content_size: 0, + out_buff_flushed_size: 0, + stream_stage: StreamStage::Init, + frame_ended: false, + local_dict: None, + cdict: None, + mtctx: None, + } + } + + /// `ZSTD_CCtx_setParameter`: `param` is one of [`super::params::c_param`]. + pub fn set_parameter(&mut self, param: i32, value: i32) -> Result { + if self.stream_stage != StreamStage::Init { + if is_update_authorized(param) { + self.c_params_changed = true; + } else { + return Err(ErrorCode::StageWrong); + } + } + self.requested_params.set_parameter(param, value) + } + + /// `ZSTD_CCtx_setPledgedSrcSize` + pub fn set_pledged_src_size(&mut self, pledged_src_size: u64) -> Result<()> { + if self.stream_stage != StreamStage::Init { + return Err(ErrorCode::StageWrong); + } + self.frame.pledged_src_size_plus_one = pledged_src_size.wrapping_add(1); + Ok(()) + } + + /// `ZSTD_clearAllDicts` + fn clear_all_dicts(&mut self) { + self.local_dict = None; + self.cdict = None; + } + + /// `ZSTD_CCtx_loadDictionary`: every frame from now on starts with `dict`, of which a copy is kept. If it is empty, + /// with no dictionary. + pub fn load_dictionary(&mut self, dict: &[u8]) -> Result<()> { + if self.stream_stage != StreamStage::Init { + return Err(ErrorCode::StageWrong); + } + self.clear_all_dicts(); + if dict.is_empty() { + return Ok(()); + } + let mut copy = Vec::new(); + copy.try_reserve_exact(dict.len()) + .map_err(|_| ErrorCode::MemoryAllocation)?; + copy.extend_from_slice(dict); + self.local_dict = Some(Arc::from(copy)); + Ok(()) + } + + /// `ZSTD_CCtx_reset` + pub fn reset(&mut self, reset: ResetDirective) -> Result<()> { + if matches!( + reset, + ResetDirective::SessionOnly | ResetDirective::SessionAndParameters + ) { + self.stream_stage = StreamStage::Init; + self.frame.pledged_src_size_plus_one = 0; + } + if matches!( + reset, + ResetDirective::Parameters | ResetDirective::SessionAndParameters + ) { + if self.stream_stage != StreamStage::Init { + return Err(ErrorCode::StageWrong); + } + self.clear_all_dicts(); + self.requested_params = CCtxParams::new(ZSTD_CLEVEL_DEFAULT); + } + Ok(()) + } + + fn reset_session(&mut self) { + self.stream_stage = StreamStage::Init; + self.frame.pledged_src_size_plus_one = 0; + } + + /// `ZSTD_initLocalDict` + fn init_local_dict(&mut self) -> Result<()> { + let Some(dict) = &self.local_dict else { + return Ok(()); + }; + if self.cdict.is_some() { + return Ok(()); + } + // Whatever is wrong with the dictionary, the C code only knows that it got no `ZSTD_CDict`. + let cdict = CDict::new(Arc::clone(dict), &self.requested_params) + .map_err(|_| ErrorCode::MemoryAllocation)?; + self.cdict = Some(Arc::new(cdict)); + Ok(()) + } + + /// `ZSTD_CCtx_init_compressStream2` + fn init_compress_stream2(&mut self, end_op: EndDirective, in_size: usize) -> Result<()> { + let mut params = self.requested_params; + self.init_local_dict()?; + if end_op == EndDirective::End { + // All the input is there. + self.frame.pledged_src_size_plus_one = in_size as u64 + 1; + } + let pledged_src_size = self.frame.pledged_src_size_plus_one.wrapping_sub(1); + let cdict = self.cdict.as_deref(); + + { + let dict_size = cdict.map_or(0, |cdict| cdict.dict_content().len()); + // `ZSTD_getCParamMode` + let mode = match cdict { + Some(cdict) if cdict.should_attach(¶ms, pledged_src_size) => { + CParamMode::AttachDict + } + _ => CParamMode::NoAttachDict, + }; + params.c_params = + get_c_params_from_cctx_params(¶ms, pledged_src_size, dict_size, mode); + } + + params.post_block_splitter = + resolve_block_splitter_mode(params.post_block_splitter, ¶ms.c_params); + params.ldm_params.enable_ldm = + resolve_enable_ldm(params.ldm_params.enable_ldm, ¶ms.c_params); + params.use_row_match_finder = + resolve_row_match_finder_mode(params.use_row_match_finder, ¶ms.c_params); + + if pledged_src_size <= ZSTDMT_JOBSIZE_MIN as u64 { + // Not worth several threads. + params.nb_workers = 0; + } + if params.nb_workers > 0 { + if self.mtctx.is_none() { + self.mtctx = Some(Box::new(MtCtx::new(params.nb_workers as u32)?)); + } + let mtctx = self.mtctx.as_mut().ok_or(ErrorCode::MemoryAllocation)?; + mtctx.init_c_stream_internal(self.cdict.clone(), params, pledged_src_size)?; + self.frame.dict_id = cdict.map_or(0, CDict::dict_id); + self.frame.dict_content_size = cdict.map_or(0, |cdict| cdict.dict_content().len()); + self.frame.consumed_src_size = 0; + self.frame.produced_c_size = 0; + self.stream_stage = StreamStage::Load; + self.frame.applied_params = params; + } else { + let input = Input { + bufs: Buffers { + in_buff: &[], + user: &[], + dict: cdict.map_or(&[], CDict::dict_content), + }, + cdict, + }; + let sizes = self.frame.begin( + &input, + None, + DictContentType::Auto, + DictTableLoadMethod::Fast, + ¶ms, + pledged_src_size, + true, + )?; + resize_buffer(&mut self.in_buff, sizes.buff_in_size)?; + resize_buffer(&mut self.out_buff, sizes.buff_out_size)?; + self.in_to_compress = 0; + self.in_buff_pos = 0; + // One more byte, to tell an input of exactly one block from a longer one. + self.in_buff_target = self.frame.block_size_max + + usize::from(self.frame.block_size_max as u64 == pledged_src_size); + self.out_buff_content_size = 0; + self.out_buff_flushed_size = 0; + self.stream_stage = StreamStage::Load; + self.frame_ended = false; + } + Ok(()) + } + + /// `ZSTD_nextInputSizeHint` + fn next_input_size_hint(&self) -> usize { + match self.in_buff_target - self.in_buff_pos { + 0 => self.frame.block_size_max, + hint_in_size => hint_in_size, + } + } + + /// `ZSTD_compressStream_generic` + fn compress_stream_generic( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + flush_mode: EndDirective, + ) -> Result { + let iend = input.src.len(); + let mut ip = input.pos; + let mut op = output.pos; + let cdict = self.cdict.clone(); + let cdict = cdict.as_deref(); + let dict = cdict.map_or(&[][..], CDict::dict_content); + + loop { + match self.stream_stage { + StreamStage::Init => return Err(ErrorCode::InitMissing), + + StreamStage::Load => { + if flush_mode == EndDirective::End + && output.dst.len() - op >= compress_bound(iend - ip) + && self.in_buff_pos == 0 + { + // There is room for the worst case: no need for the buffers. + let c_size = self.frame.compress_end( + output.dst.sub_mut(op..)?, + &Input { + bufs: Buffers { + in_buff: &self.in_buff, + user: input.src, + dict, + }, + cdict, + }, + Span { + buf: BufId::User, + start: ip, + end: iend, + }, + )?; + ip = iend; + op += c_size; + self.frame_ended = true; + self.reset_session(); + break; + } + { + let loaded = limit_copy( + self.in_buff + .sub_mut(self.in_buff_pos..self.in_buff_target)?, + input.src.sub(ip..)?, + ); + self.in_buff_pos += loaded; + ip += loaded; + if flush_mode == EndDirective::Continue + && self.in_buff_pos < self.in_buff_target + { + // Not enough for a block yet. + break; + } + if flush_mode == EndDirective::Flush + && self.in_buff_pos == self.in_to_compress + { + // Nothing to flush. + break; + } + } + let i_size = self.in_buff_pos - self.in_to_compress; + let direct = output.dst.len() - op >= compress_bound(i_size); + let c_dst = if direct { + output.dst.sub_mut(op..)? + } else { + self.out_buff.as_mut_slice() + }; + let last_block = flush_mode == EndDirective::End && ip == iend; + let frame_input = Input { + bufs: Buffers { + in_buff: &self.in_buff, + user: &[], + dict, + }, + cdict, + }; + let src = Span { + buf: BufId::InBuff, + start: self.in_to_compress, + end: self.in_buff_pos, + }; + let c_size = if last_block { + self.frame.compress_end(c_dst, &frame_input, src)? + } else { + self.frame.compress_continue(c_dst, &frame_input, src)? + }; + self.frame_ended = last_block; + // The next block goes after this one, or at the start if there is no room left. + self.in_buff_target = self.in_buff_pos + self.frame.block_size_max; + if self.in_buff_target > self.in_buff.len() { + self.in_buff_pos = 0; + self.in_buff_target = self.frame.block_size_max; + } + self.in_to_compress = self.in_buff_pos; + if direct { + op += c_size; + if self.frame_ended { + self.reset_session(); + break; + } + continue; + } + self.out_buff_content_size = c_size; + self.out_buff_flushed_size = 0; + self.stream_stage = StreamStage::Flush; + } + + StreamStage::Flush => { + let to_flush = self.out_buff_content_size - self.out_buff_flushed_size; + let flushed = limit_copy( + output.dst.sub_mut(op..)?, + self.out_buff.span(self.out_buff_flushed_size, to_flush)?, + ); + op += flushed; + self.out_buff_flushed_size += flushed; + if to_flush != flushed { + // The output is full. + break; + } + self.out_buff_content_size = 0; + self.out_buff_flushed_size = 0; + if self.frame_ended { + self.reset_session(); + break; + } + self.stream_stage = StreamStage::Load; + } + } + } + + input.pos = ip; + output.pos = op; + if self.frame_ended { + return Ok(0); + } + Ok(self.next_input_size_hint()) + } + + /// `ZSTD_compressStream2`: returns the least number of bytes that are left to flush, which for + /// [`EndDirective::End`] is 0 once the frame is complete. + pub fn compress_stream2( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + end_op: EndDirective, + ) -> Result { + if output.pos > output.dst.len() { + return Err(ErrorCode::DstSizeTooSmall); + } + if input.pos > input.src.len() { + return Err(ErrorCode::SrcSizeWrong); + } + + if self.stream_stage == StreamStage::Init { + self.init_compress_stream2(end_op, input.src.len() - input.pos)?; + } + + if self.frame.applied_params.nb_workers > 0 { + let mtctx = self.mtctx.as_mut().ok_or(ErrorCode::Generic)?; + if self.c_params_changed { + mtctx.update_c_params_while_compressing(&self.requested_params); + self.c_params_changed = false; + } + loop { + let ipos = input.pos; + let opos = output.pos; + let flush_min = mtctx.compress_stream_generic(output, input, end_op); + self.frame.consumed_src_size += (input.pos - ipos) as u64; + self.frame.produced_c_size += (output.pos - opos) as u64; + if flush_min.is_err() || (end_op == EndDirective::End && flush_min == Ok(0)) { + self.stream_stage = StreamStage::Init; + self.frame.pledged_src_size_plus_one = 0; + } + let flush_min = flush_min?; + + if end_op == EndDirective::Continue { + // Some progress is enough. Otherwise, wait for a job to make some. + if input.pos != ipos + || output.pos != opos + || input.pos == input.src.len() + || output.pos == output.dst.len() + { + return Ok(flush_min); + } + } else if flush_min == 0 || output.pos == output.dst.len() { + // Flushing goes on for as long as there is room. + return Ok(flush_min); + } + } + } + self.compress_stream_generic(output, input, end_op)?; + Ok(self.out_buff_content_size - self.out_buff_flushed_size) + } +} + +/// `ZSTD_compress`: returns the size of the frame. +pub fn compress(dst: &mut [u8], src: &[u8], compression_level: i32) -> Result { + let mut frame = FrameCompressor::new(); + let params = get_params_internal( + compression_level, + src.len() as u64, + 0, + CParamMode::NoAttachDict, + ); + let cctx_params = CCtxParams::from_params( + ¶ms, + if compression_level == 0 { + ZSTD_CLEVEL_DEFAULT + } else { + compression_level + }, + ); + let input = Input { + bufs: Buffers { + in_buff: &[], + user: src, + dict: &[], + }, + cdict: None, + }; + frame.begin( + &input, + None, + DictContentType::Auto, + DictTableLoadMethod::Fast, + &cctx_params, + src.len() as u64, + false, + )?; + frame.compress_end( + dst, + &input, + Span { + buf: BufId::User, + start: 0, + end: src.len(), + }, + ) +} diff --git a/src/zstd/compress/superblock.rs b/src/zstd/compress/superblock.rs new file mode 100644 index 000000000000..56134573a1ee --- /dev/null +++ b/src/zstd/compress/superblock.rs @@ -0,0 +1,601 @@ +//! `zstd_compress_superblock.c`: a block written as several smaller ones, of about the size asked for. + +use super::block_entropy::{build_block_entropy_stats, write_nb_seq}; +use super::fse::FseCTable; +use super::internal::{ + CompressedBlockState, EntropyCTables, EntropyCTablesMetadata, FseCTables, FseCTablesMetadata, + HufCTables, HufCTablesMetadata, SeqDef, SeqStore, no_compress_block, update_rep, +}; +use super::literals::{compress_rle_literals_block, no_compress_literals}; +use super::params::CCtxParams; +use super::sequences::{cross_entropy_cost, encode_sequences, fse_bit_cost}; +use super::{hist, huf}; +use crate::common::{ + BlockType, DEFAULT_MAX_OFF, HUF_SYMBOLVALUE_MAX, LL_BITS, LL_DEFAULT_NORM, + LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MINMATCH, ML_BITS, ML_DEFAULT_NORM, + ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, SymbolEncodingType, + ZSTD_BLOCKHEADERSIZE, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, write_le24, write_le32}; + +const ZSTD_TARGETCBLOCKSIZE_MIN: usize = 1340; + +/// `ZSTD_compressSubBlock_literal`: the literals section of a sub-block. Returns its size, which is 0 if it could not +/// be compressed, and whether it holds the description of the Huffman table. +fn compress_sub_block_literal( + huf_table: &huf::HufCTable, + huf_metadata: &HufCTablesMetadata, + literals: &[u8], + dst: &mut [u8], + write_entropy: bool, +) -> Result<(usize, bool)> { + let lit_size = literals.len(); + let header = if write_entropy { 200 } else { 0 }; + let lh_size = 3 + + usize::from(lit_size >= (1 << 10) - header) + + usize::from(lit_size >= (16 << 10) - header); + let mut op = lh_size; + let single_stream = lh_size == 3; + let h_type = if write_entropy { + huf_metadata.h_type + } else { + SymbolEncodingType::Repeat + }; + let mut c_lit_size = 0usize; + + if lit_size == 0 || huf_metadata.h_type == SymbolEncodingType::Basic { + return Ok((no_compress_literals(dst, literals)?, false)); + } else if huf_metadata.h_type == SymbolEncodingType::Rle { + return Ok((compress_rle_literals_block(dst, literals)?, false)); + } + + if write_entropy && huf_metadata.h_type == SymbolEncodingType::Compressed { + let description = huf_metadata + .huf_des_buffer + .sub(..huf_metadata.huf_des_size)?; + dst.write_at(op, description)?; + op += description.len(); + c_lit_size += description.len(); + } + + { + let out = dst.sub_mut(op..)?; + let c_size = if single_stream { + huf::compress_1x_using_ctable(out, literals, huf_table) + } else { + huf::compress_4x_using_ctable(out, literals, huf_table) + }; + let Ok(c_size @ 1..) = c_size else { + return Ok((0, false)); + }; + op += c_size; + c_lit_size += c_size; + // With the table already written, literals that expand can still be left as they are. + if !write_entropy && c_lit_size >= lit_size { + return Ok((no_compress_literals(dst, literals)?, false)); + } + // The size of the header was a guess. + if lh_size + < 3 + usize::from(c_lit_size >= 1 << 10) + usize::from(c_lit_size >= 16 << 10) + { + return Ok((no_compress_literals(dst, literals)?, false)); + } + } + + let h_type = h_type as u32; + let lit = lit_size as u32; + let c_lit = c_lit_size as u32; + match lh_size { + 3 => { + let lhc = h_type + (u32::from(!single_stream) << 2) + (lit << 4) + (c_lit << 14); + write_le24(dst, 0, lhc)?; + } + 4 => { + let lhc = h_type + (2 << 2) + (lit << 4) + (c_lit << 18); + write_le32(dst, 0, lhc)?; + } + _ => { + let lhc = h_type + (3 << 2) + (lit << 4) + (c_lit << 22); + write_le32(dst, 0, lhc)?; + dst.put(4, (c_lit_size >> 10) as u8)?; + } + } + Ok((op, true)) +} + +/// `ZSTD_seqDecompressedSize`: how many bytes the `nb_seqs` sequences from `first` on stand for, with `lit_size` bytes +/// of literals. +fn seq_decompressed_size( + seq_store: &SeqStore, + first: usize, + nb_seqs: usize, + lit_size: usize, +) -> Result { + let mut match_length_sum = 0usize; + for n in first..first + nb_seqs { + match_length_sum += seq_store.get_sequence_length(n)?.match_length as usize; + } + Ok(match_length_sum + lit_size) +} + +/// The codes of some sequences. +#[derive(Clone, Copy)] +struct Codes<'a> { + ll: &'a [u8], + ml: &'a [u8], + of: &'a [u8], +} + +/// `ZSTD_compressSubBlock_sequences`: the sequences section of a sub-block. Returns its size, which is 0 if it could +/// not be compressed, and whether it holds the descriptions of the tables. +fn compress_sub_block_sequences( + fse_tables: &FseCTables, + fse_metadata: &FseCTablesMetadata, + sequences: &[SeqDef], + codes: Codes<'_>, + dst: &mut [u8], + write_entropy: bool, +) -> Result<(usize, bool)> { + let nb_seq = sequences.len(); + if dst.len() < 3 + 1 { + return Err(ErrorCode::DstSizeTooSmall); + } + let mut op = write_nb_seq(dst, nb_seq)?; + if nb_seq == 0 { + return Ok((op, false)); + } + + let seq_head = op; + op += 1; + + if write_entropy { + dst.put( + seq_head, + (((fse_metadata.ll_type as u32) << 6) + + ((fse_metadata.of_type as u32) << 4) + + ((fse_metadata.ml_type as u32) << 2)) as u8, + )?; + let tables = fse_metadata + .fse_tables_buffer + .sub(..fse_metadata.fse_tables_size)?; + dst.write_at(op, tables)?; + op += tables.len(); + } else { + let repeat = SymbolEncodingType::Repeat as u32; + dst.put(seq_head, ((repeat << 6) + (repeat << 4) + (repeat << 2)) as u8)?; + } + + let bitstream_size = encode_sequences( + dst.sub_mut(op..)?, + &fse_tables.matchlength_c_table, + codes.ml, + &fse_tables.offcode_c_table, + codes.of, + &fse_tables.litlength_c_table, + codes.ll, + sequences, + )?; + op += bitstream_size; + // Decoders up to 1.3.4 fail on a table header with fewer than 4 bytes after it, + if write_entropy + && fse_metadata.last_count_size != 0 + && fse_metadata.last_count_size + bitstream_size < 4 + { + return Ok((0, false)); + } + // and those up to 1.4.0 on a sequences section of fewer than 4 bytes. + if op - seq_head < 4 { + return Ok((0, false)); + } + Ok((op, true)) +} + +/// What [`compress_sub_block`] wrote. +struct SubBlock { + /// 0 if it could not be compressed. + c_size: usize, + lit_entropy_written: bool, + seq_entropy_written: bool, +} + +/// `ZSTD_compressSubBlock` +fn compress_sub_block( + entropy: &EntropyCTables, + entropy_metadata: &EntropyCTablesMetadata, + sequences: &[SeqDef], + literals: &[u8], + codes: Codes<'_>, + dst: &mut [u8], + write_lit_entropy: bool, + write_seq_entropy: bool, + last_block: bool, +) -> Result { + let mut result = SubBlock { + c_size: 0, + lit_entropy_written: false, + seq_entropy_written: false, + }; + let mut op = ZSTD_BLOCKHEADERSIZE; + { + let (c_lit_size, written) = compress_sub_block_literal( + &entropy.huf.c_table, + &entropy_metadata.huf_metadata, + literals, + dst.sub_mut(op..)?, + write_lit_entropy, + )?; + result.lit_entropy_written = written; + if c_lit_size == 0 { + return Ok(result); + } + op += c_lit_size; + } + { + let (c_seq_size, written) = compress_sub_block_sequences( + &entropy.fse, + &entropy_metadata.fse_metadata, + sequences, + codes, + dst.sub_mut(op..)?, + write_seq_entropy, + )?; + result.seq_entropy_written = written; + if c_seq_size == 0 { + return Ok(result); + } + op += c_seq_size; + } + let c_size = op - ZSTD_BLOCKHEADERSIZE; + let c_block_header24 = + u32::from(last_block) + ((BlockType::Compressed as u32) << 1) + ((c_size as u32) << 3); + write_le24(dst, 0, c_block_header24)?; + result.c_size = op; + Ok(result) +} + +/// `ZSTD_estimateSubBlockSize_literal` +fn estimate_sub_block_size_literal( + literals: &[u8], + huf: &HufCTables, + huf_metadata: &HufCTablesMetadata, + write_entropy: bool, +) -> usize { + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + let mut max_symbol_value = HUF_SYMBOLVALUE_MAX; + const LITERAL_SECTION_HEADER_SIZE: usize = 3; + match huf_metadata.h_type { + SymbolEncodingType::Basic => literals.len(), + SymbolEncodingType::Rle => 1, + SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { + if hist::count_wksp(&mut count, &mut max_symbol_value, literals).is_err() { + return literals.len(); + } + let mut c_lit_size_estimate = + huf::estimate_compressed_size(&huf.c_table, &count, max_symbol_value); + if write_entropy { + c_lit_size_estimate += huf_metadata.huf_des_size; + } + c_lit_size_estimate + LITERAL_SECTION_HEADER_SIZE + } + } +} + +/// `ZSTD_estimateSubBlockSize_symbolType` +fn estimate_sub_block_size_symbol_type( + ty: SymbolEncodingType, + code_table: &[u8], + max_code: u32, + fse_c_table: &FseCTable, + additional_bits: Option<&[u8]>, + default_norm: &[i16], + default_norm_log: u32, + default_max: u32, +) -> Result { + let mut count = [0u32; MAX_SEQ as usize + 1]; + let mut max = max_code; + hist::count_fast_wksp(&mut count, &mut max, code_table)?; + let estimate_in_bits = match ty { + SymbolEncodingType::Basic if max <= default_max => { + cross_entropy_cost(default_norm, default_norm_log, &count, max) + } + SymbolEncodingType::Basic => Err(ErrorCode::Generic), + SymbolEncodingType::Rle => Ok(0), + SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { + fse_bit_cost(fse_c_table, &count, max) + } + }; + let Ok(mut estimate_in_bits) = estimate_in_bits else { + return Ok(code_table.len() * 10); + }; + for &code in code_table { + estimate_in_bits += usize::from(match additional_bits { + Some(bits) => bits.at(usize::from(code))?, + None => code, + }); + } + Ok(estimate_in_bits / 8) +} + +/// `ZSTD_estimateSubBlockSize_sequences` +fn estimate_sub_block_size_sequences( + codes: Codes<'_>, + fse_tables: &FseCTables, + fse_metadata: &FseCTablesMetadata, + write_entropy: bool, +) -> Result { + const SEQUENCES_SECTION_HEADER_SIZE: usize = 3; + if codes.of.is_empty() { + return Ok(SEQUENCES_SECTION_HEADER_SIZE); + } + let mut c_seq_size_estimate = estimate_sub_block_size_symbol_type( + fse_metadata.of_type, + codes.of, + MAX_OFF, + &fse_tables.offcode_c_table, + None, + &OF_DEFAULT_NORM, + OF_DEFAULT_NORM_LOG, + DEFAULT_MAX_OFF, + )?; + c_seq_size_estimate += estimate_sub_block_size_symbol_type( + fse_metadata.ll_type, + codes.ll, + MAX_LL, + &fse_tables.litlength_c_table, + Some(&LL_BITS), + &LL_DEFAULT_NORM, + LL_DEFAULT_NORM_LOG, + MAX_LL, + )?; + c_seq_size_estimate += estimate_sub_block_size_symbol_type( + fse_metadata.ml_type, + codes.ml, + MAX_ML, + &fse_tables.matchlength_c_table, + Some(&ML_BITS), + &ML_DEFAULT_NORM, + ML_DEFAULT_NORM_LOG, + MAX_ML, + )?; + if write_entropy { + c_seq_size_estimate += fse_metadata.fse_tables_size; + } + Ok(c_seq_size_estimate + SEQUENCES_SECTION_HEADER_SIZE) +} + +/// `ZSTD_needSequenceEntropyTables` +fn need_sequence_entropy_tables(fse_metadata: &FseCTablesMetadata) -> bool { + [ + fse_metadata.ll_type, + fse_metadata.ml_type, + fse_metadata.of_type, + ] + .iter() + .any(|ty| matches!(ty, SymbolEncodingType::Compressed | SymbolEncodingType::Rle)) +} + +/// `countLiterals` +fn count_literals(seq_store: &SeqStore, first: usize, seq_count: usize) -> Result { + let mut total = 0usize; + for n in first..first + seq_count { + total += seq_store.get_sequence_length(n)?.lit_length as usize; + } + Ok(total) +} + +const BYTESCALE: usize = 256; + +/// `sizeBlockSequences`: how many of the sequences `sp`, of which there is at least one, fit the budget. +fn size_block_sequences( + sp: &[SeqDef], + target_budget: usize, + avg_lit_cost: usize, + avg_seq_cost: usize, + first_sub_block: bool, +) -> Result { + let (first, rest) = sp.split_first().ok_or(ErrorCode::Generic)?; + // The first sub-block also has the tables. + let header_size = usize::from(first_sub_block) * 120 * BYTESCALE; + let mut budget = header_size; + budget += usize::from(first.lit_length) * avg_lit_cost + avg_seq_cost; + if budget > target_budget { + return Ok(1); + } + let mut in_size = usize::from(first.lit_length) + usize::from(first.ml_base) + MINMATCH as usize; + + let mut n = 1; + for seq in rest { + budget += usize::from(seq.lit_length) * avg_lit_cost + avg_seq_cost; + in_size += usize::from(seq.lit_length) + usize::from(seq.ml_base) + MINMATCH as usize; + // Only stop where the sub-block is expected to compress. + if budget > target_budget && budget < in_size * BYTESCALE { + break; + } + n += 1; + } + Ok(n) +} + +/// `ZSTD_compressSubBlock_multi` +fn compress_sub_block_multi( + seq_store: &SeqStore, + prev_c_block: &CompressedBlockState, + next_c_block: &mut CompressedBlockState, + entropy_metadata: &EntropyCTablesMetadata, + cctx_params: &CCtxParams, + dst: &mut [u8], + src: &[u8], + last_block: bool, +) -> Result { + let sequences = seq_store.sequences.as_slice(); + let nb_seqs = sequences.len(); + let mut sp = 0usize; + let literals = seq_store.literals(); + let nb_literals = literals.len(); + let mut lp = 0usize; + let mut ip = 0usize; + let mut op = 0usize; + let target_c_block_size = ZSTD_TARGETCBLOCKSIZE_MIN.max(cctx_params.target_c_block_size); + let mut write_lit_entropy = + entropy_metadata.huf_metadata.h_type == SymbolEncodingType::Compressed; + let mut write_seq_entropy = true; + let codes = |from: usize, count: usize| -> Result> { + Ok(Codes { + ll: seq_store.ll_code.span(from, count)?, + ml: seq_store.ml_code.span(from, count)?, + of: seq_store.of_code.span(from, count)?, + }) + }; + + if nb_seqs > 0 { + let est_lit_size = estimate_sub_block_size_literal( + literals, + &next_c_block.entropy.huf, + &entropy_metadata.huf_metadata, + write_lit_entropy, + ); + let est_block_size = estimate_sub_block_size_sequences( + codes(0, nb_seqs)?, + &next_c_block.entropy.fse, + &entropy_metadata.fse_metadata, + write_seq_entropy, + )? + est_lit_size + + ZSTD_BLOCKHEADERSIZE; + let avg_lit_cost = if nb_literals != 0 { + (est_lit_size * BYTESCALE) / nb_literals + } else { + BYTESCALE + }; + let avg_seq_cost = ((est_block_size - est_lit_size) * BYTESCALE) / nb_seqs; + let nb_sub_blocks = ((est_block_size + target_c_block_size / 2) / target_c_block_size).max(1); + let avg_block_budget = (est_block_size * BYTESCALE) / nb_sub_blocks; + if est_block_size > src.len() { + return Ok(0); + } + + for n in 0..nb_sub_blocks - 1 { + let seq_count = size_block_sequences( + sequences.sub(sp..)?, + avg_block_budget, + avg_lit_cost, + avg_seq_cost, + n == 0, + )?; + // The last sequences go with the last literals. + if sp + seq_count == nb_seqs { + break; + } + let lit_size = count_literals(seq_store, sp, seq_count)?; + let decompressed_size = seq_decompressed_size(seq_store, sp, seq_count, lit_size)?; + let sub_block = compress_sub_block( + &next_c_block.entropy, + entropy_metadata, + sequences.span(sp, seq_count)?, + literals.span(lp, lit_size)?, + codes(sp, seq_count)?, + dst.sub_mut(op..)?, + write_lit_entropy, + write_seq_entropy, + false, + )?; + if sub_block.c_size > 0 && sub_block.c_size < decompressed_size { + ip += decompressed_size; + lp += lit_size; + op += sub_block.c_size; + if sub_block.lit_entropy_written { + write_lit_entropy = false; + } + if sub_block.seq_entropy_written { + write_seq_entropy = false; + } + sp += seq_count; + } + } + } + + { + let lit_size = nb_literals - lp; + let seq_count = nb_seqs - sp; + let decompressed_size = seq_decompressed_size(seq_store, sp, seq_count, lit_size)?; + let sub_block = compress_sub_block( + &next_c_block.entropy, + entropy_metadata, + sequences.span(sp, seq_count)?, + literals.span(lp, lit_size)?, + codes(sp, seq_count)?, + dst.sub_mut(op..)?, + write_lit_entropy, + write_seq_entropy, + last_block, + )?; + if sub_block.c_size > 0 && sub_block.c_size < decompressed_size { + ip += decompressed_size; + op += sub_block.c_size; + if sub_block.lit_entropy_written { + write_lit_entropy = false; + } + if sub_block.seq_entropy_written { + write_seq_entropy = false; + } + sp += seq_count; + } + } + + if write_lit_entropy { + // The table was not written, so the next block cannot use it again. + next_c_block.entropy.huf.clone_from(&prev_c_block.entropy.huf); + } + if write_seq_entropy && need_sequence_entropy_tables(&entropy_metadata.fse_metadata) { + // Neither were these, and there is no going back to the previous ones. + return Ok(0); + } + + if ip < src.len() { + // What did not compress. + op += no_compress_block(dst.sub_mut(op..)?, src.sub(ip..)?, last_block)?; + if sp < nb_seqs { + // The sequences that were dropped did not update the repeat offsets. + let mut rep = prev_c_block.rep; + for (n, seq) in sequences.sub(..sp)?.iter().enumerate() { + let ll0 = seq_store.get_sequence_length(n)?.lit_length == 0; + update_rep(&mut rep, seq.off_base, ll0); + } + next_c_block.rep = rep; + } + } + Ok(op) +} + +/// `ZSTD_compressSuperBlock`: writes the block `src`, of which `seq_store` has the sequences, header included. Returns +/// 0 if it is better left uncompressed. +pub(crate) fn compress_super_block( + seq_store: &mut SeqStore, + prev_c_block: &CompressedBlockState, + next_c_block: &mut CompressedBlockState, + cctx_params: &CCtxParams, + dst: &mut [u8], + src: &[u8], + last_block: bool, +) -> Result { + let mut entropy_metadata = EntropyCTablesMetadata::new(); + let whole = seq_store.whole(); + build_block_entropy_stats( + seq_store, + &whole, + &prev_c_block.entropy, + &mut next_c_block.entropy, + cctx_params, + &mut entropy_metadata, + )?; + compress_sub_block_multi( + seq_store, + prev_c_block, + next_c_block, + &entropy_metadata, + cctx_params, + dst, + src, + last_block, + ) +} diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 68bc85b136f1..0b5beda4b583 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -235,34 +235,51 @@ fn read_skippable_frame_size(src: &[u8]) -> Result { Ok(skippable_size) } -/// `ZSTD_findFrameCompressedSize`: how many bytes the frame at the start of `src` takes. -pub fn find_frame_compressed_size(src: &[u8]) -> Result { +/// How far a frame goes. +pub enum FrameExtent { + /// It is this many bytes. + Whole(usize), + /// It goes on past what there is of it, and is at least this many bytes. + AtLeast(usize), +} + +/// How far the frame at the start of `src` goes, for when where it ends is not known in advance. +pub fn frame_extent(src: &[u8]) -> Result { if src.len() >= ZSTD_SKIPPABLEHEADERSIZE && is_skippable_magic(read_le32(src, 0)?) { - return read_skippable_frame_size(src); + return read_skippable_frame_size(src).map(FrameExtent::Whole); } let header = match get_frame_header(src)? { HeaderStatus::Complete(header) => header, - HeaderStatus::Needs(_) => return Err(ErrorCode::SrcSizeWrong), + HeaderStatus::Needs(size) => return Ok(FrameExtent::AtLeast(size)), }; let mut ip = header.header_size; loop { - let remaining = src.sub(ip..)?; + let Some(remaining) = src.get(ip..).filter(|rest| rest.len() >= ZSTD_BLOCKHEADERSIZE) + else { + return Ok(FrameExtent::AtLeast(ip + ZSTD_BLOCKHEADERSIZE)); + }; let (c_block_size, properties) = get_c_block_size(remaining)?; - if ZSTD_BLOCKHEADERSIZE + c_block_size > remaining.len() { - return Err(ErrorCode::SrcSizeWrong); - } ip += ZSTD_BLOCKHEADERSIZE + c_block_size; if properties.last_block { break; } } if header.checksum_flag { - if src.len() - ip < 4 { - return Err(ErrorCode::SrcSizeWrong); - } ip += 4; } - Ok(ip) + Ok(if ip <= src.len() { + FrameExtent::Whole(ip) + } else { + FrameExtent::AtLeast(ip) + }) +} + +/// `ZSTD_findFrameCompressedSize`: how many bytes the frame at the start of `src` takes. +pub fn find_frame_compressed_size(src: &[u8]) -> Result { + match frame_extent(src)? { + FrameExtent::Whole(size) => Ok(size), + FrameExtent::AtLeast(_) => Err(ErrorCode::SrcSizeWrong), + } } /// `ZSTD_findDecompressedSize`: the size of all the frames of `src` together, once decompressed. diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 9d373f48f62f..03e72a9213f8 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -1,5 +1,6 @@ //! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the //! same bytes in, in both directions, and the same errors. +#![feature(hint_prefetch)] #![forbid(unsafe_code)] // Frames come from anywhere and a panic aborts the process. #![deny( @@ -12,14 +13,19 @@ #![warn(unused_must_use)] mod common; +mod compress; mod decompress; mod error; mod util; pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; +pub use compress::params::{ + ZSTD_CLEVEL_DEFAULT, ZSTD_MAX_CLEVEL, ZSTD_MIN_CLEVEL, c_param, compress_bound, +}; +pub use compress::stream::{CCtx, EndDirective, compress}; pub use decompress::{ - DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, - find_frame_compressed_size, get_frame_content_size, + DCtx, DDict, FrameExtent, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, + find_frame_compressed_size, frame_extent, get_frame_content_size, }; pub use error::ErrorCode; @@ -75,6 +81,25 @@ impl ZstdError { } } +/// [`compress`] to the end of `out`, within its capacity: [`compress_bound`] more than its length is always enough. +/// Returns how many bytes are added. +pub fn compress_append(out: &mut Vec, src: &[u8], level: i32) -> Result { + let start = out.len(); + let room = (out.capacity() - start).min(compress_bound(src.len())); + out.resize(start + room, 0); + let result = compress(out.get_mut(start..).unwrap_or_default(), src, level); + out.truncate(start + result.unwrap_or(0)); + result +} + +/// [`compress`] to a new buffer. +pub fn compress_alloc(src: &[u8], level: i32) -> Result, ErrorCode> { + let mut out = util::try_zeroed(compress_bound(src.len()))?; + let size = compress(&mut out, src, level)?; + out.truncate(size); + Ok(out) +} + /// `len` zeros for a decoder to write to. fn output_buffer(len: usize) -> Result, ZstdError> { util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) diff --git a/src/zstd/wip/build-ref.sh b/src/zstd/wip/build-ref.sh new file mode 100755 index 000000000000..209061b63378 --- /dev/null +++ b/src/zstd/wip/build-ref.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Builds the reference C zstd exactly as Bun's scripts/build/deps/zstd.ts did (pin f8745da6 + Bun's two patches). +set -e +cd /root/zstd-ref +SRC="common/debug common/entropy_common common/error_private common/fse_decompress common/pool common/threading common/xxhash common/zstd_common compress/fse_compress compress/hist compress/huf_compress compress/zstd_compress compress/zstd_compress_literals compress/zstd_compress_sequences compress/zstd_compress_superblock compress/zstd_double_fast compress/zstd_fast compress/zstd_lazy compress/zstd_ldm compress/zstd_opt compress/zstd_preSplit compress/zstdmt_compress decompress/huf_decompress decompress/zstd_ddict decompress/zstd_decompress decompress/zstd_decompress_block" +build() { # name, extra flags, asm? + local name=$1; shift; local asm=$1; shift + mkdir -p build/$name; local objs="" + for s in $SRC; do o=build/$name/$(basename $s).o; clang-23 -c -O2 -g -fPIC -march=nehalem -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=0 -DXXH_NAMESPACE=ZSTD_ -Izstd/lib -Izstd/lib/common "$@" zstd/lib/$s.c -o $o & objs="$objs $o"; done + if [ $asm = 1 ]; then clang-23 -c -fPIC zstd/lib/decompress/huf_decompress_amd64.S -o build/$name/huf_asm.o; objs="$objs build/$name/huf_asm.o"; fi + wait + rm -f build/$name/libzstd.a; llvm-ar-23 rcs build/$name/libzstd.a $objs +} +build ref 1 +build noasm 0 -DZSTD_DISABLE_ASM=1 -DDYNAMIC_BMI2=0 diff --git a/src/zstd/wip/clevels.rs b/src/zstd/wip/clevels.rs new file mode 100644 index 000000000000..f3c9b55dedaa --- /dev/null +++ b/src/zstd/wip/clevels.rs @@ -0,0 +1,100 @@ + [ + cp(19, 12, 13, 1, 6, 1, Strategy::Fast), + cp(19, 13, 14, 1, 7, 0, Strategy::Fast), + cp(20, 15, 16, 1, 6, 0, Strategy::Fast), + cp(21, 16, 17, 1, 5, 0, Strategy::DFast), + cp(21, 18, 18, 1, 5, 0, Strategy::DFast), + cp(21, 18, 19, 3, 5, 2, Strategy::Greedy), + cp(21, 18, 19, 3, 5, 4, Strategy::Lazy), + cp(21, 19, 20, 4, 5, 8, Strategy::Lazy), + cp(21, 19, 20, 4, 5, 16, Strategy::Lazy2), + cp(22, 20, 21, 4, 5, 16, Strategy::Lazy2), + cp(22, 21, 22, 5, 5, 16, Strategy::Lazy2), + cp(22, 21, 22, 6, 5, 16, Strategy::Lazy2), + cp(22, 22, 23, 6, 5, 32, Strategy::Lazy2), + cp(22, 22, 22, 4, 5, 32, Strategy::BtLazy2), + cp(22, 22, 23, 5, 5, 32, Strategy::BtLazy2), + cp(22, 23, 23, 6, 5, 32, Strategy::BtLazy2), + cp(22, 22, 22, 5, 5, 48, Strategy::BtOpt), + cp(23, 23, 22, 5, 4, 64, Strategy::BtOpt), + cp(23, 23, 22, 6, 3, 64, Strategy::BtUltra), + cp(23, 24, 22, 7, 3, 256, Strategy::BtUltra2), + cp(25, 25, 23, 7, 3, 256, Strategy::BtUltra2), + cp(26, 26, 24, 7, 3, 512, Strategy::BtUltra2), + cp(27, 27, 25, 9, 3, 999, Strategy::BtUltra2), + ], + [ + cp(18, 12, 13, 1, 5, 1, Strategy::Fast), + cp(18, 13, 14, 1, 6, 0, Strategy::Fast), + cp(18, 14, 14, 1, 5, 0, Strategy::DFast), + cp(18, 16, 16, 1, 4, 0, Strategy::DFast), + cp(18, 16, 17, 3, 5, 2, Strategy::Greedy), + cp(18, 17, 18, 5, 5, 2, Strategy::Greedy), + cp(18, 18, 19, 3, 5, 4, Strategy::Lazy), + cp(18, 18, 19, 4, 4, 4, Strategy::Lazy), + cp(18, 18, 19, 4, 4, 8, Strategy::Lazy2), + cp(18, 18, 19, 5, 4, 8, Strategy::Lazy2), + cp(18, 18, 19, 6, 4, 8, Strategy::Lazy2), + cp(18, 18, 19, 5, 4, 12, Strategy::BtLazy2), + cp(18, 19, 19, 7, 4, 12, Strategy::BtLazy2), + cp(18, 18, 19, 4, 4, 16, Strategy::BtOpt), + cp(18, 18, 19, 4, 3, 32, Strategy::BtOpt), + cp(18, 18, 19, 6, 3, 128, Strategy::BtOpt), + cp(18, 19, 19, 6, 3, 128, Strategy::BtUltra), + cp(18, 19, 19, 8, 3, 256, Strategy::BtUltra), + cp(18, 19, 19, 6, 3, 128, Strategy::BtUltra2), + cp(18, 19, 19, 8, 3, 256, Strategy::BtUltra2), + cp(18, 19, 19, 10, 3, 512, Strategy::BtUltra2), + cp(18, 19, 19, 12, 3, 512, Strategy::BtUltra2), + cp(18, 19, 19, 13, 3, 999, Strategy::BtUltra2), + ], + [ + cp(17, 12, 12, 1, 5, 1, Strategy::Fast), + cp(17, 12, 13, 1, 6, 0, Strategy::Fast), + cp(17, 13, 15, 1, 5, 0, Strategy::Fast), + cp(17, 15, 16, 2, 5, 0, Strategy::DFast), + cp(17, 17, 17, 2, 4, 0, Strategy::DFast), + cp(17, 16, 17, 3, 4, 2, Strategy::Greedy), + cp(17, 16, 17, 3, 4, 4, Strategy::Lazy), + cp(17, 16, 17, 3, 4, 8, Strategy::Lazy2), + cp(17, 16, 17, 4, 4, 8, Strategy::Lazy2), + cp(17, 16, 17, 5, 4, 8, Strategy::Lazy2), + cp(17, 16, 17, 6, 4, 8, Strategy::Lazy2), + cp(17, 17, 17, 5, 4, 8, Strategy::BtLazy2), + cp(17, 18, 17, 7, 4, 12, Strategy::BtLazy2), + cp(17, 18, 17, 3, 4, 12, Strategy::BtOpt), + cp(17, 18, 17, 4, 3, 32, Strategy::BtOpt), + cp(17, 18, 17, 6, 3, 256, Strategy::BtOpt), + cp(17, 18, 17, 6, 3, 128, Strategy::BtUltra), + cp(17, 18, 17, 8, 3, 256, Strategy::BtUltra), + cp(17, 18, 17, 10, 3, 512, Strategy::BtUltra), + cp(17, 18, 17, 5, 3, 256, Strategy::BtUltra2), + cp(17, 18, 17, 7, 3, 512, Strategy::BtUltra2), + cp(17, 18, 17, 9, 3, 512, Strategy::BtUltra2), + cp(17, 18, 17, 11, 3, 999, Strategy::BtUltra2), + ], + [ + cp(14, 12, 13, 1, 5, 1, Strategy::Fast), + cp(14, 14, 15, 1, 5, 0, Strategy::Fast), + cp(14, 14, 15, 1, 4, 0, Strategy::Fast), + cp(14, 14, 15, 2, 4, 0, Strategy::DFast), + cp(14, 14, 14, 4, 4, 2, Strategy::Greedy), + cp(14, 14, 14, 3, 4, 4, Strategy::Lazy), + cp(14, 14, 14, 4, 4, 8, Strategy::Lazy2), + cp(14, 14, 14, 6, 4, 8, Strategy::Lazy2), + cp(14, 14, 14, 8, 4, 8, Strategy::Lazy2), + cp(14, 15, 14, 5, 4, 8, Strategy::BtLazy2), + cp(14, 15, 14, 9, 4, 8, Strategy::BtLazy2), + cp(14, 15, 14, 3, 4, 12, Strategy::BtOpt), + cp(14, 15, 14, 4, 3, 24, Strategy::BtOpt), + cp(14, 15, 14, 5, 3, 32, Strategy::BtUltra), + cp(14, 15, 15, 6, 3, 64, Strategy::BtUltra), + cp(14, 15, 15, 7, 3, 256, Strategy::BtUltra), + cp(14, 15, 15, 5, 3, 48, Strategy::BtUltra2), + cp(14, 15, 15, 6, 3, 128, Strategy::BtUltra2), + cp(14, 15, 15, 7, 3, 256, Strategy::BtUltra2), + cp(14, 15, 15, 8, 3, 256, Strategy::BtUltra2), + cp(14, 15, 15, 8, 3, 512, Strategy::BtUltra2), + cp(14, 15, 15, 9, 3, 512, Strategy::BtUltra2), + cp(14, 15, 15, 10, 3, 999, Strategy::BtUltra2), + ], diff --git a/src/zstd/wip/default_tables.rs b/src/zstd/wip/default_tables.rs new file mode 100644 index 000000000000..5237a892d9ba --- /dev/null +++ b/src/zstd/wip/default_tables.rs @@ -0,0 +1,51 @@ +#[rustfmt::skip] +const LL_DEFAULT: [SeqSymbol; 64] = [ + s( 0, 0, 4, 0), s(16, 0, 4, 0), s(32, 0, 5, 1), s( 0, 0, 5, 3), + s( 0, 0, 5, 4), s( 0, 0, 5, 6), s( 0, 0, 5, 7), s( 0, 0, 5, 9), + s( 0, 0, 5, 10), s( 0, 0, 5, 12), s( 0, 0, 6, 14), s( 0, 1, 5, 16), + s( 0, 1, 5, 20), s( 0, 1, 5, 22), s( 0, 2, 5, 28), s( 0, 3, 5, 32), + s( 0, 4, 5, 48), s(32, 6, 5, 64), s( 0, 7, 5, 128), s( 0, 8, 6, 256), + s( 0, 10, 6, 1024), s( 0, 12, 6, 4096), s(32, 0, 4, 0), s( 0, 0, 4, 1), + s( 0, 0, 5, 2), s(32, 0, 5, 4), s( 0, 0, 5, 5), s(32, 0, 5, 7), + s( 0, 0, 5, 8), s(32, 0, 5, 10), s( 0, 0, 5, 11), s( 0, 0, 6, 13), + s(32, 1, 5, 16), s( 0, 1, 5, 18), s(32, 1, 5, 22), s( 0, 2, 5, 24), + s(32, 3, 5, 32), s( 0, 3, 5, 40), s( 0, 6, 4, 64), s(16, 6, 4, 64), + s(32, 7, 5, 128), s( 0, 9, 6, 512), s( 0, 11, 6, 2048), s(48, 0, 4, 0), + s(16, 0, 4, 1), s(32, 0, 5, 2), s(32, 0, 5, 3), s(32, 0, 5, 5), + s(32, 0, 5, 6), s(32, 0, 5, 8), s(32, 0, 5, 9), s(32, 0, 5, 11), + s(32, 0, 5, 12), s( 0, 0, 6, 15), s(32, 1, 5, 18), s(32, 1, 5, 20), + s(32, 2, 5, 24), s(32, 2, 5, 28), s(32, 3, 5, 40), s(32, 4, 5, 48), + s( 0, 16, 6, 65536), s( 0, 15, 6, 32768), s( 0, 14, 6, 16384), s( 0, 13, 6, 8192), +]; + +#[rustfmt::skip] +const OF_DEFAULT: [SeqSymbol; 32] = [ + s( 0, 0, 5, 0), s( 0, 6, 4, 61), s( 0, 9, 5, 509), s( 0, 15, 5, 32765), + s( 0, 21, 5, 2097149), s( 0, 3, 5, 5), s( 0, 7, 4, 125), s( 0, 12, 5, 4093), + s( 0, 18, 5, 262141), s( 0, 23, 5, 8388605), s( 0, 5, 5, 29), s( 0, 8, 4, 253), + s( 0, 14, 5, 16381), s( 0, 20, 5, 1048573), s( 0, 2, 5, 1), s(16, 7, 4, 125), + s( 0, 11, 5, 2045), s( 0, 17, 5, 131069), s( 0, 22, 5, 4194301), s( 0, 4, 5, 13), + s(16, 8, 4, 253), s( 0, 13, 5, 8189), s( 0, 19, 5, 524285), s( 0, 1, 5, 1), + s(16, 6, 4, 61), s( 0, 10, 5, 1021), s( 0, 16, 5, 65533), s( 0, 28, 5, 268435453), + s( 0, 27, 5, 134217725), s( 0, 26, 5, 67108861), s( 0, 25, 5, 33554429), s( 0, 24, 5, 16777213), +]; + +#[rustfmt::skip] +const ML_DEFAULT: [SeqSymbol; 64] = [ + s( 0, 0, 6, 3), s( 0, 0, 4, 4), s(32, 0, 5, 5), s( 0, 0, 5, 6), + s( 0, 0, 5, 8), s( 0, 0, 5, 9), s( 0, 0, 5, 11), s( 0, 0, 6, 13), + s( 0, 0, 6, 16), s( 0, 0, 6, 19), s( 0, 0, 6, 22), s( 0, 0, 6, 25), + s( 0, 0, 6, 28), s( 0, 0, 6, 31), s( 0, 0, 6, 34), s( 0, 1, 6, 37), + s( 0, 1, 6, 41), s( 0, 2, 6, 47), s( 0, 3, 6, 59), s( 0, 4, 6, 83), + s( 0, 7, 6, 131), s( 0, 9, 6, 515), s(16, 0, 4, 4), s( 0, 0, 4, 5), + s(32, 0, 5, 6), s( 0, 0, 5, 7), s(32, 0, 5, 9), s( 0, 0, 5, 10), + s( 0, 0, 6, 12), s( 0, 0, 6, 15), s( 0, 0, 6, 18), s( 0, 0, 6, 21), + s( 0, 0, 6, 24), s( 0, 0, 6, 27), s( 0, 0, 6, 30), s( 0, 0, 6, 33), + s( 0, 1, 6, 35), s( 0, 1, 6, 39), s( 0, 2, 6, 43), s( 0, 3, 6, 51), + s( 0, 4, 6, 67), s( 0, 5, 6, 99), s( 0, 8, 6, 259), s(32, 0, 4, 4), + s(48, 0, 4, 4), s(16, 0, 4, 5), s(32, 0, 5, 7), s(32, 0, 5, 8), + s(32, 0, 5, 10), s(32, 0, 5, 11), s( 0, 0, 6, 14), s( 0, 0, 6, 17), + s( 0, 0, 6, 20), s( 0, 0, 6, 23), s( 0, 0, 6, 26), s( 0, 0, 6, 29), + s( 0, 0, 6, 32), s( 0, 16, 6, 65539), s( 0, 15, 6, 32771), s( 0, 14, 6, 16387), + s( 0, 13, 6, 8195), s( 0, 12, 6, 4099), s( 0, 11, 6, 2051), s( 0, 10, 6, 1027), +]; diff --git a/src/zstd/wip/harness/Cargo.toml b/src/zstd/wip/harness/Cargo.toml new file mode 100644 index 000000000000..e9cca9310c3e --- /dev/null +++ b/src/zstd/wip/harness/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "zstd_harness" +version = "0.0.0" +edition = "2024" + +[dependencies] +bun_zstd = { path = "/root/repos/oven-sh/bun/src/zstd" } + +[profile.release] +debug = "line-tables-only" +overflow-checks = true +debug-assertions = true +panic = "abort" + +[profile.bench] +inherits = "release" +overflow-checks = false +debug-assertions = false + +[workspace] diff --git a/src/zstd/wip/harness/build.rs b/src/zstd/wip/harness/build.rs new file mode 100644 index 000000000000..e0b8fe9e7a87 --- /dev/null +++ b/src/zstd/wip/harness/build.rs @@ -0,0 +1,6 @@ +fn main() { + let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); + println!("cargo:rerun-if-env-changed=ZSTD_REF"); + println!("cargo:rustc-link-search=native=/root/zstd-ref/build/{variant}"); + println!("cargo:rustc-link-lib=static=zstd"); +} diff --git a/src/zstd/wip/harness/rust-toolchain.toml b/src/zstd/wip/harness/rust-toolchain.toml new file mode 100644 index 000000000000..c71778e8ccd3 --- /dev/null +++ b/src/zstd/wip/harness/rust-toolchain.toml @@ -0,0 +1,35 @@ +[toolchain] +channel = "nightly-2026-09-15" +# rust-src is needed for -Zbuild-std (Tier 3 targets like +# aarch64-unknown-freebsd have no prebuilt std). miri is for +# `bun run rust:miri`. targets ensures the +# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain +# (CI's images install this same channel with these components and +# targets, from scripts/build/ci-images/spec.ts; a source lint keeps +# the two files in step). Note: rustup ignores this whole file when +# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree +# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` +# self-heals that path. Keep this list as a fallback for direct +# `cargo build`/`cargo check`/rust-analyzer use. +# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd +# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). +# darwin/windows triples build natively on their own agents, so the host std +# comes with the toolchain install regardless; they're listed here so a dev +# running `bun run rust:check-all` or `cargo check --target` from any host +# still gets prebuilt std for every Tier 1/2 triple. +# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at +# what it emits. The build does not use them. +components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] +targets = [ + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-musl", + "aarch64-linux-android", + "x86_64-linux-android", + "x86_64-unknown-freebsd", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-pc-windows-msvc", + "x86_64-pc-windows-msvc", +] diff --git a/src/zstd/wip/harness/src/c.rs b/src/zstd/wip/harness/src/c.rs new file mode 100644 index 000000000000..d5a82e1f6b78 --- /dev/null +++ b/src/zstd/wip/harness/src/c.rs @@ -0,0 +1,104 @@ +//! The reference C library. +#![allow(non_camel_case_types, dead_code)] +use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; + +#[repr(C)] +pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } +#[repr(C)] +pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } +pub enum ZSTD_CCtx {} +pub enum ZSTD_DCtx {} +pub enum ZSTD_DDict {} + +unsafe extern "C" { + pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; + pub fn ZSTD_compressBound(size: usize) -> usize; + pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_isError(code: usize) -> c_uint; + pub fn ZSTD_getErrorCode(code: usize) -> c_uint; + pub fn ZSTD_getErrorName(code: usize) -> *const c_char; + pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; + pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; + pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; + pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; + pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; + pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; + pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; + pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; + pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; + pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; + pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; + pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; + pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; + pub fn ZSTD_defaultCLevel() -> c_int; + pub fn ZSTD_minCLevel() -> c_int; + pub fn ZSTD_maxCLevel() -> c_int; +} + +/// `Ok(value)` or `Err(error code)`. +pub fn res(code: usize) -> Result { + unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } +} + +pub struct CCtx(pub *mut ZSTD_CCtx); +impl CCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } + pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + /// Returns (result, out pos, in pos). + pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); + (r, o.pos, i.pos) + } +} +impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } + +pub struct DCtx(pub *mut ZSTD_DCtx); +impl DCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } + pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } + pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); + (r, o.pos, i.pos) + } +} +impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } + +pub fn compress(src: &[u8], level: i32) -> Vec { + let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; + let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); + dst.truncate(n); + dst +} +pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) +} diff --git a/src/zstd/wip/harness/src/comp.rs b/src/zstd/wip/harness/src/comp.rs new file mode 100644 index 000000000000..2f86d9854aff --- /dev/null +++ b/src/zstd/wip/harness/src/comp.rs @@ -0,0 +1,296 @@ +//! The compressor against the C one: same calls, same results, same bytes, same positions. +use crate::c; +use crate::data::{Rng, data}; +use bun_zstd as z; + +fn r2(r: Result) -> Result { r.map_err(|e| e as u32) } +fn r0(r: Result<(), z::ErrorCode>) -> Result { r.map(|()| 0).map_err(|e| e as u32) } + +/// A dictionary: raw content, or in zstd's format (made from a real one by the C trainer is not available, so entropy +/// tables are taken from a frame... here: raw, or magic + garbage to exercise the error path). +fn dictionary(rng: &mut Rng, base: &[u8]) -> Vec { + let len = rng.size(1 << 17); + let mut d = if rng.chance(1, 2) && !base.is_empty() { + let start = rng.below(base.len()); + let end = (start + len).min(base.len()); + base[start..end].to_vec() + } else { + data(rng, len) + }; + if rng.chance(1, 12) && d.len() >= 8 { + d[..4].copy_from_slice(&0xEC30A437u32.to_le_bytes()); + } + d +} + +fn level(rng: &mut Rng) -> i32 { + match rng.below(10) { + 0 => -(rng.range(1, 50) as i32), + 1 => rng.range(0, 30) as i32 - 4, + 2 => -(rng.size(200000) as i32), + _ => rng.range(1, 22) as i32, + } +} + +pub fn one_shot(rng: &mut Rng, max: usize) { + let len = rng.size(max); + let src = data(rng, len); + let lvl = level(rng); + let bound = unsafe { c::ZSTD_compressBound(len) }; + assert_eq!(bound, z::compress_bound(len)); + let cap = match rng.below(8) { 0 => rng.below(bound + 1), 1 => bound.saturating_sub(rng.below(20)), _ => bound + rng.below(50) }; + let mut a = vec![0u8; cap]; + let mut b = vec![0u8; cap]; + let ra = c::res(unsafe { c::ZSTD_compress(a.as_mut_ptr().cast(), cap, src.as_ptr().cast(), len, lvl) }); + let rb = r2(z::compress(&mut b, &src, lvl)); + assert_eq!(ra, rb, "one shot: len {len} level {lvl} cap {cap}"); + if let Ok(n) = ra { + if a[..n] != b[..n] { + let at = a[..n].iter().zip(&b[..n]).position(|(x, y)| x != y); + panic!("one shot bytes differ at {at:?}/{n}: len {len} level {lvl}"); + } + } +} + +const PARAMS: &[(i32, i32, i32)] = &[ + (10, 0, 1), (100, -50, 22), (101, 10, 27), (102, 6, 26), (103, 6, 26), (104, 1, 12), (105, 3, 7), (106, 0, 4096), + (107, 1, 9), (130, 0, 20000), (160, 0, 2), (161, 6, 24), (162, 4, 4096), (163, 1, 8), (164, 0, 12), + (200, 0, 1), (201, 0, 1), (202, 0, 1), +]; + +fn set_params(rng: &mut Rng, a: &mut c::CCtx, b: &mut z::CCtx, log: &mut Vec, mt: bool) { + let n = match rng.below(4) { 0 => 0, 1 => 1, 2 => rng.below(4), _ => rng.below(9) }; + for _ in 0..n { + let (p, v) = if rng.chance(1, 25) { + (rng.below(403) as i32, rng.range(0, 40) as i32 - 5) + } else { + let &(p, lo, hi) = rng.pick(PARAMS); + let v = if rng.chance(1, 20) { rng.range(0, 3000000) as i32 - 200000 } else { lo + rng.below((hi - lo + 1) as usize) as i32 }; + (p, v) + }; + if !mt && (400..=402).contains(&p) { continue; } + let ra = a.set(p as u32, v); + let rb = r2(b.set_parameter(p, v)); + log.push(format!("set {p} {v} -> {ra:?}")); + assert_eq!(ra, rb, "{log:?}"); + } +} + +/// One context, several frames, random calls. +pub fn stream(rng: &mut Rng, max: usize) { + let mut a = c::CCtx::new(); + let mut b = z::CCtx::new(); + let mut log: Vec = Vec::new(); + let frames = if rng.chance(1, 3) { rng.range(2, 5) } else { 1 }; + for _ in 0..frames { + if rng.chance(1, 6) { + let r = *rng.pick(&[1u32, 2, 3]); + let ra = a.reset(r); + let rb = r0(b.reset(match r { 1 => z::ResetDirective::SessionOnly, 2 => z::ResetDirective::Parameters, _ => z::ResetDirective::SessionAndParameters })); + log.push(format!("reset {r} -> {ra:?}")); + assert_eq!(ra, rb, "{log:?}"); + } + set_params(rng, &mut a, &mut b, &mut log, false); + let len = rng.size(max); + let src = data(rng, len); + if rng.chance(1, 4) && std::env::var_os("NODICT").is_none() { + let d = if rng.chance(1, 10) { Vec::new() } else { dictionary(rng, &src) }; + let ra = a.load_dict(&d); + let rb = r0(b.load_dictionary(&d)); + log.push(format!("dict {} -> {ra:?}", d.len())); + assert_eq!(ra, rb, "{log:?}"); + } + if rng.chance(1, 4) { + let n = if rng.chance(1, 8) { rng.size(max) as u64 } else { len as u64 }; + let ra = a.pledge(n); + let rb = r0(b.set_pledged_src_size(n)); + log.push(format!("pledge {n} (len {len}) -> {ra:?}")); + assert_eq!(ra, rb, "{log:?}"); + } + log.push(format!("frame len {len}")); + // how the input and the output are cut + let in_mode = rng.below(4); + let out_mode = rng.below(4); + let small = len < 5000; + let mut spos_total = 0usize; + let mut calls = 0usize; + let mut failed = false; + loop { + calls += 1; + let left = len - spos_total; + let chunk = match in_mode { + 0 => left, + 1 => rng.range(0, if small { 64 } else { 100_000 }).min(left), + 2 => rng.size(1 << 19).min(left), + _ => if rng.chance(1, 2) { left } else { rng.below(left + 1) }, + }; + let cap = match out_mode { + 0 => z::compress_bound(chunk) + rng.below(100), + 1 => rng.range(0, if small { 40 } else { 50_000 }), + 2 => rng.size(1 << 18), + _ => 1 << 17, + }; + let last = spos_total + chunk == len; + let end = if last && rng.chance(3, 4) { 2 } else { *rng.pick(&[0u32, 0, 0, 1, 1, 2]) }; + let end = if calls > 4000 { 2 } else { end }; + let input = &src[spos_total..spos_total + chunk]; + let mut oa = vec![0u8; cap]; + let mut ob = vec![0u8; cap]; + let (ra, opa, ipa) = a.stream2(&mut oa, 0, input, 0, end); + let (rb, opb, ipb) = { + let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; + let mut i = z::InBuffer { src: input, pos: 0 }; + let r = r2(b.compress_stream2(&mut o, &mut i, match end { 0 => z::EndDirective::Continue, 1 => z::EndDirective::Flush, _ => z::EndDirective::End })); + (r, o.pos, i.pos) + }; + if (ra, opa, ipa) != (rb, opb, ipb) || oa[..opa] != ob[..opb] { + let at = oa.iter().zip(&ob).position(|(x, y)| x != y); + panic!("stream: call {calls} chunk {chunk} cap {cap} end {end}: C {:?} Rust {:?} first diff {at:?}\n{log:?}", (ra, opa, ipa), (rb, opb, ipb)); + } + spos_total += ipa; + if ra.is_err() { failed = true; break; } + if end == 2 && ra == Ok(0) { + if spos_total == len { break; } + log.push(format!("frame ended at {spos_total}")); + } + if calls > 6000 { panic!("no end {log:?}"); } + } + if failed { + log.push("failed".into()); + let ra = a.reset(1); + let rb = r0(b.reset(z::ResetDirective::SessionOnly)); + assert_eq!(ra, rb); + } + } +} + +pub fn run(seed: u64, iters: u64, max: usize) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if std::env::var_os("TRACE").is_some() { eprintln!("iter {i} seed {s}"); } + if rng.chance(1, 3) { one_shot(&mut rng, max) } else { stream(&mut rng, max) } + } + println!("comp ok: seed {seed}, {iters} iterations"); +} + +/// Multithreaded: when output comes out depends on timing, so only the whole output is compared. +pub fn mt(rng: &mut Rng, max: usize) { + let mut a = c::CCtx::new(); + let mut b = z::CCtx::new(); + let mut log: Vec = Vec::new(); + let frames = if rng.chance(1, 3) { rng.range(2, 3) } else { 1 }; + for _ in 0..frames { + let mut set = |p: i32, v: i32, log: &mut Vec| { + let ra = a.set(p as u32, v); + let rb = r2(b.set_parameter(p, v)); + log.push(format!("set {p} {v} -> {ra:?}")); + assert_eq!(ra, rb, "{log:?}"); + }; + set(400, rng.range(1, 4) as i32, &mut log); + if rng.chance(1, 2) { set(401, rng.range(0, 3 << 20) as i32, &mut log); } + if rng.chance(1, 2) { set(402, rng.range(0, 9) as i32, &mut log); } + set(100, if rng.chance(1, 6) { rng.range(13, 19) as i32 } else { rng.range(1, 12) as i32 - 3 }, &mut log); + if rng.chance(1, 2) { set(101, rng.range(10, 22) as i32, &mut log); } + if rng.chance(1, 3) { set(160, 1, &mut log); if rng.chance(1, 2) { set(162, rng.range(4, 200) as i32, &mut log); } if rng.chance(1,2) { set(161, rng.range(6, 20) as i32, &mut log); } } + if rng.chance(1, 2) { set(201, 1, &mut log); } + if rng.chance(1, 8) { set(130, rng.range(1340, 60000) as i32, &mut log); } + let len = if rng.chance(1, 4) { rng.size(max) } else { rng.range(max / 4, max) }; + let src = data(rng, len); + if rng.chance(1, 5) { + let d = dictionary(rng, &src); + let ra = a.load_dict(&d); + let rb = r0(b.load_dictionary(&d)); + log.push(format!("dict {} -> {ra:?}", d.len())); + assert_eq!(ra, rb, "{log:?}"); + } + if rng.chance(1, 4) { + let ra = a.pledge(len as u64); + let rb = r0(b.set_pledged_src_size(len as u64)); + log.push(format!("pledge {len}")); + assert_eq!(ra, rb, "{log:?}"); + } + log.push(format!("frame len {len}")); + let mut out_a: Vec = Vec::new(); + let mut out_b: Vec = Vec::new(); + let mut pos = 0usize; + let in_mode = rng.below(3); + loop { + let left = len - pos; + let chunk = match in_mode { 0 => left, 1 => rng.size(1 << 21).min(left), _ => rng.range(0, 300_000).min(left) }; + let last = pos + chunk == len; + let end = if last { 2u32 } else { *rng.pick(&[0u32, 0, 0, 0, 1]) }; + let input = &src[pos..pos + chunk]; + let cap = z::compress_bound(len.max(1 << 20)) + 1024; + let mut buf = vec![0u8; cap]; + let mut err_a = None; + let mut err_b = None; + let mut ip = 0usize; + loop { + let (r, op, np) = a.stream2(&mut buf, 0, input, ip, end); + out_a.extend_from_slice(&buf[..op]); + ip = np; + match r { Err(e) => { err_a = Some(e); break; } Ok(left) => if (end == 0 && ip == chunk) || (end != 0 && left == 0) { break; } } + } + let mut ip = 0usize; + loop { + let mut o = z::OutBuffer { dst: &mut buf, pos: 0 }; + let mut i = z::InBuffer { src: input, pos: ip }; + let r = r2(b.compress_stream2(&mut o, &mut i, match end { 0 => z::EndDirective::Continue, 1 => z::EndDirective::Flush, _ => z::EndDirective::End })); + let op = o.pos; + ip = i.pos; + out_b.extend_from_slice(&buf[..op]); + match r { Err(e) => { err_b = Some(e); break; } Ok(left) => if (end == 0 && ip == chunk) || (end != 0 && left == 0) { break; } } + } + assert_eq!(err_a, err_b, "{log:?}"); + log.push(format!("chunk {chunk} end {end}")); + if err_a.is_some() { let _ = a.reset(1); let _ = b.reset(z::ResetDirective::SessionOnly); break; } + pos += chunk; + if last { break; } + } + if out_a != out_b { + let at = out_a.iter().zip(&out_b).position(|(x, y)| x != y); + panic!("mt: {} vs {} bytes, first diff {at:?}\n{log:?}", out_a.len(), out_b.len()); + } + } +} + +pub fn run_mt(seed: u64, iters: u64, max: usize) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if std::env::var_os("TRACE").is_some() { eprintln!("iter {i} seed {s}"); } + mt(&mut rng, max); + } + println!("mt ok: seed {seed}, {iters} iterations"); +} + +/// Dictionaries, by strategy. +pub fn run_dict(seed: u64, iters: u64, max: usize) { + let mut fails = std::collections::BTreeMap::new(); + for i in 0..iters { + let mut rng = Rng(seed.wrapping_mul(1_000_003).wrapping_add(i)); + let strat = rng.range(1, 9) as i32; + let len = rng.size(max); + let src = data(&mut rng, len); + let d = dictionary(&mut rng, &src); + let pledge = rng.chance(1, 2); + let mut a = c::CCtx::new(); + let mut b = z::CCtx::new(); + a.set(107, strat).unwrap(); b.set_parameter(107, strat).unwrap(); + a.load_dict(&d).unwrap(); b.load_dictionary(&d).unwrap(); + let cap = z::compress_bound(len) + 64; + let (mut oa, mut ob) = (vec![0u8; cap], vec![0u8; cap]); + let end = if pledge { 2 } else { 1 }; + let (ra, pa, _) = a.stream2(&mut oa, 0, &src, 0, end); + let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; + let mut inp = z::InBuffer { src: &src, pos: 0 }; + let rb = r2(b.compress_stream2(&mut o, &mut inp, if pledge { z::EndDirective::End } else { z::EndDirective::Flush })); + let pb = o.pos; + let ok = ra == rb && pa == pb && oa[..pa] == ob[..pb]; + let e = fails.entry((strat, pledge)).or_insert((0u32, 0u32)); + e.0 += 1; + if !ok { e.1 += 1; if e.1 == 1 { eprintln!("first fail strat {strat} pledge {pledge} len {len} dict {} iter {i}: {ra:?} {rb:?} {pa} {pb}", d.len()); } } + } + for (k, v) in fails { println!("{k:?}: {} of {} differ", v.1, v.0); } +} diff --git a/src/zstd/wip/harness/src/data.rs b/src/zstd/wip/harness/src/data.rs new file mode 100644 index 000000000000..c48a645e1b5d --- /dev/null +++ b/src/zstd/wip/harness/src/data.rs @@ -0,0 +1,65 @@ +//! Deterministic inputs. +pub struct Rng(pub u64); +impl Rng { + pub fn next(&mut self) -> u64 { + // splitmix64 + self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + } + pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } + pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } + pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } + pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } + /// Skewed towards small values. + pub fn size(&mut self, max: usize) -> usize { + let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); + (self.next() as usize & ((1usize << bits) - 1)).min(max) + } +} + +static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; + +/// One buffer of about `len` bytes, of a kind picked by `rng`. +pub fn data(rng: &mut Rng, len: usize) -> Vec { + let mut v = Vec::with_capacity(len + 64); + match rng.below(9) { + 0 => v.resize(len, rng.next() as u8), + 1 => while v.len() < len { v.push(rng.next() as u8) }, + 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, + 3 => { // few symbols, skewed + let n = rng.range(2, 40); + while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } + } + 4 => { // LZ-like: copies of earlier parts, at all distances + while v.len() < len { + if v.len() > 8 && rng.chance(3, 4) { + let dist = 1 + rng.size(v.len() - 1); + let n = 3 + rng.size(300); + for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } + } else { + for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } + } + } + } + 5 => { // periodic with noise + let period = rng.range(1, 300); + let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); + while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } + } + 6 => { // sections of different kinds + while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } + } + 7 => { // numbers as text + let mut x = rng.next() % 1000; + while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } + } + _ => { // long runs + while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } + } + } + v.truncate(len); + v +} diff --git a/src/zstd/wip/harness/src/dec.rs b/src/zstd/wip/harness/src/dec.rs new file mode 100644 index 000000000000..d117a936bbf6 --- /dev/null +++ b/src/zstd/wip/harness/src/dec.rs @@ -0,0 +1,184 @@ +//! The decoder against the C decoder: valid frames, damaged frames, one call and streaming. +use crate::c; +use crate::data::{Rng, data}; +use bun_zstd as z; + +fn rs(r: Result) -> Result { r.map_err(|e| e as u32) } + +/// A frame (or several) made by the C compressor with parameters picked by `rng`. Returns (frames, content, dictionary). +pub fn make_frames(rng: &mut Rng) -> (Vec, Vec, Vec) { + let mut out = Vec::new(); + let mut content = Vec::new(); + let dict = if rng.chance(1, 4) { let n = rng.size(200_000); data(rng, n) } else { Vec::new() }; + let nframes = if rng.chance(1, 5) { rng.range(2, 4) } else { 1 }; + for _ in 0..nframes { + if rng.chance(1, 12) { + // skippable frame + let n = rng.size(300); + out.extend_from_slice(&(0x184D2A50u32 + rng.below(16) as u32).to_le_bytes()); + out.extend_from_slice(&(n as u32).to_le_bytes()); + out.extend_from_slice(&data(rng, n)); + continue; + } + let max = *rng.pick(&[0usize, 10, 300, 5_000, 70_000, 300_000, 1_500_000]); + let len = rng.size(max); + let mut src = data(rng, len); + if !dict.is_empty() && rng.chance(1, 2) && !src.is_empty() { + // make the dictionary useful + let n = rng.size(dict.len().min(src.len())); + let at = rng.below(dict.len() - n + 1); + let to = rng.below(src.len() - n + 1); + src[to..to + n].copy_from_slice(&dict[at..at + n]); + } + let mut cctx = c::CCtx::new(); + let level = *rng.pick(&[-50, -5, -1, 1, 1, 2, 3, 3, 3, 4, 5, 6, 7, 9, 12, 13, 16, 19, 22]); + cctx.set(100, if src.len() > 400_000 { level.min(12) } else { level }).unwrap(); + if rng.chance(1, 3) { cctx.set(201, 1).unwrap(); } // checksum + if rng.chance(1, 4) { cctx.set(200, 0).unwrap(); } // no content size + if rng.chance(1, 4) { cctx.set(101, rng.range(10, 22) as i32).unwrap(); } // windowLog + if rng.chance(1, 10) { cctx.set(130, rng.range(1340, 20_000) as i32).unwrap(); } // targetCBlockSize + if rng.chance(1, 10) { cctx.set(160, 1).unwrap(); } // LDM + if rng.chance(1, 8) { cctx.set(105, rng.range(3, 7) as i32).unwrap(); } // minMatch + if !dict.is_empty() { cctx.load_dict(&dict).unwrap(); } + let mut dst = vec![0u8; unsafe { c::ZSTD_compressBound(src.len()) } + 1024]; + let n = if rng.chance(1, 2) { + cctx.compress2(&mut dst, &src).unwrap() + } else { + // streaming, with flushes: several blocks, no content size + let (mut dpos, mut spos) = (0, 0); + loop { + let chunk = (spos + 1 + rng.size(200_000)).min(src.len()); + let end = if chunk == src.len() { 2 } else if rng.chance(1, 3) { 1 } else { 0 }; + let (r, d, s) = cctx.stream2(&mut dst, dpos, &src[..chunk], spos, end); + dpos = d; spos = s; + if r.unwrap() == 0 && end == 2 { break dpos; } + } + }; + out.extend_from_slice(&dst[..n]); + content.extend_from_slice(&src); + } + (out, content, dict) +} + +fn mutate(rng: &mut Rng, frames: &mut Vec) { + if frames.is_empty() { return; } + for _ in 0..1 + rng.size(4) { + if frames.is_empty() { return; } + // damage is more interesting near the start of a frame + let at = if rng.chance(1, 2) { rng.size(frames.len() - 1) } else { rng.below(frames.len()) }; + match rng.below(8) { + 0 => frames[at] ^= 1 << rng.below(8), + 1 => frames[at] = rng.next() as u8, + 2 => frames.truncate(at), + 3 => { frames.remove(at); } + 4 => frames.insert(at, rng.next() as u8), + 5 => { let n = rng.size(16).min(frames.len() - at); for b in &mut frames[at..at + n] { *b = rng.next() as u8 } } + 6 => { let n = rng.size(64); let extra = data(rng, n); frames.extend_from_slice(&extra) } + _ => { let from = rng.below(frames.len()); let n = rng.size(32).min(frames.len() - from.max(at)); frames.copy_within(from..from + n, at) } + } + } +} + +fn check_sizes(frames: &[u8]) { + let p = frames.as_ptr().cast(); + unsafe { + assert_eq!(c::ZSTD_findDecompressedSize(p, frames.len()), z::find_decompressed_size(frames), "findDecompressedSize"); + assert_eq!(c::ZSTD_getFrameContentSize(p, frames.len()), z::get_frame_content_size(frames), "getFrameContentSize"); + assert_eq!(c::res(c::ZSTD_findFrameCompressedSize(p, frames.len())), rs(z::find_frame_compressed_size(frames)), "findFrameCompressedSize"); + } +} + +fn one_shot(frames: &[u8], dict: &[u8], cap: usize, prefix: bool) { + let mut cd = vec![0xAAu8; cap]; + let mut rd = vec![0xAAu8; cap]; + let mut cctx = c::DCtx::new(); + let mut rctx = z::DCtx::new(); + let (cr, rr); + if prefix { + cctx.ref_prefix(dict).unwrap(); + cr = cctx.decompress(&mut cd, frames); + // an empty prefix is no dictionary at all + rr = if dict.is_empty() { rs(rctx.decompress(&mut rd, frames)) } else { rs(z::DDict::prefix(dict).and_then(|d| rctx.decompress_using_ddict(&mut rd, frames, &d))) }; + } else { + assert_eq!(cctx.load_dict(dict).map(|_| ()), rs(rctx.load_dictionary(dict)), "loadDictionary"); + cr = cctx.decompress(&mut cd, frames); + rr = rs(rctx.decompress(&mut rd, frames)); + } + assert_eq!(cr, rr, "one-shot result (cap {cap})"); + if let Ok(n) = cr { assert!(cd[..n] == rd[..n], "one-shot output"); } +} + +fn streaming(rng: &mut Rng, frames: &[u8], dict: &[u8], expect: Option<&[u8]>) { + let mut cctx = c::DCtx::new(); + let mut rctx = z::DCtx::new(); + if rng.chance(1, 6) { + let (p, v) = (*rng.pick(&[100u32, 100, 100, 0, 99, 1]), *rng.pick(&[0i32, 9, 10, 15, 20, 27, 31, 32, -1])); + assert_eq!(cctx.set(p, v).map(|_| ()), rs(rctx.set_parameter(p as i32, v)), "setParameter {p} {v}"); + } + assert_eq!(cctx.load_dict(dict).map(|_| ()), rs(rctx.load_dictionary(dict)), "loadDictionary"); + // tiny steps only where they do not take forever + let small = frames.len() < 3_000; + let in_max = *rng.pick(if small { &[1usize, 2, 3, 7, 64, 1000, 70_000, 10_000_000] } else { &[300usize, 1000, 5000, 20_000, 70_000, 200_000, 10_000_000, 10_000_000] }); + let out_max = *rng.pick(if small && expect.is_none_or(|e| e.len() < 20_000) { &[1usize, 2, 5, 100, 4096, 16_384, 140_000, 10_000_000] } else { &[1000usize, 4096, 4096, 16_384, 16_384, 140_000, 1_000_000, 10_000_000] }); + let mut total = Vec::new(); + let (mut cbuf, mut rbuf) = (Vec::new(), Vec::new()); + let mut ipos = 0usize; + let mut stalls = 0; + loop { + let iend = (ipos + rng.range(if stalls > 0 { 1 } else { 0 }, in_max)).min(frames.len()); + let cap = rng.range(if stalls > 1 { 1 } else { 0 }, out_max); + let chunk = &frames[ipos..iend]; + if cbuf.len() < cap { cbuf.resize(cap, 0x55); rbuf.resize(cap, 0x55); } + let cd = &mut cbuf[..cap]; + let rd = &mut rbuf[..cap]; + let (cr, co, ci) = cctx.stream(cd, 0, chunk, 0); + let mut ob = z::OutBuffer { dst: rd, pos: 0 }; + let mut ib = z::InBuffer { src: chunk, pos: 0 }; + let rr = rs(rctx.decompress_stream(&mut ob, &mut ib)); + let (ro, ri) = (ob.pos, ib.pos); + assert_eq!((cr, co, ci), (rr, ro, ri), "stream step at {ipos} (in {} out {cap})", chunk.len()); + assert!(cbuf[..co] == rbuf[..ro], "stream output at {ipos}"); + total.extend_from_slice(&rbuf[..ro]); + ipos += ri; + if cr.is_err() { return; } + if ri == 0 && ro == 0 { stalls += 1 } else { stalls = 0 } + if ipos == frames.len() && (cr == Ok(0) || stalls > 3) { break; } + if cr == Ok(0) && rng.chance(1, 8) { + match rng.below(3) { + 0 => { cctx.init_stream(); rctx.init_stream(); if !dict.is_empty() { return; } } + 1 => { assert_eq!(cctx.reset(1).map(|_| ()), rs(rctx.reset(z::ResetDirective::SessionOnly))); } + _ => {} + } + } + } + if let Some(expect) = expect { assert!(total == expect, "stream total"); } +} + +pub fn run(seed: u64, iters: u64) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if i % 200 == 0 { eprintln!("dec iter {i} (seed {s})"); } + let (mut frames, content, dict) = make_frames(&mut rng); + // valid + check_sizes(&frames); + one_shot(&frames, &dict, content.len(), false); + one_shot(&frames, &dict, content.len() + rng.size(300_000), false); + if !content.is_empty() { let cap = rng.below(content.len()); one_shot(&frames, &dict, cap, false); } + streaming(&mut rng, &frames, &dict, Some(&content)); + streaming(&mut rng, &frames, &dict, Some(&content)); + // wrong or missing dictionary + if rng.chance(1, 4) { let n = rng.size(5000); let other = data(&mut rng, n); one_shot(&frames, &other, content.len(), rng.chance(1, 2)); streaming(&mut rng, &frames, &other, None); } + // damaged + for _ in 0..6 { + let mut bad = frames.clone(); + mutate(&mut rng, &mut bad); + check_sizes(&bad); + let cap = content.len() + if rng.chance(1, 2) { rng.size(200_000) } else { 0 }; + one_shot(&bad, &dict, cap, false); + streaming(&mut rng, &bad, &dict, None); + } + mutate(&mut rng, &mut frames); + } + println!("dec ok: {iters} iterations from seed {seed}"); +} diff --git a/src/zstd/wip/harness/src/main.rs b/src/zstd/wip/harness/src/main.rs new file mode 100644 index 000000000000..bf2b969a3a4b --- /dev/null +++ b/src/zstd/wip/harness/src/main.rs @@ -0,0 +1,18 @@ +mod c; +mod comp; +mod data; +mod dec; + +fn main() { + let args: Vec = std::env::args().collect(); + let cmd = args.get(1).map(String::as_str).unwrap_or(""); + let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); + let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); + match cmd { + "dec" => dec::run(seed, iters), + "dict" => comp::run_dict(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(1 << 18)), + "mt" => comp::run_mt(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(8 << 20)), + "comp" => comp::run(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(1 << 20)), + _ => eprintln!("usage: zstd_harness dec "), + } +} diff --git a/src/zstd/wip/par.sh b/src/zstd/wip/par.sh new file mode 100755 index 000000000000..eb2cc7acaa98 --- /dev/null +++ b/src/zstd/wip/par.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# par.sh : runs the harness in parallel, prints failures. +cmd=$1; first=$2; procs=$3; iters=$4 +mkdir -p /root/zstd-ref/logs +for i in $(seq 0 $((procs-1))); do + seed=$((first+i)) + ( /root/zstd-ref/harness/target/release/zstd_harness $cmd $seed $iters > /root/zstd-ref/logs/$cmd-$seed.log 2>&1 || echo "FAIL $cmd seed $seed" ) & +done +wait +grep -l "panicked\|Sanitizer\|Aborted" /root/zstd-ref/logs/$cmd-*.log 2>/dev/null | head +echo "par done: $cmd $first+$procs x $iters" diff --git a/src/zstd/wip/strip.py b/src/zstd/wip/strip.py new file mode 100644 index 000000000000..0cc7c6a7f0c8 --- /dev/null +++ b/src/zstd/wip/strip.py @@ -0,0 +1,7 @@ +import re,sys +src=open(sys.argv[1]).read() +# remove comments, respecting strings +out=re.sub(r'//[^\n]*|/\*.*?\*/|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'', lambda m: m.group(0) if m.group(0)[0] in '"\'' else ' ', src, flags=re.S) +lines=[l.rstrip() for l in out.split('\n')] +lines=[l for l in lines if l.strip() and not re.match(r'\s*(DEBUGLOG|RAWLOG)\(.*\);\s*$', l)] +open(sys.argv[2],'w').write('\n'.join(lines)+'\n') diff --git a/src/zstd/wip/template/mkwork.sh b/src/zstd/wip/template/mkwork.sh new file mode 100755 index 000000000000..e3150a313902 --- /dev/null +++ b/src/zstd/wip/template/mkwork.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# mkwork.sh : a scratch copy of the crate and of the harness, at /root/zstd-ref/work//{zstd,harness} +set -e +name=$1; w=/root/zstd-ref/work/$name +mkdir -p $w && rm -rf $w/zstd $w/harness +cp -r /root/repos/oven-sh/bun/src/zstd $w/zstd +cp /root/zstd-ref/template/testing.rs $w/zstd/testing.rs +echo 'pub mod testing;' >> $w/zstd/lib.rs +python3 -I - "$w" <<'PY' +import re,sys +w=sys.argv[1] +root=open('/root/repos/oven-sh/bun/Cargo.toml').read() +def table(name): + m=re.search(r'^\[workspace\.lints\.'+name+r'\]\n(.*?)(?=^\[)',root,re.S|re.M) + return m.group(1) +open(w+'/zstd/Cargo.toml','w').write(f'''[package] +name = "bun_zstd" +version = "0.0.0" +edition = "2024" + +[lib] +path = "lib.rs" + +[dependencies] +strum = {{ version = "0.26", features = ["derive"] }} +bun_pod = {{ path = "/root/repos/oven-sh/bun/src/pod", features = ["alloc"] }} + +[workspace] + +[lints.rust] +{table("rust")} +[lints.clippy] +{table("clippy")} +''') +PY +cp /root/repos/oven-sh/bun/rust-toolchain.toml /root/repos/oven-sh/bun/clippy.toml /root/repos/oven-sh/bun/rustfmt.toml $w/zstd/ 2>/dev/null || true +mkdir -p $w/harness/src +cp /root/zstd-ref/harness/{Cargo.toml,build.rs,rust-toolchain.toml} $w/harness/ +cp /root/zstd-ref/harness/src/{c.rs,data.rs} /root/zstd-ref/template/seqs.rs $w/harness/src/ +sed -i "s#/root/repos/oven-sh/bun/src/zstd#$w/zstd#" $w/harness/Cargo.toml +cat > $w/harness/src/main.rs <<'RS' +mod c; +mod data; +mod seqs; + +fn main() { + let args: Vec = std::env::args().collect(); + let cmd = args.get(1).map(String::as_str).unwrap_or(""); + let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); + let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); + match cmd { + // seqs + "seqs" => { + let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); + seqs::run(seed, iters, &strategies, args.get(5).is_some_and(|s| s == "1")); + } + _ => eprintln!("unknown command"), + } +} +RS +echo "$w ready" diff --git a/src/zstd/wip/template/seqs.rs b/src/zstd/wip/template/seqs.rs new file mode 100644 index 000000000000..7c03d3b0f952 --- /dev/null +++ b/src/zstd/wip/template/seqs.rs @@ -0,0 +1,73 @@ +//! One block through a Rust block compressor and through the C library (`ZSTD_generateSequences`). +use crate::c; +use crate::data::{Rng, data}; +use std::ffi::{c_uint, c_void}; + +#[repr(C)] +#[derive(Clone, Copy, Default, Debug)] +struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } +#[repr(C)] +#[derive(Clone, Copy, Debug)] +struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } +unsafe extern "C" { + fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; + fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; + fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; +} + +fn c_sequences(cp: [u32; 7], use_row: bool, dict: &[u8], src: &[u8]) -> Vec<(u32, u32, u32)> { + let mut cctx = c::CCtx::new(); + for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } + cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder + cctx.set(1010, 2).unwrap(); // no block splitter + if !dict.is_empty() { cctx.ref_prefix(dict).unwrap(); } + let mut out = vec![Sequence::default(); src.len() / 3 + 8]; + let n = c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }).unwrap(); + out[..n].iter().map(|s| (s.lit_length, s.match_length, s.offset)).collect() +} + +/// `strategies`: which of 1..=9 to test. `with_dict`: also test with a prefix, which is the extDict variants. +pub fn run(seed: u64, iters: u64, strategies: &[u32], with_dict: bool) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if i % 500 == 0 { eprintln!("seqs iter {i} (seed {s})"); } + let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071]); + let len = rng.range(7, max); + let mut src = data(&mut rng, len); + let dict = if with_dict && rng.chance(1, 2) { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; + if !dict.is_empty() && rng.chance(2, 3) { + for _ in 0..rng.range(1, 6) { + let n = rng.size(dict.len().min(src.len())); + let at = rng.below(dict.len() - n + 1); + let to = rng.below(src.len() - n + 1); + src[to..to + n].copy_from_slice(&dict[at..at + n]); + } + } + let strategy = *rng.pick(strategies); + let p = if rng.chance(1, 3) { + // the parameters of a level, with the strategy forced + let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; + p.strategy = strategy; + p + } else { + CParams { + window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, + search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, + target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, + } + }; + let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; + let cp = [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy]; + let use_row = rng.chance(1, 2); + let expected = c_sequences(cp, use_row, &dict, &src); + let got = bun_zstd::testing::first_block(cp, use_row, &dict, &src); + let got = got.unwrap_or_else(|e| panic!("seed {s}: error {e:?} (cp {cp:?} row {use_row} dict {} src {})", dict.len(), src.len())); + if got != expected { + let at = got.iter().zip(&expected).position(|(a, b)| a != b).unwrap_or(got.len().min(expected.len())); + panic!("seed {s}: cp {cp:?} row {use_row} dict {} src {}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", + dict.len(), src.len(), got.get(at), expected.get(at), got.len(), expected.len()); + } + } + println!("seqs ok: {iters} iterations from seed {seed}, strategies {strategies:?}, dict {with_dict}"); +} diff --git a/src/zstd/wip/template/testing.rs b/src/zstd/wip/template/testing.rs new file mode 100644 index 000000000000..ccf602c2a591 --- /dev/null +++ b/src/zstd/wip/template/testing.rs @@ -0,0 +1,140 @@ +//! Hooks for the differential tests. Only in scratch copies of the crate. +#![allow(clippy::all, dead_code, unreachable_pub)] + +use crate::ErrorCode; +use crate::compress::internal::*; +use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; + +fn bitmix(mut val: u64, len: u64) -> u64 { + val ^= val.rotate_right(49) ^ val.rotate_right(24); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^= (val >> 35).wrapping_add(len); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^ (val >> 28) +} + +/// The sequences of the first block of a frame, as (literal length, match length, offset), and then the literals +/// that are left as (length, 0, 0). +/// +/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, already adjusted to the sizes. +/// `dict` is a prefix (raw content, as `ZSTD_CCtx_refPrefix`), or empty. `src` is from 7 to 131071 bytes. +pub fn first_block(cp: [u32; 7], use_row: bool, dict: &[u8], src: &[u8]) -> Result, ErrorCode> { + let strategy = Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?; + let c_params = CompressionParameters { + window_log: cp[0], chain_log: cp[1], hash_log: cp[2], search_log: cp[3], min_match: cp[4], target_length: cp[5], + strategy: Some(strategy), + }; + let row_used = row_match_finder_supported(strategy) && use_row; + + // ZSTD_reset_matchState + let mut ms = MatchState::new(); + ms.hash_log3 = if c_params.min_match == 3 { c_params.window_log.min(17) } else { 0 }; + ms.window.clear(); + ms.next_to_update = ms.window.dict_limit; + ms.hash_table = vec![0; 1 << c_params.hash_log]; + ms.chain_table = vec![0; if strategy != Strategy::Fast && !row_used { 1 << c_params.chain_log } else { 0 }]; + ms.hash_table3 = vec![0; if ms.hash_log3 != 0 { 1 << ms.hash_log3 } else { 0 }]; + if row_used { + ms.tag_table = vec![0; 1 << c_params.hash_log]; + ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4); + ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); + } + if strategy >= Strategy::BtOpt { + ms.opt.lit_freq = vec![0; 256]; + ms.opt.lit_length_freq = vec![0; 36]; + ms.opt.match_length_freq = vec![0; 53]; + ms.opt.off_code_freq = vec![0; 32]; + ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; + ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; + } + ms.c_params = c_params; + + let block_size = (1usize << c_params.window_log).min(128 << 10); + let mut seq_store = SeqStore::new(); + seq_store.reserve(block_size / if c_params.min_match == 3 { 3 } else { 4 }, block_size)?; + + let bufs = Buffers { in_buff: &[], user: src, dict }; + + // ZSTD_loadDictionaryContent(dtlm_fast, tfp_forCCtx) + if !dict.is_empty() { + let iend = dict.len(); + ms.window.update(BufId::Dict.start(), BufId::Dict, dict.len(), false); + let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); + let ip = iend - dict.len().min(max_dict_size); + let prefix = Prefix { buf: dict, base: ms.window.base.wrapping_sub(BufId::Dict.start()) }; + ms.next_to_update = prefix.index_of(ip); + ms.loaded_dict_end = prefix.index_of(iend); + if iend - ip > HASH_READ_SIZE { + match strategy { + Strategy::Fast => fast::fill_hash_table(&mut ms, prefix, iend, DictTableLoadMethod::Fast, TableFillPurpose::ForCCtx)?, + Strategy::DFast => double_fast::fill_double_hash_table(&mut ms, prefix, iend, DictTableLoadMethod::Fast, TableFillPurpose::ForCCtx)?, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { + if use_row { + ms.tag_table.fill(0); + lazy::row_update(&mut ms, prefix, iend - HASH_READ_SIZE)?; + } else { + lazy::insert_and_find_first_index(&mut ms, prefix, iend - HASH_READ_SIZE)?; + } + } + _ => opt::update_tree(&mut ms, prefix, iend - HASH_READ_SIZE, iend)?, + } + ms.next_to_update = prefix.index_of(iend); + } + } + + // ZSTD_compressContinue_internal + let src_addr = BufId::User.start(); + if !ms.window.update(src_addr, BufId::User, src.len(), ms.force_non_contiguous) { + ms.force_non_contiguous = false; + ms.next_to_update = ms.window.dict_limit; + } + // ZSTD_compress_frameChunk + let max_dist = 1u32 << c_params.window_log; + if ms.window.dict_is_invalid(src_addr + src.len(), max_dist, ms.loaded_dict_end) { + ms.loaded_dict_end = 0; + } + if ms.window.enforce_max_dist(src_addr, max_dist, ms.loaded_dict_end) { + ms.loaded_dict_end = 0; + } + if ms.next_to_update < ms.window.low_limit { + ms.next_to_update = ms.window.low_limit; + } + // ZSTD_buildSeqStore + let prev = CompressedBlockState::new(); + let block = BlockSrc { + buf: bufs.get(ms.window.base_buf), + base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), + dict_buf: bufs.get(ms.window.dict_buf), + dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), + start: 0, + end: src.len(), + dms: None, + symbol_costs: &prev.entropy, + ldm_seq_store: None, + }; + let curr = block.index_of(0); + if curr > ms.next_to_update + 384 { + ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); + } + let dict_mode = ms.dict_mode(None); + let mut rep = prev.rep; + let last_ll = compress::compress_block(&mut ms, &mut seq_store, &mut rep, &block, strategy, use_row, dict_mode)?; + + let mut out = Vec::new(); + let mut reps = prev.rep; + for (i, seq) in seq_store.sequences.iter().enumerate() { + let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); + if seq_store.long_length_pos as usize == i { + match seq_store.long_length_type { + LongLengthType::LiteralLength => ll += 0x10000, + LongLengthType::MatchLength => ml += 0x10000, + LongLengthType::None => {} + } + } + update_rep(&mut reps, seq.off_base, seq.lit_length == 0); + out.push((ll, ml, reps[0])); + } + out.push((last_ll as u32, 0, 0)); + assert_eq!(reps, rep, "the repeat offsets the block compressor leaves are those of its sequences"); + Ok(out) +} diff --git a/src/zstd/wip/work/entropy/harness/Cargo.toml b/src/zstd/wip/work/entropy/harness/Cargo.toml new file mode 100644 index 000000000000..814bb28bb616 --- /dev/null +++ b/src/zstd/wip/work/entropy/harness/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "zstd_harness" +version = "0.0.0" +edition = "2024" + +[dependencies] +bun_zstd = { path = "/root/zstd-ref/work/entropy/zstd" } + +[profile.release] +debug = "line-tables-only" +overflow-checks = true +debug-assertions = true +panic = "abort" + +[profile.bench] +inherits = "release" +overflow-checks = false +debug-assertions = false + +[workspace] diff --git a/src/zstd/wip/work/entropy/harness/build.rs b/src/zstd/wip/work/entropy/harness/build.rs new file mode 100644 index 000000000000..582ebed0a7b9 --- /dev/null +++ b/src/zstd/wip/work/entropy/harness/build.rs @@ -0,0 +1,20 @@ +use std::process::Command; + +fn main() { + let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); + println!("cargo:rerun-if-env-changed=ZSTD_REF"); + println!("cargo:rustc-link-search=native=/root/zstd-ref/build/{variant}"); + println!("cargo:rustc-link-lib=static=zstd"); + + // The inline functions of fse.h. + println!("cargo:rerun-if-changed=entropy_shim.c"); + let out = std::env::var("OUT_DIR").unwrap(); + let ok = Command::new("clang-23") + .args(["-c", "-O2", "-g", "-fPIC", "-march=nehalem", "-I/root/zstd-ref/zstd/lib", "-I/root/zstd-ref/zstd/lib/common", "entropy_shim.c", "-o"]) + .arg(format!("{out}/entropy_shim.o")).status().unwrap().success(); + assert!(ok); + let ok = Command::new("llvm-ar-23").args(["rcs", &format!("{out}/libentropy_shim.a"), &format!("{out}/entropy_shim.o")]).status().unwrap().success(); + assert!(ok); + println!("cargo:rustc-link-search=native={out}"); + println!("cargo:rustc-link-lib=static=entropy_shim"); +} diff --git a/src/zstd/wip/work/entropy/harness/entropy_shim.c b/src/zstd/wip/work/entropy/harness/entropy_shim.c new file mode 100644 index 000000000000..32426c9fe716 --- /dev/null +++ b/src/zstd/wip/work/entropy/harness/entropy_shim.c @@ -0,0 +1,43 @@ +/* The inline functions of fse.h, for the differential test. */ +#define FSE_STATIC_LINKING_ONLY +#include "mem.h" +#include "bitstream.h" +#include "fse.h" + +size_t shim_fse_encode3(void* dst, size_t cap, const FSE_CTable* ta, const FSE_CTable* tb, const FSE_CTable* tc, + const BYTE* sa, const BYTE* sb, const BYTE* sc, size_t n) +{ + BIT_CStream_t bitC; + FSE_CState_t a, b, c, d; + size_t i; + { size_t const e = BIT_initCStream(&bitC, dst, cap); if (ERR_isError(e)) return e; } + FSE_initCState2(&a, ta, sa[n-1]); + FSE_initCState2(&b, tb, sb[n-1]); + FSE_initCState2(&c, tc, sc[n-1]); + for (i = n-2; i < n; i--) { + FSE_encodeSymbol(&bitC, &b, sb[i]); + FSE_encodeSymbol(&bitC, &a, sa[i]); + FSE_encodeSymbol(&bitC, &c, sc[i]); + BIT_flushBits(&bitC); + } + FSE_flushCState(&bitC, &a); + FSE_flushCState(&bitC, &b); + FSE_flushCState(&bitC, &c); + FSE_initCState(&d, ta); + FSE_flushCState(&bitC, &d); + return BIT_closeCStream(&bitC); +} + +U32 shim_fse_getMaxNbBits(const FSE_CTable* ct, U32 symbol) +{ + FSE_CState_t s; + FSE_initCState(&s, ct); + return FSE_getMaxNbBits(s.symbolTT, symbol); +} + +U32 shim_fse_bitCost(const FSE_CTable* ct, U32 symbol, U32 accuracyLog) +{ + FSE_CState_t s; + FSE_initCState(&s, ct); + return FSE_bitCost(s.symbolTT, s.stateLog, symbol, accuracyLog); +} diff --git a/src/zstd/wip/work/entropy/harness/rust-toolchain.toml b/src/zstd/wip/work/entropy/harness/rust-toolchain.toml new file mode 100644 index 000000000000..c71778e8ccd3 --- /dev/null +++ b/src/zstd/wip/work/entropy/harness/rust-toolchain.toml @@ -0,0 +1,35 @@ +[toolchain] +channel = "nightly-2026-09-15" +# rust-src is needed for -Zbuild-std (Tier 3 targets like +# aarch64-unknown-freebsd have no prebuilt std). miri is for +# `bun run rust:miri`. targets ensures the +# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain +# (CI's images install this same channel with these components and +# targets, from scripts/build/ci-images/spec.ts; a source lint keeps +# the two files in step). Note: rustup ignores this whole file when +# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree +# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` +# self-heals that path. Keep this list as a fallback for direct +# `cargo build`/`cargo check`/rust-analyzer use. +# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd +# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). +# darwin/windows triples build natively on their own agents, so the host std +# comes with the toolchain install regardless; they're listed here so a dev +# running `bun run rust:check-all` or `cargo check --target` from any host +# still gets prebuilt std for every Tier 1/2 triple. +# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at +# what it emits. The build does not use them. +components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] +targets = [ + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-musl", + "aarch64-linux-android", + "x86_64-linux-android", + "x86_64-unknown-freebsd", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-pc-windows-msvc", + "x86_64-pc-windows-msvc", +] diff --git a/src/zstd/wip/work/entropy/harness/src/c.rs b/src/zstd/wip/work/entropy/harness/src/c.rs new file mode 100644 index 000000000000..d5a82e1f6b78 --- /dev/null +++ b/src/zstd/wip/work/entropy/harness/src/c.rs @@ -0,0 +1,104 @@ +//! The reference C library. +#![allow(non_camel_case_types, dead_code)] +use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; + +#[repr(C)] +pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } +#[repr(C)] +pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } +pub enum ZSTD_CCtx {} +pub enum ZSTD_DCtx {} +pub enum ZSTD_DDict {} + +unsafe extern "C" { + pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; + pub fn ZSTD_compressBound(size: usize) -> usize; + pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_isError(code: usize) -> c_uint; + pub fn ZSTD_getErrorCode(code: usize) -> c_uint; + pub fn ZSTD_getErrorName(code: usize) -> *const c_char; + pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; + pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; + pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; + pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; + pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; + pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; + pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; + pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; + pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; + pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; + pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; + pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; + pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; + pub fn ZSTD_defaultCLevel() -> c_int; + pub fn ZSTD_minCLevel() -> c_int; + pub fn ZSTD_maxCLevel() -> c_int; +} + +/// `Ok(value)` or `Err(error code)`. +pub fn res(code: usize) -> Result { + unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } +} + +pub struct CCtx(pub *mut ZSTD_CCtx); +impl CCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } + pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + /// Returns (result, out pos, in pos). + pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); + (r, o.pos, i.pos) + } +} +impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } + +pub struct DCtx(pub *mut ZSTD_DCtx); +impl DCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } + pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } + pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); + (r, o.pos, i.pos) + } +} +impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } + +pub fn compress(src: &[u8], level: i32) -> Vec { + let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; + let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); + dst.truncate(n); + dst +} +pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) +} diff --git a/src/zstd/wip/work/entropy/harness/src/data.rs b/src/zstd/wip/work/entropy/harness/src/data.rs new file mode 100644 index 000000000000..c48a645e1b5d --- /dev/null +++ b/src/zstd/wip/work/entropy/harness/src/data.rs @@ -0,0 +1,65 @@ +//! Deterministic inputs. +pub struct Rng(pub u64); +impl Rng { + pub fn next(&mut self) -> u64 { + // splitmix64 + self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + } + pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } + pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } + pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } + pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } + /// Skewed towards small values. + pub fn size(&mut self, max: usize) -> usize { + let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); + (self.next() as usize & ((1usize << bits) - 1)).min(max) + } +} + +static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; + +/// One buffer of about `len` bytes, of a kind picked by `rng`. +pub fn data(rng: &mut Rng, len: usize) -> Vec { + let mut v = Vec::with_capacity(len + 64); + match rng.below(9) { + 0 => v.resize(len, rng.next() as u8), + 1 => while v.len() < len { v.push(rng.next() as u8) }, + 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, + 3 => { // few symbols, skewed + let n = rng.range(2, 40); + while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } + } + 4 => { // LZ-like: copies of earlier parts, at all distances + while v.len() < len { + if v.len() > 8 && rng.chance(3, 4) { + let dist = 1 + rng.size(v.len() - 1); + let n = 3 + rng.size(300); + for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } + } else { + for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } + } + } + } + 5 => { // periodic with noise + let period = rng.range(1, 300); + let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); + while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } + } + 6 => { // sections of different kinds + while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } + } + 7 => { // numbers as text + let mut x = rng.next() % 1000; + while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } + } + _ => { // long runs + while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } + } + } + v.truncate(len); + v +} diff --git a/src/zstd/wip/work/entropy/harness/src/entropy.rs b/src/zstd/wip/work/entropy/harness/src/entropy.rs new file mode 100644 index 000000000000..120a61fc48ce --- /dev/null +++ b/src/zstd/wip/work/entropy/harness/src/entropy.rs @@ -0,0 +1,774 @@ +//! The entropy encoders (`hist`, `fse`, `huf`) against the C library, function by function. +use crate::c::res; +use crate::data::{Rng, data}; +use bun_zstd::ErrorCode; +use bun_zstd::testing::entropy as rs; +use std::ffi::{c_int, c_uint, c_void}; + +#[repr(C)] +struct CTableHeader { table_log: u8, max_symbol_value: u8, unused: [u8; 6] } + +unsafe extern "C" { + fn HIST_add(count: *mut c_uint, src: *const c_void, size: usize); + fn HIST_count_simple(count: *mut c_uint, max: *mut c_uint, src: *const c_void, size: usize) -> c_uint; + fn HIST_countFast_wksp(count: *mut c_uint, max: *mut c_uint, src: *const c_void, size: usize, wksp: *mut c_void, wksp_size: usize) -> usize; + fn HIST_count_wksp(count: *mut c_uint, max: *mut c_uint, src: *const c_void, size: usize, wksp: *mut c_void, wksp_size: usize) -> usize; + + fn FSE_buildCTable_wksp(ct: *mut u32, norm: *const i16, max: c_uint, table_log: c_uint, wksp: *mut c_void, wksp_size: usize) -> usize; + fn FSE_buildCTable_rle(ct: *mut u32, symbol: u8) -> usize; + fn FSE_NCountWriteBound(max: c_uint, table_log: c_uint) -> usize; + fn FSE_writeNCount(buf: *mut c_void, size: usize, norm: *const i16, max: c_uint, table_log: c_uint) -> usize; + fn FSE_optimalTableLog(max_log: c_uint, size: usize, max: c_uint) -> c_uint; + fn FSE_optimalTableLog_internal(max_log: c_uint, size: usize, max: c_uint, minus: c_uint) -> c_uint; + fn FSE_normalizeCount(norm: *mut i16, table_log: c_uint, count: *const c_uint, total: usize, max: c_uint, low_prob: c_uint) -> usize; + fn FSE_compress_usingCTable(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, ct: *const u32) -> usize; + + fn HUF_compressBound(size: usize) -> usize; + fn HUF_buildCTable_wksp(t: *mut u64, count: *const c_uint, max: u32, max_nb_bits: u32, wksp: *mut c_void, wksp_size: usize) -> usize; + fn HUF_writeCTable_wksp(dst: *mut c_void, cap: usize, t: *const u64, max: c_uint, huff_log: c_uint, wksp: *mut c_void, wksp_size: usize) -> usize; + fn HUF_readCTable(t: *mut u64, max: *mut c_uint, src: *const c_void, size: usize, has_zero: *mut c_uint) -> usize; + fn HUF_estimateCompressedSize(t: *const u64, count: *const c_uint, max: c_uint) -> usize; + fn HUF_validateCTable(t: *const u64, count: *const c_uint, max: c_uint) -> c_int; + fn HUF_optimalTableLog(max_log: c_uint, size: usize, max: c_uint, wksp: *mut c_void, wksp_size: usize, t: *mut u64, count: *const c_uint, flags: c_int) -> c_uint; + fn HUF_minTableLog(cardinality: c_uint) -> c_uint; + fn HUF_cardinality(count: *const c_uint, max: c_uint) -> c_uint; + fn HUF_getNbBitsFromCTable(t: *const u64, symbol: u32) -> u32; + fn HUF_readCTableHeader(t: *const u64) -> CTableHeader; + fn HUF_compress1X_usingCTable(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, t: *const u64, flags: c_int) -> usize; + fn HUF_compress4X_usingCTable(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, t: *const u64, flags: c_int) -> usize; + fn HUF_compress1X_repeat(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, max: c_uint, log: c_uint, wksp: *mut c_void, wksp_size: usize, t: *mut u64, repeat: *mut c_uint, flags: c_int) -> usize; + fn HUF_compress4X_repeat(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, max: c_uint, log: c_uint, wksp: *mut c_void, wksp_size: usize, t: *mut u64, repeat: *mut c_uint, flags: c_int) -> usize; + + fn shim_fse_encode3(dst: *mut c_void, cap: usize, ta: *const u32, tb: *const u32, tc: *const u32, sa: *const u8, sb: *const u8, sc: *const u8, n: usize) -> usize; + fn shim_fse_getMaxNbBits(ct: *const u32, symbol: u32) -> u32; + fn shim_fse_bitCost(ct: *const u32, symbol: u32, accuracy_log: u32) -> u32; +} + +type R = Result; +fn conv(r: Result) -> R { r.map_err(|e| e as u8 as u32) } + +/// What the C code may write past the end of, in cases that are bugs of the caller. +const SLACK: usize = 2048; +const FILL: u8 = 0xA5; + +/// A workspace for the C code, zeroed. +fn wksp() -> Vec { vec![0u64; 8704 / 8 + 64] } +const WKSP_SIZE: usize = 8704; + +pub struct Stats { pub cases: u64 } +impl Stats { fn hit(&mut self) { self.cases += 1; } } + +/// An output buffer for each side. The C one has slack after `cap`. +struct Out { c: Vec, r: Vec, cap: usize } +impl Out { + fn new(cap: usize) -> Self { Self { c: vec![FILL; cap + SLACK], r: vec![FILL; cap], cap } } + fn c_ptr(&mut self) -> *mut c_void { self.c.as_mut_ptr().cast() } + #[track_caller] + fn check(&self, what: &str, ctx: &dyn std::fmt::Debug) { + assert!(self.c[..self.cap] == self.r[..], "{what}: output differs at {:?} ({ctx:?})", self.c.iter().zip(&self.r).position(|(a, b)| a != b)); + assert!(self.c[self.cap..].iter().all(|&b| b == FILL), "{what}: the C code wrote past the end ({ctx:?})"); + } +} + +// ---------------------------------------------------------------- inputs + +fn sizes(rng: &mut Rng) -> usize { + match rng.below(8) { + 0 => *rng.pick(&[0usize, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 15, 16, 17, 19, 20, 21, 31, 32, 33, 63, 64, 65]), + 1 => *rng.pick(&[255usize, 256, 257, 1023, 1024, 1025, 1499, 1500, 1501, 4095, 4096, 40959, 40960, 40961, 65535, 65536, 131071, 131072]), + 2 => rng.range(0, 64), + 3 | 4 => rng.range(0, 2000), + 5 | 6 => rng.range(0, 20000), + _ => rng.range(0, 131072), + } +} + +/// 256 counts, of which those above `max` are 0. At least one is not 0. They add up to at most about `budget`. +fn counts(rng: &mut Rng, max: usize, budget: usize) -> Vec { + let n = max + 1; + let mut c = vec![0u32; 256]; + match rng.below(12) { + 0 => { let len = rng.range(1, budget); for b in data(rng, len) { c[b as usize % n] += 1; } } + 1 => { // geometric + let mut v = budget as f64 / 2.0; let ratio = 0.3 + 0.69 * (rng.below(1000) as f64 / 1000.0); + for x in c[..n].iter_mut() { *x = v as u32; v *= ratio; } + } + 2 => { // Fibonacci: the deepest trees + let (mut a, mut b) = (1u32, 1u32); let mut total = 0usize; + for x in c[..n].iter_mut() { if total + a as usize > budget { break; } *x = a; total += a as usize; (a, b) = (b, a + b); } + } + 3 => { let v = (*rng.pick(&[1u32, 2, 3, 163, 164, 165, 166, 200, 255, 256, 1000])).min((budget / n).max(1) as u32); c[..n].fill(v); } + 4 => { // many ties, around the counts that share buckets + let lo = rng.range(1, 600).min((budget / n).max(1)); let span = rng.range(1, 300); + for x in c[..n].iter_mut() { *x = (lo + rng.below(span)).min((budget / n).max(1)) as u32; } + } + 5 => { for _ in 0..rng.range(1, 6) { c[rng.below(n)] = rng.range(1, (budget / 8).max(1)) as u32; } } + 6 => { c[..n].fill(1); c[rng.below(n)] = (budget.saturating_sub(n)).max(1) as u32; } + 7 => { let top = (usize::BITS - (budget / n).max(1).leading_zeros()) as usize; for x in c[..n].iter_mut() { *x = 1 << rng.below(top); } } + 8 => { for x in c[..n].iter_mut() { if rng.chance(1, 2) { *x = rng.size((2 * budget / n).max(1)) as u32; } } } + 9 => { for x in c[..n].iter_mut() { *x = rng.range(0, 3) as u32; } } + 10 => { // two plateaus + let hi = (budget / n).max(2) as u32; let cut = rng.below(n); + for (i, x) in c[..n].iter_mut().enumerate() { *x = if i < cut { hi } else { 1 }; } + } + _ => { let m = (budget / n).max(1); for x in c[..n].iter_mut() { *x = rng.range(0, m) as u32; } } + } + if rng.chance(1, 2) { // any order + for i in (1..n).rev() { let j = rng.below(i + 1); c.swap(i, j); } + } + if c.iter().all(|&x| x == 0) { c[rng.below(n)] = rng.range(1, 5) as u32; } + c +} + +/// Symbols that occur as `counts` says, in any order. +fn data_of(rng: &mut Rng, counts: &[u32]) -> Vec { + let mut v = Vec::new(); + for (s, &c) in counts.iter().enumerate() { v.resize(v.len() + c as usize, s as u8); } + match rng.below(4) { + 0 => {} + 1 => v.reverse(), + _ => for i in (1..v.len()).rev() { let j = rng.below(i + 1); v.swap(i, j); }, + } + v +} + +/// `len` symbols among those whose weight is not 0. +fn data_among(rng: &mut Rng, weights: &[i16], len: usize) -> Vec { + let mut pool = Vec::new(); + for (s, &w) in weights.iter().enumerate() { for _ in 0..w.unsigned_abs() { pool.push(s as u8); } } + let uniform: Vec = weights.iter().enumerate().filter(|(_, w)| **w != 0).map(|(s, _)| s as u8).collect(); + let pool = if rng.chance(1, 4) { uniform } else { pool }; + (0..len).map(|_| *rng.pick(&pool)).collect() +} + +/// Normalized counts that add up to `1 << table_log`, made directly. +fn random_norm(rng: &mut Rng, max: usize, table_log: u32) -> Vec { + let size = 1usize << table_log; + let mut norm = vec![0i16; max + 1]; + let mut left = size; + let low = if rng.chance(1, 2) { rng.below((max + 1).min(size - 1) + 1) } else { 0 }; + let mut order: Vec = (0..=max).collect(); + for i in (1..order.len()).rev() { let j = rng.below(i + 1); order.swap(i, j); } + for &s in &order[..low] { norm[s] = -1; left -= 1; } + let others = &order[low..]; + if others.is_empty() || left == 0 { + // All the slots must be taken: give what is left to a low probability symbol. + if left > 0 { let s = order[0]; norm[s] = 1 + left as i16; } + return norm; + } + let used = rng.range(1, others.len().min(left)); + for &s in &others[..used] { norm[s] = 1; left -= 1; } + while left > 0 { + let s = others[rng.below(used)]; + let n = if rng.chance(1, 3) { left } else { 1 + rng.size(left - 1) }; + norm[s] += n as i16; left -= n; + } + norm +} + +// ---------------------------------------------------------------- hist + +fn t_hist(rng: &mut Rng, st: &mut Stats) { + let len = sizes(rng); + let mut src = data(rng, len); + if rng.chance(1, 2) { let m = rng.range(1, 256); for b in src.iter_mut() { *b = (*b as usize % m) as u8; } } + let actual = src.iter().copied().max().unwrap_or(0) as u32; + let mut padded = src.clone(); padded.extend_from_slice(&[0xEE; 32]); + let p: *const c_void = padded.as_ptr().cast(); + let garbage: Vec = (0..256).map(|_| rng.next() as u32 >> 8).collect(); + let mut w = wksp(); + + // Checked. + for _ in 0..3 { + let max = match rng.below(4) { 0 => rng.below(256) as u32, 1 => actual, 2 => actual.saturating_sub(1), _ => *rng.pick(&[255u32, 256, 1000]) }; + let (mut cc, mut cr, mut mc, mut mr) = (garbage.clone(), garbage.clone(), max, max); + let a = res(unsafe { HIST_count_wksp(cc.as_mut_ptr(), &mut mc, p, len, w.as_mut_ptr().cast(), 4096) }); + let b = conv(rs::hist_count_wksp(&mut cr, &mut mr, &src)); + assert_eq!((a, mc, &cc), (b, mr, &cr), "HIST_count_wksp len {len} max {max}"); + st.hit(); + } + // Trusted. + let max = rng.range(actual as usize, 255) as u32; + let (mut cc, mut cr, mut mc, mut mr) = (garbage.clone(), garbage.clone(), max, max); + let a = res(unsafe { HIST_countFast_wksp(cc.as_mut_ptr(), &mut mc, p, len, w.as_mut_ptr().cast(), 4096) }); + let b = conv(rs::hist_count_fast_wksp(&mut cr, &mut mr, &src)); + assert_eq!((a, mc, &cc), (b, mr, &cr), "HIST_countFast_wksp len {len} max {max}"); + st.hit(); + + let (mut cc, mut cr, mut mc, mut mr) = (garbage.clone(), garbage.clone(), max, max); + let a = unsafe { HIST_count_simple(cc.as_mut_ptr(), &mut mc, p, len) }; + let b = rs::hist_count_simple(&mut cr, &mut mr, &src).unwrap(); + assert_eq!((a, mc, &cc), (b, mr, &cr), "HIST_count_simple len {len} max {max}"); + st.hit(); + + let (mut cc, mut cr) = (garbage.clone(), garbage.clone()); + unsafe { HIST_add(cc.as_mut_ptr(), p, len) }; + rs::hist_add(&mut cr, &src).unwrap(); + assert_eq!(cc, cr, "HIST_add"); + st.hit(); +} + +// ---------------------------------------------------------------- fse + +fn t_fse_logs(rng: &mut Rng, st: &mut Stats) { + for _ in 0..50 { + let max_log = rng.below(15) as u32; + let size = match rng.below(3) { 0 => rng.range(2, 40), 1 => rng.range(2, 5000), _ => 2 + rng.size(1 << 20) }; + let max = rng.range(1, 255) as u32; + let minus = rng.range(1, 2) as u32; + assert_eq!(unsafe { FSE_optimalTableLog(max_log, size, max) }, rs::fse_optimal_table_log(max_log, size, max), "FSE_optimalTableLog {max_log} {size} {max}"); + assert_eq!(unsafe { FSE_optimalTableLog_internal(max_log, size, max, minus) }, rs::fse_optimal_table_log_internal(max_log, size, max, minus), "FSE_optimalTableLog_internal {max_log} {size} {max} {minus}"); + let max0 = rng.below(256) as u32; + assert_eq!(unsafe { FSE_NCountWriteBound(max0, max_log) }, rs::fse_ncount_write_bound(max0, max_log)); + assert_eq!(unsafe { HUF_compressBound(size) }, rs::huf_compress_bound(size)); + assert_eq!(unsafe { HUF_minTableLog(max + 1) }, rs::huf_min_table_log(max + 1)); + st.cases += 5; + } +} + +/// `FSE_CTable` of the C code, with garbage that it is not to touch... which is zero, as for the other side. +fn c_fse_table() -> Vec { vec![0u32; 1 + (1 << 11) + 256 * 2 + 16] } + +fn write_ncount_both(rng: &mut Rng, st: &mut Stats, norm: &[i16], max: u32, table_log: u32) -> R { + let bound = unsafe { FSE_NCountWriteBound(max, table_log) }; + let mut full = Err(0); + for k in 0..4 { + let cap = match k { 0 => 512 + rng.below(8), 1 => bound, 2 => bound.saturating_sub(1 + rng.below(4)), _ => rng.below(bound + 2) }; + let mut out = Out::new(cap); + let a = res(unsafe { FSE_writeNCount(out.c_ptr(), cap, norm.as_ptr(), max, table_log) }); + let b = conv(rs::fse_write_ncount(&mut out.r, norm, max, table_log)); + assert_eq!(a, b, "FSE_writeNCount cap {cap} max {max} log {table_log} {norm:?}"); + out.check("FSE_writeNCount", &(cap, max, table_log, norm)); + st.hit(); + if k == 0 { full = a; } + } + full +} + +/// Builds the table on both sides and uses it. `norm` is valid, `table_log <= 9`, `max <= 52`. +fn use_fse_table(rng: &mut Rng, st: &mut Stats, norm: &[i16], max: u32, table_log: u32) -> (Vec, rs::Fse) { + let mut w = wksp(); + let mut ct = c_fse_table(); + let mut rt = rs::Fse::new(); + let a = res(unsafe { FSE_buildCTable_wksp(ct.as_mut_ptr(), norm.as_ptr(), max, table_log, w.as_mut_ptr().cast(), WKSP_SIZE) }); + let b = conv(rt.build(norm, max, table_log)); + assert_eq!(a, b.map(|_| 0), "FSE_buildCTable_wksp"); + let conv_t = rt.to_c(); + assert_eq!(&ct[..conv_t.len()], &conv_t[..], "FSE_buildCTable_wksp table: max {max} log {table_log} {norm:?}"); + st.hit(); + + for s in 0..=max { + assert_eq!(unsafe { shim_fse_getMaxNbBits(ct.as_ptr(), s) }, rt.get_max_nb_bits(s), "FSE_getMaxNbBits"); + if norm[s as usize] != 0 { + assert_eq!(unsafe { shim_fse_bitCost(ct.as_ptr(), s, 8) }, rt.bit_cost(s, 8), "FSE_bitCost"); + } + } + st.hit(); + + for _ in 0..3 { + let len = match rng.below(4) { 0 => rng.below(12), 1 => rng.below(300), 2 => rng.below(300), _ => rng.below(3000) }; + let src = data_among(rng, norm, len); + let bound = len + (len >> 7) + 4 + 8; + let cap = match rng.below(5) { 0 => bound, 1 => bound - 1, 2 => bound + rng.below(50), 3 => rng.below(bound), _ => rng.below(20) }; + let mut out = Out::new(cap); + let a = unsafe { FSE_compress_usingCTable(out.c_ptr(), cap, src.as_ptr().cast(), len, ct.as_ptr()) }; + let b = rt.compress(&mut out.r, &src); + // `FSE_BLOCKBOUND` is no bound for rare symbols with a large table log, and the C code trusts it. + if cap >= bound && out.c[cap..].iter().any(|&b| b != FILL) { assert!(table_log > 7); assert_eq!(b, 0); continue; } + assert_eq!(a, b, "FSE_compress_usingCTable len {len} cap {cap}"); + out.check("FSE_compress_usingCTable", &(len, cap, table_log)); + st.hit(); + } + (ct, rt) +} + +fn t_fse(rng: &mut Rng, st: &mut Stats) { + // The alphabets of sequences and of Huffman weights, or any. + let max = match rng.below(6) { 0 => 12, 1 => 35, 2 => 52, 3 => 31, 4 => rng.range(1, 52), _ => rng.range(1, 255) }; + let budget = *rng.pick(&[8usize, 40, 255, 1000, 20000, 131072, 1 << 22]); + let mut count = counts(rng, max, budget); + // As the callers do, `max` is the last symbol present, most of the time. + let max = if rng.chance(4, 5) { count.iter().rposition(|&c| c != 0).unwrap().max(1) } else { max } as u32; + let mut total: usize = count.iter().map(|&c| c as usize).sum(); + if total < 2 { count[0] += 2; total += 2; } + let low_prob = rng.chance(1, 2); + let table_log = match rng.below(4) { + 0 => rng.below(15) as u32, + _ => { let max_log = *rng.pick(&[5u32, 6, 8, 9, 9, 11, 12, 0]); unsafe { FSE_optimalTableLog(max_log, total, max) } } + }; + let mut nc = vec![0x5A5Ai16; 256]; + let mut nr = nc.clone(); + let a = res(unsafe { FSE_normalizeCount(nc.as_mut_ptr(), table_log, count.as_ptr(), total, max, low_prob as u32) }); + let b = conv(rs::fse_normalize_count(&mut nr, table_log, &count, total, max, low_prob)); + assert_eq!((a, &nc), (b, &nr), "FSE_normalizeCount log {table_log} total {total} max {max} low {low_prob} {:?}", &count[..=max as usize]); + st.hit(); + let Ok(log) = a else { return }; + if log == 0 { return; } + let log = log as u32; + let norm = &nc[..=max as usize]; + assert_eq!(norm.iter().map(|n| n.unsigned_abs() as usize).sum::(), 1 << log); + write_ncount_both(rng, st, norm, max, log).unwrap(); + if log <= 9 && max <= 52 { use_fse_table(rng, st, norm, max, log); } +} + +fn t_fse_tables(rng: &mut Rng, st: &mut Stats) { + let mut made = Vec::new(); + for _ in 0..3 { + let table_log = rng.range(5, 9) as u32; + let max = match rng.below(4) { 0 => 52, 1 => 35, 2 => 31, _ => rng.range(1, 52) }; + let norm = random_norm(rng, max, table_log); + assert_eq!(norm.iter().map(|n| n.unsigned_abs() as usize).sum::(), 1 << table_log, "{norm:?}"); + if norm.iter().filter(|&&n| n != 0).count() > 1 { write_ncount_both(rng, st, &norm, max as u32, table_log).unwrap(); } + let (ct, rt) = use_fse_table(rng, st, &norm, max as u32, table_log); + made.push((ct, rt, norm)); + } + // A run-length table in place of one of them, built over what is there. + if rng.chance(1, 3) { + let i = rng.below(3); + let symbol = rng.below(53) as u8; + let (ct, rt, norm) = &mut made[i]; + assert_eq!(unsafe { FSE_buildCTable_rle(ct.as_mut_ptr(), symbol) }, 0); + rt.build_rle(symbol).unwrap(); + let t = rt.to_c(); + // Only the header, the two states and the transform of the symbol are defined. + assert_eq!(ct[..2], t[..2], "FSE_buildCTable_rle"); + assert_eq!(ct[2 + 2 * symbol as usize..][..2], t[2 + 2 * symbol as usize..][..2], "FSE_buildCTable_rle"); + *norm = vec![0; 53]; norm[symbol as usize] = 1; + st.hit(); + let len = rng.below(40); + let src = vec![symbol; len]; + let cap = rng.below(60); + let mut out = Out::new(cap); + let a = unsafe { FSE_compress_usingCTable(out.c_ptr(), cap, src.as_ptr().cast(), len, ct.as_ptr()) }; + let b = rt.compress(&mut out.r, &src); + assert_eq!(a, b, "FSE_compress_usingCTable rle"); + out.check("FSE_compress_usingCTable rle", &(len, cap)); + st.hit(); + } + let n = 1 + rng.size(2000); + let syms: Vec> = made.iter().map(|(_, _, norm)| data_among(rng, norm, n)).collect(); + let bound = n * 4 + 32; + let cap = match rng.below(4) { 0 => rng.below(20), 1 => rng.below(bound), _ => bound }; + let mut out = Out::new(cap); + let a = res(unsafe { shim_fse_encode3(out.c_ptr(), cap, made[0].0.as_ptr(), made[1].0.as_ptr(), made[2].0.as_ptr(), syms[0].as_ptr(), syms[1].as_ptr(), syms[2].as_ptr(), n) }); + let b = conv(rs::fse_encode3(&mut out.r, [&made[0].1, &made[1].1, &made[2].1], [&syms[0], &syms[1], &syms[2]])); + assert_eq!(a, b, "encode3 n {n} cap {cap}"); + out.check("encode3", &(n, cap)); + st.hit(); + + // Too large for the table. + let norm = random_norm(rng, 52, 10); + let mut ct = c_fse_table(); + let mut w = wksp(); + // The workspace that `ZSTD_buildCTable` has. + let a = res(unsafe { FSE_buildCTable_wksp(ct.as_mut_ptr(), norm.as_ptr(), 52, 10, w.as_mut_ptr().cast(), 1140) }); + assert_eq!(a, conv(rs::Fse::new().build(&norm, 52, 10)).map(|_| 0)); + st.hit(); +} + +/// Counts that are not normalized. +fn t_fse_bad_norm(rng: &mut Rng, st: &mut Stats) { + let table_log = rng.below(15) as u32; + let max = rng.range(1, 255); + let mut norm = if (5..=12).contains(&table_log) { random_norm(rng, max, table_log) } else { vec![1; max + 1] }; + for _ in 0..rng.below(4) { + let i = rng.below(norm.len()); + norm[i] = match rng.below(4) { 0 => 0, 1 => -1, 2 => norm[i].wrapping_add(rng.range(0, 6) as i16 - 3), _ => rng.below(1 << table_log.min(12)) as i16 }; + } + // A large buffer: past the bound, the C code does not check what it writes. + let cap = 1024; + let mut out = Out::new(cap); + let a = res(unsafe { FSE_writeNCount(out.c_ptr(), cap, norm.as_ptr(), max as u32, table_log) }); + let b = conv(rs::fse_write_ncount(&mut out.r, &norm, max as u32, table_log)); + assert_eq!(a, b, "FSE_writeNCount bad: log {table_log} {norm:?}"); + out.check("FSE_writeNCount bad", &(table_log, &norm)); + st.hit(); +} + +// ---------------------------------------------------------------- huf + +fn bmi2(rng: &mut Rng) -> c_int { (std::arch::is_x86_feature_detected!("bmi2") && rng.chance(1, 2)) as c_int } + +fn c_build(count: &[u32], max: u32, bits: u32, init: &rs::Huf) -> (R, rs::Huf) { + let mut t = *init; + let mut w = wksp(); + let r = res(unsafe { HUF_buildCTable_wksp(t.as_mut_ptr(), count.as_ptr(), max, bits, w.as_mut_ptr().cast(), WKSP_SIZE) }); + (r, t) +} + +fn using_ctable_both(rng: &mut Rng, st: &mut Stats, t: &rs::Huf, src: &[u8]) { + let log = t[0] as u8 as usize; + let tight = ((src.len() * log) >> 3) + 8; + for four in [false, true] { + for _ in 0..2 { + let cap = match rng.below(8) { + 0 => tight, 1 => tight.saturating_sub(1), 2 => tight + 1 + rng.below(30), 3 => rng.below(tight + 1), 4 => rng.below(24), + 5 => tight / 4 + rng.below(40), _ => unsafe { HUF_compressBound(src.len()) }, + }; + let mut out = Out::new(cap); + let f = bmi2(rng); + let (a, b) = if four { + (res(unsafe { HUF_compress4X_usingCTable(out.c_ptr(), cap, src.as_ptr().cast(), src.len(), t.as_ptr(), f) }), conv(rs::huf_compress_4x_using_ctable(&mut out.r, src, t))) + } else { + (res(unsafe { HUF_compress1X_usingCTable(out.c_ptr(), cap, src.as_ptr().cast(), src.len(), t.as_ptr(), f) }), conv(rs::huf_compress_1x_using_ctable(&mut out.r, src, t))) + }; + assert_eq!(a, b, "HUF_compress{}X_usingCTable len {} cap {cap} log {log}", if four { 4 } else { 1 }, src.len()); + out.check("HUF_compressNX_usingCTable", &(four, src.len(), cap, log)); + st.hit(); + } + } +} + +fn t_huf_table(rng: &mut Rng, st: &mut Stats) { + let max = match rng.below(4) { 0 => 255, 1 => rng.range(0, 255), 2 => rng.range(0, 40), _ => rng.range(100, 255) }; + let budget = *rng.pick(&[10usize, 100, 1000, 5000, 20000, 131072, 131072]); + let count = counts(rng, max, budget); + let max = if rng.chance(3, 4) { count.iter().rposition(|&c| c != 0).unwrap() } else { max } as u32; + let total: usize = count.iter().map(|&c| c as usize).sum(); + + let card = unsafe { HUF_cardinality(count.as_ptr(), max) }; + assert_eq!(card, rs::huf_cardinality(&count, max), "HUF_cardinality"); + let min_log = unsafe { HUF_minTableLog(card) }; + assert_eq!(min_log, rs::huf_min_table_log(card)); + st.hit(); + + let bits = match rng.below(6) { 0 => 0, 1 => 11, 2 => min_log, _ => rng.range(min_log as usize, 12) as u32 }; + let mut init = [0u64; 257]; + if rng.chance(1, 2) { for x in init.iter_mut() { *x = rng.next(); } } + let (a, ct) = c_build(&count, max, bits, &init); + let mut rt = init; + let b = conv(rs::huf_build_ctable(&mut rt, &count, max, bits)); + assert_eq!(a, b, "HUF_buildCTable_wksp max {max} bits {bits} {:?}", &count[..=max as usize]); + assert!(ct == rt, "HUF_buildCTable_wksp table: max {max} bits {bits} {:?}", &count[..=max as usize]); + st.hit(); + let log = a.unwrap() as u32; + + // Out of range. + if rng.chance(1, 8) { + let (a, t) = c_build(&count, max, 13, &init); + let mut tr = init; + assert_eq!(a, conv(rs::huf_build_ctable(&mut tr, &count, max, 13)), "HUF_buildCTable_wksp 13 bits"); + assert!(t == tr); + let big = 256 + rng.below(3) as u32; + let (a, t) = c_build(&count, big, bits, &init); + let mut tr = init; + assert_eq!(a, conv(rs::huf_build_ctable(&mut tr, &count, big, bits)), "HUF_buildCTable_wksp max {big}"); + assert!(t == tr); + let mut out = Out::new(300); + let mut w = wksp(); + let a = res(unsafe { HUF_writeCTable_wksp(out.c_ptr(), 300, ct.as_ptr(), big, log, w.as_mut_ptr().cast(), WKSP_SIZE) }); + assert_eq!(a, conv(rs::huf_write_ctable(&mut out.r, &rt, big, log)), "HUF_writeCTable_wksp max {big}"); + out.check("HUF_writeCTable_wksp", &big); + st.cases += 3; + } + + let h = unsafe { HUF_readCTableHeader(ct.as_ptr()) }; + assert_eq!((h.table_log, h.max_symbol_value), rs::huf_read_ctable_header(&rt)); + for s in 0..256 { assert_eq!(unsafe { HUF_getNbBitsFromCTable(ct.as_ptr(), s) }, rs::huf_get_nb_bits(&rt, s)); } + assert_eq!(unsafe { HUF_estimateCompressedSize(ct.as_ptr(), count.as_ptr(), max) }, rs::huf_estimate_compressed_size(&rt, &count, max)); + st.cases += 3; + for _ in 0..3 { + let max2 = rng.below(256); + let other = if rng.chance(1, 2) { counts(rng, max2, 1000) } else { let mut o = count.clone(); let i = rng.below(256); o[i] = rng.below(2) as u32; o }; + let max2 = if rng.chance(1, 2) { max2 as u32 } else { max }; + assert_eq!(unsafe { HUF_validateCTable(ct.as_ptr(), other.as_ptr(), max2) } != 0, rs::huf_validate_ctable(&rt, &other, max2), "HUF_validateCTable"); + assert_eq!(unsafe { HUF_estimateCompressedSize(ct.as_ptr(), other.as_ptr(), max2) }, rs::huf_estimate_compressed_size(&rt, &other, max2)); + st.cases += 2; + } + + // The header. + let mut header = Vec::new(); + for k in 0..4 { + let cap = match k { 0 => 300, 1 => rng.below(140), 2 => rng.below(20), _ => 129 + rng.below(3) }; + let mut out = Out::new(cap); + let mut w = wksp(); + let a = res(unsafe { HUF_writeCTable_wksp(out.c_ptr(), cap, ct.as_ptr(), max, log, w.as_mut_ptr().cast(), WKSP_SIZE) }); + let b = conv(rs::huf_write_ctable(&mut out.r, &rt, max, log)); + assert_eq!(a, b, "HUF_writeCTable_wksp cap {cap} max {max} log {log}"); + out.check("HUF_writeCTable_wksp", &(cap, max, log)); + st.hit(); + if let (0, Ok(n)) = (k, a) { header = out.r[..n].to_vec(); } + } + if !header.is_empty() { + header.extend((0..rng.below(8)).map(|_| rng.next() as u8)); + let any = rng.below(256) as u32; + let limit = *rng.pick(&[255, max, max.saturating_sub(1), any]); + read_ctable_both(rng, st, &header, limit); + let n = rng.below(header.len() + 1); + read_ctable_both(rng, st, &header[..n], 255); + let i = rng.below(header.len()); + header[i] = rng.next() as u8; + read_ctable_both(rng, st, &header, 255); + } + + // The best limit on the depth. + if total > 1 && card > 1 { + for flags in [0, 2] { + let max_log = if flags == 0 { rng.below(13) as u32 } else { rng.range(min_log as usize, 12) as u32 }; + let (mut tc, mut tr) = (init, init); + let mut w = wksp(); + // The sizes that the two callers give. + let wsize = *rng.pick(&[4864usize, 7896]); + let a = unsafe { HUF_optimalTableLog(max_log, total, max, w.as_mut_ptr().cast(), wsize, tc.as_mut_ptr(), count.as_ptr(), flags) }; + let b = rs::huf_optimal_table_log(max_log, total, max, &mut tr, &count, flags as u32); + assert_eq!(a, b, "HUF_optimalTableLog flags {flags} max_log {max_log} total {total} max {max} {:?}", &count[..=max as usize]); + assert!(tc == tr, "HUF_optimalTableLog table"); + st.hit(); + } + } + + // Encoding. + let src = data_of(rng, &count); + let n = if rng.chance(1, 2) { src.len() } else { rng.below(src.len() + 1) }; + using_ctable_both(rng, st, &ct, &src[..n]); +} + +fn read_ctable_both(rng: &mut Rng, st: &mut Stats, src: &[u8], max: u32) { + let mut init = [0u64; 257]; + if rng.chance(1, 2) { for x in init.iter_mut() { *x = rng.next(); } } + let (mut tc, mut tr) = (init, init); + let (mut mc, mut zc) = (max, 7u32); + // The C code may look at bytes after the end, without using them. + let mut padded = src.to_vec(); padded.extend_from_slice(&[0; 16]); + let a = res(unsafe { HUF_readCTable(tc.as_mut_ptr(), &mut mc, padded.as_ptr().cast(), src.len(), &mut zc) }); + let b = conv(rs::huf_read_ctable(&mut tr, max, src)); + match (a, b) { + (Ok(n), Ok((size, m, z))) => assert_eq!((n, mc, zc != 0), (size, m, z), "HUF_readCTable"), + (a, b) => assert_eq!(a.err(), b.err(), "HUF_readCTable error: {src:?}"), + } + assert!(tc == tr, "HUF_readCTable table: {src:?}"); + st.hit(); +} + +fn t_huf_read(rng: &mut Rng, st: &mut Stats) { + let len = rng.below(140); + let mut src: Vec = (0..len).map(|_| rng.next() as u8).collect(); + if len > 0 { + match rng.below(3) { + 0 => src[0] = 128 + rng.below(128) as u8, + 1 => { src[0] = 128 + rng.below(128) as u8; for b in src[1..].iter_mut() { *b = (rng.below(4) << 4 | rng.below(4)) as u8; } } + _ => {} + } + } + let limit = *rng.pick(&[255, 255, 100, 10]); + read_ctable_both(rng, st, &src, limit); +} + +fn literals(rng: &mut Rng, len: usize) -> Vec { + match rng.below(6) { + 0 | 1 => data(rng, len), + 2 => { let max = rng.range(1, 255); let c = counts(rng, max, len.max(1)); let mut v = data_of(rng, &c); v.truncate(len); v } + 3 => { // nearly incompressible + let m = rng.range(100, 256); (0..len).map(|_| rng.below(m) as u8).collect() + } + 4 => { // compressible in the middle only + let mut v: Vec = (0..len).map(|_| rng.next() as u8).collect(); + if len > 8192 { for b in v[4096..len - 4096].iter_mut() { *b &= 7; } } + v + } + _ => { let k = rng.range(1, 8); (0..len).map(|_| { let mut b = rng.next() as u8; for _ in 0..k { b &= rng.next() as u8; } b }).collect() } + } +} + +fn t_huf_repeat(rng: &mut Rng, st: &mut Stats) { + let len = sizes(rng) + if rng.chance(1, 200) { 131073 } else { 0 }; + let src = literals(rng, len); + let len = src.len(); + let mut hist = vec![0u32; 256]; + for &b in &src { hist[b as usize] += 1; } + let actual = src.iter().copied().max().unwrap_or(0) as u32; + let card = hist.iter().filter(|&&c| c != 0).count() as u32; + + let flags = (rng.below(8) << 1) as u32; + let four = rng.chance(1, 2); + + // The previous table. + let zero = [0u64; 257]; + let (prev, mut repeat) = match rng.below(7) { + 0 => (zero, 0u32), + 1 if len > 0 => (c_build(&hist, actual, 11, &zero).1, rng.range(0, 2) as u32), // of this very input + 2 => { // has every symbol + let mut c = hist.clone(); for x in c.iter_mut() { *x = *x / rng.range(1, 3) as u32 + 1; } + (c_build(&c, 255, *rng.pick(&[11, 12, 9]), &zero).1, rng.range(0, 2) as u32) + } + 3 => { let c: Vec = (0..256).map(|_| 1 + rng.size(300) as u32).collect(); (c_build(&c, 255, 11, &zero).1, rng.range(0, 2) as u32) } + 4 => { // of other data: may lack symbols, so that it has to be checked + let m = rng.range(1, 255); let c = counts(rng, m, 5000); + let last = c.iter().rposition(|&x| x != 0).unwrap() as u32; + (c_build(&c, last, 11, &zero).1, rng.range(0, 1) as u32) + } + 5 if len > 0 => { // of a part of this input + let mut c = vec![0u32; 256]; for &b in &src[..1 + rng.below(len)] { c[b as usize] += 1; } + let last = c.iter().rposition(|&x| x != 0).unwrap() as u32; + (c_build(&c, last, 11, &zero).1, rng.range(0, 1) as u32) + } + _ => (zero, 0), + }; + let sampled = flags & 8 != 0 && len >= 40960; + let max = match rng.below(6) { + 0 => 0, + 1 => rng.range(actual as usize, 255) as u32, + 2 if !sampled => rng.below(256) as u32, + 3 => *rng.pick(&[256u32, 300]), + _ => 255, + }; + let min_log = if card > 0 { unsafe { HUF_minTableLog(card) } } else { 1 }; + let log = match rng.below(5) { + 0 => 0, + 1 => 11, + 2 => 13, + _ if flags & 2 != 0 => rng.range(min_log as usize, 12) as u32, + _ => rng.range(1, 12) as u32, + }; + let cap = match rng.below(8) { + 0 => rng.below(24), + 1 => rng.below(len + 1), + 2 => len, + 3 => len / 2 + rng.below(200), + _ => unsafe { HUF_compressBound(len) }, + }; + + let mut out = Out::new(cap); + let (mut tc, mut tr) = (prev, prev); + let (mut rc, mut rr) = (repeat, repeat); + let mut w = wksp(); + let cf = flags as c_int | bmi2(rng); + let f = if four { HUF_compress4X_repeat } else { HUF_compress1X_repeat }; + let a = res(unsafe { f(out.c_ptr(), cap, src.as_ptr().cast(), len, max, log, w.as_mut_ptr().cast(), WKSP_SIZE, tc.as_mut_ptr(), &mut rc, cf) }); + let b = conv(rs::huf_compress_repeat(four, &mut out.r, &src, max, log, &mut tr, &mut rr, flags)); + let ctx = (four, len, cap, max, log, flags, repeat); + assert_eq!((a, rc), (b, rr), "HUF_compressNX_repeat {ctx:?}"); + out.check("HUF_compressNX_repeat", &ctx); + assert!(tc == tr, "HUF_compressNX_repeat table {ctx:?}"); + st.hit(); + + // The next block, with the table that this one left, as the literals of a frame. + if let Ok(n) = a && n > 1 { + repeat = if rc == 0 { 1 } else { rc }; + let len2 = sizes(rng); + let mut src2 = if rng.chance(1, 2) { literals(rng, len2) } else { let mut v = src.clone(); v.truncate(len2); v.reverse(); v }; + if rng.chance(1, 4) && !src2.is_empty() { let i = rng.below(src2.len()); src2[i] = rng.next() as u8; } + let cap = unsafe { HUF_compressBound(src2.len()) }; + let mut out = Out::new(cap); + let (mut rc, mut rr) = (repeat, repeat); + let mut w = wksp(); + let a = res(unsafe { f(out.c_ptr(), cap, src2.as_ptr().cast(), src2.len(), 255, 11, w.as_mut_ptr().cast(), WKSP_SIZE, tc.as_mut_ptr(), &mut rc, cf) }); + let b = conv(rs::huf_compress_repeat(four, &mut out.r, &src2, 255, 11, &mut tr, &mut rr, flags)); + assert_eq!((a, rc), (b, rr), "HUF_compressNX_repeat second {ctx:?}"); + out.check("HUF_compressNX_repeat second", &ctx); + assert!(tc == tr, "HUF_compressNX_repeat second table {ctx:?}"); + st.hit(); + } +} + +/// `which`: `all`, or one of the names below. +pub fn run(seed: u64, iters: u64, which: &str) { + assert_eq!(rs::HUF_CONSTANTS, (8704, 131072, 11, 8)); + assert_eq!(rs::FSE_NCOUNTBOUND, 512); + let tests: &[(&str, fn(&mut Rng, &mut Stats))] = &[ + ("hist", t_hist), ("fse_logs", t_fse_logs), ("fse", t_fse), ("fse_tables", t_fse_tables), ("fse_bad_norm", t_fse_bad_norm), + ("huf_table", t_huf_table), ("huf_read", t_huf_read), ("huf_repeat", t_huf_repeat), + ]; + let mut totals = vec![0u64; tests.len()]; + for i in 0..iters { + for (k, (name, test)) in tests.iter().enumerate() { + if which != "all" && which != *name { continue; } + let s = seed.wrapping_mul(1_000_003).wrapping_add(i).wrapping_mul(31).wrapping_add(k as u64); + if std::env::var_os("ENTROPY_TRACE").is_some() { eprintln!("{name} seed {s}"); } + let mut st = Stats { cases: 0 }; + test(&mut Rng(s), &mut st); + totals[k] += st.cases; + } + } + for ((name, _), n) in tests.iter().zip(&totals) { if *n > 0 { println!("{name}: {n} comparisons"); } } + println!("entropy ok: seed {seed}, {iters} iterations, {} comparisons", totals.iter().sum::()); +} + +// ---------------------------------------------------------------- speed + +fn time(name: &str, bytes: usize, mut c: impl FnMut() -> usize, mut r: impl FnMut() -> usize) { + let mut best = [f64::MAX; 2]; + let reps = (200_000_000 / bytes.max(1)).clamp(20, 200_000) / 20; + for _ in 0..7 { + let t = std::time::Instant::now(); + for _ in 0..reps { std::hint::black_box(c()); } + best[0] = best[0].min(t.elapsed().as_secs_f64() / reps as f64); + let t = std::time::Instant::now(); + for _ in 0..reps { std::hint::black_box(r()); } + best[1] = best[1].min(t.elapsed().as_secs_f64() / reps as f64); + } + println!("{name:40} C {:10.0} ns Rust {:10.0} ns ratio {:.2}", best[0] * 1e9, best[1] * 1e9, best[1] / best[0]); +} + +/// Build with `--profile bench`. +pub fn bench() { + let mut rng = Rng(7); + let text: Vec = { let mut v = Vec::new(); while v.len() < 131072 { let n = 1 + rng.size(5000); let mut r2 = Rng(rng.next() | 2); let mut part = Vec::new(); while part.len() < n { part.extend_from_slice(format!("const x{} = await foo({});\n", r2.below(50), r2.below(1000)).as_bytes()); } v.extend_from_slice(&part); } v.truncate(131072); v }; + let skew = |k: usize, rng: &mut Rng| -> Vec { (0..131072).map(|_| { let mut b = rng.next() as u8; for _ in 0..k { b &= rng.next() as u8; } b }).collect() }; + let inputs: Vec<(&str, Vec)> = vec![("text", text), ("skew1", skew(1, &mut rng)), ("skew3", skew(3, &mut rng)), ("random", skew(0, &mut rng))]; + let mut w = wksp(); + let wp: *mut c_void = w.as_mut_ptr().cast(); + for (name, src) in &inputs { + for len in [131072usize, 4000, 300] { + let src = &src[..len]; + let p: *const c_void = src.as_ptr().cast(); + let (mut cc, mut cr) = (vec![0u32; 256], vec![0u32; 256]); + time(&format!("HIST_count_wksp {name} {len}"), len, + || { let mut m = 255; unsafe { HIST_count_wksp(cc.as_mut_ptr(), &mut m, p, len, wp, 4096) } }, + || { let mut m = 255; rs::hist_count_wksp(&mut cr, &mut m, src).unwrap() }); + let max = cc.iter().rposition(|&c| c != 0).unwrap() as u32; + let mut t = [0u64; 257]; + let mut bt = bun_zstd::testing::entropy_bench::Table::new(&t); + time(&format!("HUF_buildCTable_wksp {name} {len}"), 2000, + || unsafe { HUF_buildCTable_wksp(t.as_mut_ptr(), cc.as_ptr(), max, 11, wp, WKSP_SIZE) }, + || bt.build(&cr, max, 11)); + let log = t[0] as u8 as u32; + let cap = unsafe { HUF_compressBound(len) }; + let mut dst = vec![0u8; cap]; + let mut dst2 = vec![0u8; cap]; + time(&format!("HUF_writeCTable_wksp {name} {len}"), 2000, + || unsafe { HUF_writeCTable_wksp(dst.as_mut_ptr().cast(), cap, t.as_ptr(), max, log, wp, WKSP_SIZE) }, + || bt.write(&mut dst2, max, log)); + for bits in [11u32, 10, 9, 8] { + let mut t = [0u64; 257]; + if res(unsafe { HUF_buildCTable_wksp(t.as_mut_ptr(), cc.as_ptr(), max, bits, wp, WKSP_SIZE) }).is_err() { continue; } + if (t[0] as u8 as u32) != bits { continue; } + let bt = bun_zstd::testing::entropy_bench::Table::new(&t); + time(&format!("HUF_compress4X_usingCTable {name} {len} log {bits}"), len, + || unsafe { HUF_compress4X_usingCTable(dst.as_mut_ptr().cast(), cap, p, len, t.as_ptr(), 0) }, + || bt.compress4(&mut dst2, src)); + } + for flags in [0u32, 2] { + time(&format!("HUF_compress4X_repeat {name} {len} flags {flags}"), len, + || { let mut rep = 0; unsafe { HUF_compress4X_repeat(dst.as_mut_ptr().cast(), cap, p, len, 255, 11, wp, WKSP_SIZE, t.as_mut_ptr(), &mut rep, flags as c_int) } }, + || bt.repeat4(&mut dst2, src, flags == 2)); + } + } + } + // Sequences. + for (max, log, n) in [(35u32, 9u32, 5000usize), (52, 9, 5000), (31, 8, 5000), (35, 6, 60)] { + let count = { let mut c = vec![0u32; 256]; for _ in 0..n { let a = rng.below(max as usize + 1); let b = rng.below(max as usize + 1); c[a.min(b)] += 1; } c }; + let max = count.iter().rposition(|&c| c != 0).unwrap() as u32; + let log = unsafe { FSE_optimalTableLog(log, n, max) }; + let (mut nc, mut nr) = (vec![0i16; 256], vec![0i16; 256]); + time(&format!("FSE_normalizeCount {max} {log}"), 500, + || unsafe { FSE_normalizeCount(nc.as_mut_ptr(), log, count.as_ptr(), n, max, 1) }, + || rs::fse_normalize_count(&mut nr, log, &count, n, max, true).unwrap()); + let mut buf = [0u8; 512]; + let mut buf2 = [0u8; 512]; + time(&format!("FSE_writeNCount {max} {log}"), 500, + || unsafe { FSE_writeNCount(buf.as_mut_ptr().cast(), 512, nc.as_ptr(), max, log) }, + || rs::fse_write_ncount(&mut buf2, &nr, max, log).unwrap()); + let mut ct = c_fse_table(); + let mut rt = rs::Fse::new(); + time(&format!("FSE_buildCTable_wksp {max} {log}"), 500, + || unsafe { FSE_buildCTable_wksp(ct.as_mut_ptr(), nc.as_ptr(), max, log, wp, WKSP_SIZE) }, + || { rt.build(&nr, max, log).unwrap(); 0 }); + let syms = data_among(&mut rng, &nc[..=max as usize], n); + let cap = n * 4 + 64; + let (mut d1, mut d2) = (vec![0u8; cap], vec![0u8; cap]); + time(&format!("encode3 {max} {log} n {n}"), n * 3, + || unsafe { shim_fse_encode3(d1.as_mut_ptr().cast(), cap, ct.as_ptr(), ct.as_ptr(), ct.as_ptr(), syms.as_ptr(), syms.as_ptr(), syms.as_ptr(), n) }, + || rs::fse_encode3(&mut d2, [&rt, &rt, &rt], [&syms, &syms, &syms]).unwrap()); + } +} diff --git a/src/zstd/wip/work/entropy/harness/src/main.rs b/src/zstd/wip/work/entropy/harness/src/main.rs new file mode 100644 index 000000000000..e54cc30aa80e --- /dev/null +++ b/src/zstd/wip/work/entropy/harness/src/main.rs @@ -0,0 +1,22 @@ +mod c; +mod data; +mod entropy; +mod seqs; + +fn main() { + let args: Vec = std::env::args().collect(); + let cmd = args.get(1).map(String::as_str).unwrap_or(""); + let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); + let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); + match cmd { + // seqs + "seqs" => { + let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); + seqs::run(seed, iters, &strategies, args.get(5).is_some_and(|s| s == "1")); + } + // entropy [all | the name of a test] + "entropy" => entropy::run(seed, iters, args.get(4).map(String::as_str).unwrap_or("all")), + "entropy-bench" => entropy::bench(), + _ => eprintln!("unknown command"), + } +} diff --git a/src/zstd/wip/work/entropy/harness/src/seqs.rs b/src/zstd/wip/work/entropy/harness/src/seqs.rs new file mode 100644 index 000000000000..7c03d3b0f952 --- /dev/null +++ b/src/zstd/wip/work/entropy/harness/src/seqs.rs @@ -0,0 +1,73 @@ +//! One block through a Rust block compressor and through the C library (`ZSTD_generateSequences`). +use crate::c; +use crate::data::{Rng, data}; +use std::ffi::{c_uint, c_void}; + +#[repr(C)] +#[derive(Clone, Copy, Default, Debug)] +struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } +#[repr(C)] +#[derive(Clone, Copy, Debug)] +struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } +unsafe extern "C" { + fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; + fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; + fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; +} + +fn c_sequences(cp: [u32; 7], use_row: bool, dict: &[u8], src: &[u8]) -> Vec<(u32, u32, u32)> { + let mut cctx = c::CCtx::new(); + for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } + cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder + cctx.set(1010, 2).unwrap(); // no block splitter + if !dict.is_empty() { cctx.ref_prefix(dict).unwrap(); } + let mut out = vec![Sequence::default(); src.len() / 3 + 8]; + let n = c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }).unwrap(); + out[..n].iter().map(|s| (s.lit_length, s.match_length, s.offset)).collect() +} + +/// `strategies`: which of 1..=9 to test. `with_dict`: also test with a prefix, which is the extDict variants. +pub fn run(seed: u64, iters: u64, strategies: &[u32], with_dict: bool) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if i % 500 == 0 { eprintln!("seqs iter {i} (seed {s})"); } + let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071]); + let len = rng.range(7, max); + let mut src = data(&mut rng, len); + let dict = if with_dict && rng.chance(1, 2) { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; + if !dict.is_empty() && rng.chance(2, 3) { + for _ in 0..rng.range(1, 6) { + let n = rng.size(dict.len().min(src.len())); + let at = rng.below(dict.len() - n + 1); + let to = rng.below(src.len() - n + 1); + src[to..to + n].copy_from_slice(&dict[at..at + n]); + } + } + let strategy = *rng.pick(strategies); + let p = if rng.chance(1, 3) { + // the parameters of a level, with the strategy forced + let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; + p.strategy = strategy; + p + } else { + CParams { + window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, + search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, + target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, + } + }; + let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; + let cp = [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy]; + let use_row = rng.chance(1, 2); + let expected = c_sequences(cp, use_row, &dict, &src); + let got = bun_zstd::testing::first_block(cp, use_row, &dict, &src); + let got = got.unwrap_or_else(|e| panic!("seed {s}: error {e:?} (cp {cp:?} row {use_row} dict {} src {})", dict.len(), src.len())); + if got != expected { + let at = got.iter().zip(&expected).position(|(a, b)| a != b).unwrap_or(got.len().min(expected.len())); + panic!("seed {s}: cp {cp:?} row {use_row} dict {} src {}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", + dict.len(), src.len(), got.get(at), expected.get(at), got.len(), expected.len()); + } + } + println!("seqs ok: {iters} iterations from seed {seed}, strategies {strategies:?}, dict {with_dict}"); +} diff --git a/src/zstd/wip/work/entropy/zstd/compress/double_fast.rs b/src/zstd/wip/work/entropy/zstd/compress/double_fast.rs new file mode 100644 index 000000000000..65d401675d1b --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/compress/double_fast.rs @@ -0,0 +1,48 @@ +//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_double_hash_table( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _end: usize, + _dtlm: DictTableLoadMethod, + _tfp: TableFillPurpose, +) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast` +pub(crate) fn compress_block_double_fast( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState` +pub(crate) fn compress_block_double_fast_dict_match_state( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast_extDict` +pub(crate) fn compress_block_double_fast_ext_dict( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/entropy/zstd/compress/fast.rs b/src/zstd/wip/work/entropy/zstd/compress/fast.rs new file mode 100644 index 000000000000..1f4788af28b4 --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/compress/fast.rs @@ -0,0 +1,48 @@ +//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_hash_table( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _end: usize, + _dtlm: DictTableLoadMethod, + _tfp: TableFillPurpose, +) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast` +pub(crate) fn compress_block_fast( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast_dictMatchState` +pub(crate) fn compress_block_fast_dict_match_state( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast_extDict` +pub(crate) fn compress_block_fast_ext_dict( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/entropy/zstd/compress/fse.rs b/src/zstd/wip/work/entropy/zstd/compress/fse.rs new file mode 100644 index 000000000000..7d57ceda36e4 --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/compress/fse.rs @@ -0,0 +1,678 @@ +//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. + +use crate::common::bitstream::BitCStream; +use crate::common::{FSE_MAX_TABLELOG, FSE_MIN_TABLELOG, MAX_SEQ, ML_FSE_LOG, fse_table_step}; +use crate::error::{ErrorCode, Result}; +use crate::util::{BUG, SliceExt, highbit32, write_le16}; + +pub(crate) const FSE_NCOUNTBOUND: usize = 512; +const FSE_DEFAULT_TABLELOG: u32 = 11; + +/// `FSE_BLOCKBOUND` +const fn block_bound(size: usize) -> usize { + size + (size >> 7) + 4 + size_of::() +} + +/// `FSE_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum FseRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `FSE_symbolCompressionTransform` +#[derive(Clone, Copy, Default)] +pub(crate) struct FseSymbolTransform { + pub(crate) delta_find_state: i32, + pub(crate) delta_nb_bits: u32, +} + +/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. +#[derive(Clone)] +pub(crate) struct FseCTable { + pub(crate) table_log: u32, + pub(crate) max_symbol_value: u32, + pub(crate) state_table: [u16; 1 << ML_FSE_LOG], + pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], +} + +impl FseCTable { + pub(crate) const fn new() -> Self { + Self { + table_log: 0, + max_symbol_value: 0, + state_table: [0; 1 << ML_FSE_LOG], + symbol_tt: [FseSymbolTransform { + delta_find_state: 0, + delta_nb_bits: 0, + }; MAX_SEQ as usize + 1], + } + } + + /// `FSE_buildCTable_wksp`: `normalized_counter` adds up to `1 << table_log`, a count of -1 standing for 1. + /// [`ErrorCode::TableLogTooLarge`] if the table has no room for it. + pub(crate) fn build( + &mut self, + normalized_counter: &[i16], + max_symbol_value: u32, + table_log: u32, + ) -> Result<()> { + if table_log > ML_FSE_LOG || max_symbol_value > MAX_SEQ { + return Err(ErrorCode::TableLogTooLarge); + } + let table_size = 1usize << table_log; + let table_mask = table_size - 1; + let step = fse_table_step(table_size); + let counts = normalized_counter.sub(..=max_symbol_value as usize)?; + + const STATE_TABLE_SIZE: usize = 1 << ML_FSE_LOG; + const CUMUL_SIZE: usize = (MAX_SEQ as usize + 1).next_power_of_two(); + let mut cumul = [0u16; CUMUL_SIZE]; + let mut table_symbol = [0u8; STATE_TABLE_SIZE]; + let mut high_threshold = table_size - 1; + + self.table_log = table_log; + self.max_symbol_value = max_symbol_value; + + // Where the states of each symbol start. The symbols of low probability get the highest positions. + let mut total = 0u16; + for (symbol, (&count, start)) in counts.iter().zip(&mut cumul).enumerate() { + *start = total; + if count == -1 { + total = total.wrapping_add(1); + table_symbol.put(high_threshold, symbol as u8)?; + high_threshold = high_threshold.wrapping_sub(1); + } else { + total = total.wrapping_add(count as u16); + } + } + + // Spread the symbols. + if high_threshold == table_size - 1 { + // No symbol of low probability: lay the symbols down in order, and then every position is used. + let mut spread = [0u8; (1 << ML_FSE_LOG) + 8]; + let mut pos = 0usize; + let mut sv = 0u64; + for &count in counts { + let n = usize::try_from(count).map_err(|_| BUG)?; + *spread.array_at_mut(pos)? = sv.to_le_bytes(); + for i in (8..n).step_by(8) { + *spread.array_at_mut(pos + i)? = sv.to_le_bytes(); + } + pos += n; + sv = sv.wrapping_add(0x0101_0101_0101_0101); + } + + let mut position = 0usize; + let (pairs, _) = spread.sub(..table_size)?.as_chunks::<2>(); + for &[first, second] in pairs { + table_symbol.put(position, first)?; + table_symbol.put((position + step) & table_mask, second)?; + position = (position + 2 * step) & table_mask; + } + } else { + let mut position = 0usize; + for (symbol, &freq) in counts.iter().enumerate() { + for _ in 0..freq { + table_symbol.put(position, symbol as u8)?; + position = (position + step) & table_mask; + while position > high_threshold { + position = (position + step) & table_mask; + } + } + } + } + + // The states, sorted by symbol. + for (u, &symbol) in table_symbol.sub(..table_size)?.iter().enumerate() { + let Some(next) = cumul.get_mut(usize::from(symbol) & (CUMUL_SIZE - 1)) else { + continue; + }; + let state = usize::from(*next) & (STATE_TABLE_SIZE - 1); + *next = next.wrapping_add(1); + if let Some(state) = self.state_table.get_mut(state) { + *state = (table_size + u) as u16; + } + } + + let mut total = 0u32; + for (&count, transform) in counts.iter().zip(&mut self.symbol_tt) { + match count { + // For `get_max_nb_bits` and `bit_cost`. + 0 => transform.delta_nb_bits = ((table_log + 1) << 16) - (1 << table_log), + -1 | 1 => { + transform.delta_nb_bits = (table_log << 16).wrapping_sub(1 << table_log); + transform.delta_find_state = total.wrapping_sub(1) as i32; + total += 1; + } + _ => { + let count = count as u32; + let max_bits_out = table_log.wrapping_sub(highbit32(count.wrapping_sub(1))); + let min_state_plus = count.wrapping_shl(max_bits_out); + transform.delta_nb_bits = (max_bits_out << 16).wrapping_sub(min_state_plus); + transform.delta_find_state = total.wrapping_sub(count) as i32; + total = total.wrapping_add(count); + } + } + } + Ok(()) + } + + /// `FSE_buildCTable_rle`: a table that encodes `symbol_value`, and nothing else, in 0 bits. + pub(crate) fn build_rle(&mut self, symbol_value: u8) -> Result<()> { + self.table_log = 0; + self.max_symbol_value = u32::from(symbol_value); + self.state_table.sub_mut(..2)?.fill(0); + self.symbol_tt + .put(usize::from(symbol_value), FseSymbolTransform::default()) + } + + /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log + /// if the table has no code for it. + #[inline(always)] + pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 + } + + /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. + /// The table has a code for it. + #[inline(always)] + pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + let min_nb_bits = transform.delta_nb_bits >> 16; + let threshold = (min_nb_bits + 1) << 16; + let table_size = 1u32 << self.table_log; + let delta_from_threshold = + threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); + // A linear interpolation, which is very approximate. + let normalized_delta_from_threshold = + (delta_from_threshold << accuracy_log) >> self.table_log; + let bit_multiplier = 1u32 << accuracy_log; + ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) + } +} + +/// `FSE_CState_t` +pub(crate) struct FseCState<'a> { + value: usize, + table: &'a FseCTable, +} + +impl<'a> FseCState<'a> { + /// `FSE_initCState` + #[inline] + pub(crate) fn new(table: &'a FseCTable) -> Self { + Self { + value: 1 << (table.table_log & 15), + table, + } + } + + /// `FSE_initCState2`: the first symbol costs the fewest bits this way. The table has a code for `symbol`. + #[inline] + pub(crate) fn new2(table: &'a FseCTable, symbol: u32) -> Self { + let mut state = Self::new(table); + let transform = table.symbol_tt.at(symbol as usize).unwrap_or_default(); + let nb_bits_out = transform.delta_nb_bits.wrapping_add(1 << 15) >> 16; + state.value = (nb_bits_out << 16).wrapping_sub(transform.delta_nb_bits) as usize; + state.step(nb_bits_out, transform.delta_find_state); + state + } + + #[inline(always)] + fn step(&mut self, nb_bits_out: u32, delta_find_state: i32) { + let state_table = &self.table.state_table; + let index = (self.value >> (nb_bits_out & (usize::BITS - 1))) + .wrapping_add_signed(delta_find_state as isize); + let next = state_table.get(index & (state_table.len() - 1)); + self.value = usize::from(next.copied().unwrap_or(0)); + } + + /// `FSE_encodeSymbol`: the table has a code for `symbol`. + #[inline(always)] + pub(crate) fn encode_symbol(&mut self, bit_c: &mut BitCStream<'_>, symbol: u32) { + let symbol_tt = &self.table.symbol_tt; + let transform = symbol_tt.at(symbol as usize).unwrap_or_default(); + let nb_bits_out = ((self.value + transform.delta_nb_bits as usize) >> 16) as u32; + bit_c.add_bits(self.value as u64, nb_bits_out); + self.step(nb_bits_out, transform.delta_find_state); + } + + /// `FSE_flushCState` + #[inline(always)] + pub(crate) fn flush(&self, bit_c: &mut BitCStream<'_>) { + bit_c.add_bits(self.value as u64, self.table.table_log); + bit_c.flush_bits(); + } +} + +/// `FSE_NCountWriteBound` +pub(crate) fn ncount_write_bound(max_symbol_value: u32, table_log: u32) -> usize { + let max_header_size = + ((max_symbol_value as usize + 1) * table_log as usize + 4 + 2) / 8 + 1 + 2; + if max_symbol_value != 0 { + max_header_size + } else { + FSE_NCOUNTBOUND + } +} + +/// `FSE_writeNCount_generic`: `table_log` is from 5 to 12. +fn write_ncount_generic( + header: &mut [u8], + normalized_counter: &[i16], + max_symbol_value: u32, + table_log: u32, + write_is_safe: bool, +) -> Result { + let counts = normalized_counter.sub(..=max_symbol_value as usize)?; + let alphabet_size = counts.len(); + let mut out = 0usize; + let table_size = 1i32 << table_log; + let mut bit_stream = table_log - FSE_MIN_TABLELOG; + let mut bit_count = 4i32; + let mut symbol = 0usize; + let mut previous_is_0 = false; + + // The lowest 16 bits of the stream leave it. + let mut write16 = |bit_stream: &mut u32, out: &mut usize| -> Result<()> { + if !write_is_safe && *out + 2 > header.len() { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le16(header, *out, *bit_stream as u16)?; + *out += 2; + *bit_stream >>= 16; + Ok(()) + }; + let shifted = |value: u32, bit_count: i32| value.wrapping_shl(bit_count as u32); + + let mut remaining = table_size + 1; + let mut threshold = table_size; + let mut nb_bits = table_log as i32 + 1; + + while symbol < alphabet_size && remaining > 1 { + if previous_is_0 { + // After a count of 0 comes how many more follow it. + let mut start = symbol; + while symbol < alphabet_size && counts.at(symbol)? == 0 { + symbol += 1; + } + if symbol == alphabet_size { + // The counts do not add up. + break; + } + while symbol >= start + 24 { + start += 24; + bit_stream = bit_stream.wrapping_add(shifted(0xFFFF, bit_count)); + write16(&mut bit_stream, &mut out)?; + } + while symbol >= start + 3 { + start += 3; + bit_stream = bit_stream.wrapping_add(shifted(3, bit_count)); + bit_count += 2; + } + bit_stream = bit_stream.wrapping_add(shifted((symbol - start) as u32, bit_count)); + bit_count += 2; + if bit_count > 16 { + write16(&mut bit_stream, &mut out)?; + bit_count -= 16; + } + } + { + let mut count = i32::from(counts.at(symbol)?); + symbol += 1; + let max = (2 * threshold - 1) - remaining; + remaining -= count.abs(); + // The stored value is the count plus one, since a count can be -1. + count += 1; + if count >= threshold { + count += max; + } + bit_stream = bit_stream.wrapping_add(shifted(count as u32, bit_count)); + bit_count += nb_bits; + bit_count -= i32::from(count < max); + previous_is_0 = count == 1; + if remaining < 1 { + return Err(ErrorCode::Generic); + } + while remaining < threshold { + nb_bits -= 1; + threshold >>= 1; + } + } + if bit_count > 16 { + write16(&mut bit_stream, &mut out)?; + bit_count -= 16; + } + } + + if remaining != 1 { + return Err(ErrorCode::Generic); + } + + let last = out; + write16(&mut bit_stream, &mut out)?; + Ok(last + ((bit_count + 7) / 8) as usize) +} + +/// `FSE_writeNCount`: writes the header that [`crate::common::entropy::read_ncount`] reads, and returns its size. +pub(crate) fn write_ncount( + buffer: &mut [u8], + normalized_counter: &[i16], + max_symbol_value: u32, + table_log: u32, +) -> Result { + if table_log > FSE_MAX_TABLELOG { + return Err(ErrorCode::TableLogTooLarge); + } + if table_log < FSE_MIN_TABLELOG { + return Err(ErrorCode::Generic); + } + let write_is_safe = buffer.len() >= ncount_write_bound(max_symbol_value, table_log); + write_ncount_generic( + buffer, + normalized_counter, + max_symbol_value, + table_log, + write_is_safe, + ) +} + +/// `FSE_minTableLog`: the smallest table log that can represent the distribution. `src_size` is more than 1. +fn min_table_log(src_size: usize, max_symbol_value: u32) -> u32 { + let min_bits_src = highbit32(src_size as u32) + 1; + let min_bits_symbols = highbit32(max_symbol_value) + 2; + min_bits_src.min(min_bits_symbols) +} + +/// `FSE_optimalTableLog_internal`: `src_size` is more than 1. +pub(crate) fn optimal_table_log_internal( + max_table_log: u32, + src_size: usize, + max_symbol_value: u32, + minus: u32, +) -> u32 { + let max_bits_src = highbit32(src_size.wrapping_sub(1) as u32).wrapping_sub(minus); + let mut table_log = max_table_log; + let min_bits = min_table_log(src_size, max_symbol_value); + if table_log == 0 { + table_log = FSE_DEFAULT_TABLELOG; + } + // More accuracy than the input has symbols for is of no use. + if max_bits_src < table_log { + table_log = max_bits_src; + } + if min_bits > table_log { + table_log = min_bits; + } + table_log.clamp(FSE_MIN_TABLELOG, FSE_MAX_TABLELOG) +} + +/// `FSE_optimalTableLog` +pub(crate) fn optimal_table_log(max_table_log: u32, src_size: usize, max_symbol_value: u32) -> u32 { + optimal_table_log_internal(max_table_log, src_size, max_symbol_value, 2) +} + +/// `FSE_normalizeM2`: the second normalization method, for when the first one would take too much from the most +/// frequent symbol. `norm` and `count` are of the same length. +fn normalize_m2( + norm: &mut [i16], + table_log: u32, + count: &[u32], + mut total: usize, + low_prob_count: i16, +) -> Result<()> { + const NOT_YET_ASSIGNED: i16 = -2; + let table_size = 1u32 << table_log; + let mut distributed = 0u32; + + // The symbols of low probability first. + let low_threshold = (total >> table_log) as u32; + let mut low_one = (total.wrapping_mul(3) >> (table_log + 1)) as u32; + + for (norm, &count) in norm.iter_mut().zip(count) { + if count == 0 { + *norm = 0; + } else if count <= low_threshold { + *norm = low_prob_count; + distributed += 1; + total = total.wrapping_sub(count as usize); + } else if count <= low_one { + *norm = 1; + distributed += 1; + total = total.wrapping_sub(count as usize); + } else { + *norm = NOT_YET_ASSIGNED; + } + } + let mut to_distribute = table_size.wrapping_sub(distributed); + + if to_distribute == 0 { + return Ok(()); + } + + if total / to_distribute as usize > low_one as usize { + // The rest would get too much each: more symbols get 1. + low_one = total + .wrapping_mul(3) + .checked_div(to_distribute.wrapping_mul(2) as usize) + .ok_or(BUG)? as u32; + for (norm, &count) in norm.iter_mut().zip(count) { + if *norm == NOT_YET_ASSIGNED && count <= low_one { + *norm = 1; + distributed += 1; + total = total.wrapping_sub(count as usize); + } + } + to_distribute = table_size.wrapping_sub(distributed); + } + + if distributed as usize == count.len() { + // All the values are small. The first symbol of the largest count gets what is left. + let mut max_v = 0usize; + let mut max_c = 0u32; + for (s, &count) in count.iter().enumerate() { + if count > max_c { + max_v = s; + max_c = count; + } + } + let norm = norm.at_mut(max_v)?; + *norm = norm.wrapping_add(to_distribute as i16); + return Ok(()); + } + + if total == 0 { + // All the symbols were of low enough probability to get 1 or less: what is left goes round. + if !norm.iter().any(|&norm| norm > 0) { + return Err(BUG); + } + while to_distribute > 0 { + for norm in norm.iter_mut().filter(|norm| **norm > 0) { + if to_distribute == 0 { + break; + } + to_distribute -= 1; + *norm = norm.wrapping_add(1); + } + } + return Ok(()); + } + + let v_step_log = 62 - table_log; + let mid = (1u64 << (v_step_log - 1)) - 1; + // A scale on what remains. + let r_step = ((1u64 << v_step_log) * u64::from(to_distribute) + mid) + .checked_div(u64::from(total as u32)) + .ok_or(BUG)?; + let mut tmp_total = mid; + for (norm, &count) in norm.iter_mut().zip(count) { + if *norm == NOT_YET_ASSIGNED { + let end = tmp_total.wrapping_add(u64::from(count).wrapping_mul(r_step)); + let s_start = (tmp_total >> v_step_log) as u32; + let s_end = (end >> v_step_log) as u32; + let weight = s_end.wrapping_sub(s_start); + if weight < 1 { + return Err(ErrorCode::Generic); + } + *norm = weight as i16; + tmp_total = end; + } + } + Ok(()) +} + +/// `FSE_normalizeCount`: scales `count`, which adds up to `total`, to add up to `1 << table_log`. Returns the table +/// log, or 0 if there is only one symbol, which is a case for run-length encoding. +pub(crate) fn normalize_count( + normalized_counter: &mut [i16], + mut table_log: u32, + count: &[u32], + total: usize, + max_symbol_value: u32, + use_low_prob_count: bool, +) -> Result { + if table_log == 0 { + table_log = FSE_DEFAULT_TABLELOG; + } + if table_log < FSE_MIN_TABLELOG { + return Err(ErrorCode::Generic); + } + if table_log > FSE_MAX_TABLELOG { + return Err(ErrorCode::TableLogTooLarge); + } + if table_log < min_table_log(total, max_symbol_value) { + return Err(ErrorCode::Generic); + } + + const RTB_TABLE: [u32; 8] = [0, 473195, 504333, 520860, 550000, 700000, 750000, 830000]; + let low_prob_count: i16 = if use_low_prob_count { -1 } else { 1 }; + let scale = 62 - table_log; + let step = (1u64 << 62) + .checked_div(u64::from(total as u32)) + .ok_or(BUG)?; + let v_step = 1u64 << (scale - 20); + let mut still_to_distribute = 1i32 << table_log; + let mut largest = 0usize; + let mut largest_p = 0i16; + let low_threshold = (total >> table_log) as u32; + + let count = count.sub(..=max_symbol_value as usize)?; + let norm = normalized_counter.sub_mut(..=max_symbol_value as usize)?; + + for (s, (norm, &count)) in norm.iter_mut().zip(count).enumerate() { + if count as usize == total { + return Ok(0); + } + if count == 0 { + *norm = 0; + } else if count <= low_threshold { + *norm = low_prob_count; + still_to_distribute -= 1; + } else { + let scaled = u64::from(count).wrapping_mul(step); + let mut proba = (scaled >> scale) as i16; + if proba < 8 { + let rest_to_beat = v_step * u64::from(RTB_TABLE.at(proba as usize)?); + proba += i16::from(scaled.wrapping_sub((proba as u64) << scale) > rest_to_beat); + } + if proba > largest_p { + largest_p = proba; + largest = s; + } + *norm = proba; + still_to_distribute -= i32::from(proba); + } + } + let largest = norm.at_mut(largest)?; + if -still_to_distribute >= i32::from(*largest >> 1) { + // The correction is too large for one symbol to take. + normalize_m2(norm, table_log, count, total, low_prob_count)?; + } else { + *largest = largest.wrapping_add(still_to_distribute as i16); + } + Ok(table_log as usize) +} + +/// `FSE_compress_usingCTable_generic` +#[inline(always)] +fn compress_using_ctable_generic( + dst: &mut [u8], + src: &[u8], + ct: &FseCTable, +) -> usize { + let flush = |bit_c: &mut BitCStream<'_>| { + if FAST { + bit_c.flush_bits_fast(); + } else { + bit_c.flush_bits(); + } + }; + + if src.len() <= 2 { + return 0; + } + let Ok(mut bit_c) = BitCStream::new(dst) else { + return 0; + }; + let encode = |state: &mut FseCState<'_>, bit_c: &mut BitCStream<'_>, symbol: u8| { + state.encode_symbol(bit_c, u32::from(symbol)); + }; + + // From the last symbol to the first, two states taking turns. + let mut state1; + let mut state2; + let mut rest; + if src.len() & 1 != 0 { + let [head @ .., c, b, a] = src else { + return 0; + }; + state1 = FseCState::new2(ct, u32::from(*a)); + state2 = FseCState::new2(ct, u32::from(*b)); + encode(&mut state1, &mut bit_c, *c); + flush(&mut bit_c); + rest = head; + } else { + let [head @ .., b, a] = src else { + return 0; + }; + state2 = FseCState::new2(ct, u32::from(*a)); + state1 = FseCState::new2(ct, u32::from(*b)); + rest = head; + } + + if let ([head @ .., b, a], true) = (rest, rest.len() & 2 != 0) { + encode(&mut state2, &mut bit_c, *a); + encode(&mut state1, &mut bit_c, *b); + flush(&mut bit_c); + rest = head; + } + + let (_, quads) = rest.as_rchunks::<4>(); + for &[d, c, b, a] in quads.iter().rev() { + encode(&mut state2, &mut bit_c, a); + encode(&mut state1, &mut bit_c, b); + encode(&mut state2, &mut bit_c, c); + encode(&mut state1, &mut bit_c, d); + flush(&mut bit_c); + } + + state2.flush(&mut bit_c); + state1.flush(&mut bit_c); + bit_c.close() +} + +/// `FSE_compress_usingCTable`: the table has a code for every symbol in `src`. Returns the size of the output, or 0 +/// if `src` has less than 3 symbols or `dst` is too small. +pub(crate) fn compress_using_ctable(dst: &mut [u8], src: &[u8], ct: &FseCTable) -> usize { + if dst.len() >= block_bound(src.len()) { + compress_using_ctable_generic::(dst, src, ct) + } else { + compress_using_ctable_generic::(dst, src, ct) + } +} diff --git a/src/zstd/wip/work/entropy/zstd/compress/hist.rs b/src/zstd/wip/work/entropy/zstd/compress/hist.rs new file mode 100644 index 000000000000..745fc94e641f --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/compress/hist.rs @@ -0,0 +1,132 @@ +//! `hist.c`: how many times each byte value occurs. + +use crate::error::{ErrorCode, Result}; +use crate::util::{BUG, SliceExt}; + +/// `HIST_add`: `count` has room for every value in `src`. +pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { + for &byte in src { + *count.at_mut(usize::from(byte))? += 1; + } + Ok(()) +} + +/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. +/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. +pub(crate) fn count_simple( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + let mut max_symbol = *max_symbol_value as usize; + count.sub_mut(..=max_symbol)?.fill(0); + if src.is_empty() { + *max_symbol_value = 0; + return Ok(0); + } + add(count, src)?; + while count.at(max_symbol)? == 0 { + max_symbol = max_symbol.checked_sub(1).ok_or(BUG)?; + } + *max_symbol_value = max_symbol as u32; + Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) +} + +#[inline(always)] +fn increment(counting: &mut [u32; 256], byte: u8) { + if let Some(count) = counting.get_mut(usize::from(byte)) { + *count = count.wrapping_add(1); + } +} + +/// `HIST_count_parallel_wksp`: four tables, so that a run of one value does not wait for its own counter. +/// `max_symbol_value` is at most 255. With `check`, a value above it is an error rather than trusted not to occur. +fn count_parallel( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], + check: bool, +) -> Result { + let count_size = *max_symbol_value as usize + 1; + if src.is_empty() { + count.sub_mut(..count_size)?.fill(0); + *max_symbol_value = 0; + return Ok(0); + } + + let mut counting1 = [0u32; 256]; + let mut counting2 = [0u32; 256]; + let mut counting3 = [0u32; 256]; + let mut counting4 = [0u32; 256]; + + // Each word is read before the bytes of the one before it are counted. + let (words, rest) = src.as_chunks::<8>(); + if let Some((&first, words)) = words.split_first() { + let mut cached = u64::from_le_bytes(first); + let mut count_word = |next: [u8; 8]| { + let c = cached; + cached = u64::from_le_bytes(next); + increment(&mut counting1, c as u8); + increment(&mut counting2, (c >> 8) as u8); + increment(&mut counting3, (c >> 16) as u8); + increment(&mut counting4, (c >> 24) as u8); + increment(&mut counting1, (c >> 32) as u8); + increment(&mut counting2, (c >> 40) as u8); + increment(&mut counting3, (c >> 48) as u8); + increment(&mut counting4, (c >> 56) as u8); + }; + let (stripes, words) = words.as_chunks::<2>(); + for &[a, b] in stripes { + count_word(a); + count_word(b); + } + for &word in words { + count_word(word); + } + count_word([0; 8]); + } + for &byte in rest { + increment(&mut counting1, byte); + } + + let mut max = 0u32; + let others = counting2.iter().zip(&counting3).zip(&counting4); + for (sum, ((&c2, &c3), &c4)) in counting1.iter_mut().zip(others) { + *sum = sum.wrapping_add(c2).wrapping_add(c3).wrapping_add(c4); + max = max.max(*sum); + } + + let max_symbol = counting1.iter().rposition(|&c| c != 0).ok_or(BUG)? as u32; + if check && max_symbol > *max_symbol_value { + return Err(ErrorCode::MaxSymbolValueTooSmall); + } + *max_symbol_value = max_symbol; + count.write_at(0, counting1.sub(..count_size)?)?; + Ok(max as usize) +} + +/// `HIST_countFast_wksp`: as [`count_simple`], faster on large inputs. +pub(crate) fn count_fast_wksp( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + if src.len() < 1500 { + return Ok(count_simple(count, max_symbol_value, src)? as usize); + } + count_parallel(count, max_symbol_value, src, false) +} + +/// `HIST_count_wksp`: as [`count_fast_wksp`], but a value above `max_symbol_value` in `src` is +/// [`ErrorCode::MaxSymbolValueTooSmall`]. +pub(crate) fn count_wksp( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + if *max_symbol_value < 255 { + return count_parallel(count, max_symbol_value, src, true); + } + *max_symbol_value = 255; + count_fast_wksp(count, max_symbol_value, src) +} diff --git a/src/zstd/wip/work/entropy/zstd/compress/huf.rs b/src/zstd/wip/work/entropy/zstd/compress/huf.rs new file mode 100644 index 000000000000..7052bc0be190 --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/compress/huf.rs @@ -0,0 +1,1162 @@ +//! `huf_compress.c`: Huffman coding of literals. + +use super::fse::{self, FseCTable}; +use super::hist; +use super::internal::Strategy; +use crate::common::entropy::huf_read_stats; +use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; +use crate::error::{ErrorCode, Result}; +use crate::util::{BUG, SliceExt, highbit32, write_le16}; + +pub(crate) const HUF_BLOCKSIZE_MAX: usize = 128 * 1024; +pub(crate) const HUF_WORKSPACE_SIZE: usize = (8 << 10) + 512; +pub(crate) const HUF_TABLELOG_DEFAULT: u32 = 11; +/// The strategies from this one on look for the best depth of the tree. +pub(crate) const HUF_OPTIMAL_DEPTH_THRESHOLD: Strategy = Strategy::BtUltra; +const HUF_CTABLEBOUND: usize = 129; + +/// `HUF_flags_e` +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct HufFlags { + /// Try every depth of the tree and keep the one that gives the smallest output, instead of a guess. + pub(crate) optimal_depth: bool, + /// Use the previous table again whenever it is possible, without looking for a better one. + pub(crate) prefer_repeat: bool, + /// Sample the input first, and give up at once if it looks like it cannot be compressed. + pub(crate) suspect_uncompressible: bool, +} + +/// `HUF_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum HufRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the +/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in +/// its highest bits. +#[derive(Clone)] +pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); + +impl HufCTable { + pub(crate) const fn new() -> Self { + Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) + } + + /// The elements, by symbol. + #[inline(always)] + fn elements(&self) -> &[u64; HUF_SYMBOLVALUE_MAX as usize + 1] { + let [_, elements @ ..] = &self.0; + elements + } + + #[inline(always)] + fn elements_mut(&mut self) -> &mut [u64; HUF_SYMBOLVALUE_MAX as usize + 1] { + let [_, elements @ ..] = &mut self.0; + elements + } + + /// `HUF_writeCTableHeader` + fn write_header(&mut self, table_log: u32, max_symbol_value: u32) { + let [header, ..] = &mut self.0; + *header = u64::from(table_log as u8) | (u64::from(max_symbol_value as u8) << 8); + } + + /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. + #[inline] + pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { + let [header, elements @ ..] = &self.0; + let max_symbol_value = (header >> 8) as u8; + if symbol_value > u32::from(max_symbol_value) { + return 0; + } + (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 + } +} + +/// `HUF_CTableHeader` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct HufCTableHeader { + pub(crate) table_log: u8, + pub(crate) max_symbol_value: u8, +} + +/// `HUF_readCTableHeader` +#[inline] +pub(crate) fn read_ctable_header(ctable: &HufCTable) -> HufCTableHeader { + let [header, ..] = ctable.0; + HufCTableHeader { + table_log: header as u8, + max_symbol_value: (header >> 8) as u8, + } +} + +/// `HUF_getNbBits` +#[inline(always)] +const fn get_nb_bits(elt: u64) -> u64 { + elt & 0xFF +} + +/// `HUF_setValue`: `elt` has its number of bits, and nothing else. +#[inline(always)] +fn set_value(elt: &mut u64, value: u64) { + let nb_bits = get_nb_bits(*elt) as u32; + if nb_bits > 0 { + *elt |= value.wrapping_shl(u64::BITS.wrapping_sub(nb_bits)); + } +} + +/// Gives each symbol its code. `ct` has the number of bits of each, at most `table_log`, and `nb_per_rank` has how +/// many symbols there are of each number of bits. +fn set_values( + ct: &mut [u64], + nb_per_rank: &[u16; RANKS], + table_log: u32, +) -> Result<()> { + let mut val_per_rank = [0u16; RANKS]; + // The first code of each rank: the longest codes have the lowest values. + let mut min = 0u16; + for n in (1..=table_log as usize).rev() { + val_per_rank.put(n, min)?; + min = min.wrapping_add(nb_per_rank.at(n)?) >> 1; + } + for elt in ct { + let value = val_per_rank.at_mut(get_nb_bits(*elt) as usize)?; + set_value(elt, u64::from(*value)); + *value = value.wrapping_add(1); + } + Ok(()) +} + +const MAX_FSE_TABLELOG_FOR_HUFF_HEADER: u32 = 6; + +/// `HUF_compressWeights`: returns 0 if the weights cannot be compressed, and 1 if they are all the same. +fn compress_weights(dst: &mut [u8], weight_table: &[u8]) -> Result { + let mut max_symbol_value = HUF_TABLELOG_MAX; + let mut count = [0u32; HUF_TABLELOG_MAX as usize + 1]; + let mut norm = [0i16; HUF_TABLELOG_MAX as usize + 1]; + let wt_size = weight_table.len(); + + if wt_size <= 1 { + return Ok(0); + } + + let max_count = hist::count_simple(&mut count, &mut max_symbol_value, weight_table)?; + if max_count as usize == wt_size { + return Ok(1); + } + if max_count == 1 { + return Ok(0); + } + + let table_log = + fse::optimal_table_log(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, wt_size, max_symbol_value); + fse::normalize_count( + &mut norm, + table_log, + &count, + wt_size, + max_symbol_value, + false, + )?; + + let h_size = fse::write_ncount(dst, &norm, max_symbol_value, table_log)?; + + let mut ctable = FseCTable::new(); + ctable.build(&norm, max_symbol_value, table_log)?; + let c_size = fse::compress_using_ctable(dst.sub_mut(h_size..)?, weight_table, &ctable); + if c_size == 0 { + return Ok(0); + } + Ok(h_size + c_size) +} + +/// `HUF_writeCTable_wksp`: writes the header that [`huf_read_stats`] reads, and returns its size. `max_symbol_value` +/// and `huff_log` are those of `ctable`. +pub(crate) fn write_ctable( + dst: &mut [u8], + ctable: &HufCTable, + max_symbol_value: u32, + huff_log: u32, +) -> Result { + let mut bits_to_weight = [0u8; HUF_TABLELOG_MAX as usize + 1]; + let mut huff_weight = [0u8; HUF_SYMBOLVALUE_MAX as usize]; + + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return Err(ErrorCode::MaxSymbolValueTooLarge); + } + // The weight of the last symbol is not stored. + let nb_weights = max_symbol_value as usize; + + for n in 1..=huff_log { + bits_to_weight.put(n as usize, (huff_log + 1 - n) as u8)?; + } + let weights = huff_weight.sub_mut(..nb_weights)?; + for (weight, &elt) in weights.iter_mut().zip(ctable.elements()) { + *weight = bits_to_weight.at(get_nb_bits(elt) as usize)?; + } + + let max_dst_size = dst.len(); + let Some((first, rest)) = dst.split_first_mut() else { + return Err(ErrorCode::DstSizeTooSmall); + }; + let h_size = compress_weights(rest, weights)?; + if h_size > 1 && h_size < nb_weights / 2 { + *first = h_size as u8; + return Ok(h_size + 1); + } + + // Not compressed: 4 bits to a weight. + if max_symbol_value > 256 - 128 { + return Err(ErrorCode::Generic); + } + let raw_size = nb_weights.div_ceil(2); + if raw_size + 1 > max_dst_size { + return Err(ErrorCode::DstSizeTooSmall); + } + *first = 128u32.wrapping_add(max_symbol_value.wrapping_sub(1)) as u8; + // An odd number of weights is completed with a 0, which `huff_weight` still has there. + let (pairs, _) = huff_weight.sub(..raw_size * 2)?.as_chunks::<2>(); + for (out, &[high, low]) in rest.iter_mut().zip(pairs) { + *out = (high << 4).wrapping_add(low); + } + Ok(raw_size + 1) +} + +/// What [`read_ctable`] found besides the table. +pub(crate) struct HufCTableStats { + /// How many bytes of the input the header takes. + pub(crate) size: usize, + pub(crate) max_symbol_value: u32, + /// Some symbol up to `max_symbol_value` has no code. + pub(crate) has_zero_weights: bool, +} + +/// `HUF_readCTable`: [`ErrorCode::MaxSymbolValueTooSmall`] if the table has symbols above `max_symbol_value`. +pub(crate) fn read_ctable( + ctable: &mut HufCTable, + max_symbol_value: u32, + src: &[u8], +) -> Result { + let mut huff_weight = [0u8; HUF_SYMBOLVALUE_MAX as usize + 1]; + let mut rank_val = [0u32; HUF_TABLELOG_MAX as usize + 1]; + + let stats = huf_read_stats(&mut huff_weight, &mut rank_val, src)?; + let table_log = stats.table_log; + let [zero_weights, ..] = rank_val; + + if table_log > HUF_TABLELOG_MAX { + return Err(ErrorCode::TableLogTooLarge); + } + if stats.nb_symbols > max_symbol_value.wrapping_add(1) { + return Err(ErrorCode::MaxSymbolValueTooSmall); + } + let max_symbol_value = stats.nb_symbols - 1; + ctable.write_header(table_log, max_symbol_value); + + let mut nb_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 2]; + let ct = ctable.elements_mut().sub_mut(..stats.nb_symbols as usize)?; + for (elt, &weight) in ct.iter_mut().zip(&huff_weight) { + let nb_bits = if weight != 0 { + (table_log + 1).wrapping_sub(u32::from(weight)) as u8 + } else { + 0 + }; + *elt = u64::from(nb_bits); + *nb_per_rank.at_mut(usize::from(nb_bits))? += 1; + } + set_values(ct, &nb_per_rank, table_log)?; + + Ok(HufCTableStats { + size: stats.size, + max_symbol_value, + has_zero_weights: zero_weights > 0, + }) +} + +/// `nodeElt` +#[derive(Clone, Copy, Default)] +struct NodeElt { + count: u32, + parent: u16, + byte: u8, + nb_bits: u8, +} + +/// The first node that is not a leaf. +const STARTNODE: i32 = HUF_SYMBOLVALUE_MAX as i32 + 1; + +/// `huffNodeTable`: the leaves from 0 on, sorted from the most frequent symbol to the rarest, the other nodes from +/// [`STARTNODE`] on, and a sentinel at -1. A number that is out of the table wraps around, which spares the checks. +struct HuffNodes([NodeElt; 2 * (HUF_SYMBOLVALUE_MAX as usize + 1)]); + +impl HuffNodes { + #[inline(always)] + const fn index(n: i32) -> usize { + n.wrapping_add(1) as usize & (2 * (HUF_SYMBOLVALUE_MAX as usize + 1) - 1) + } + + #[inline(always)] + fn get(&self, n: i32) -> NodeElt { + self.0.get(Self::index(n)).copied().unwrap_or_default() + } + + #[inline(always)] + fn set(&mut self, n: i32, node: NodeElt) { + if let Some(slot) = self.0.get_mut(Self::index(n)) { + *slot = node; + } + } + + #[inline(always)] + fn count(&self, n: i32) -> u32 { + self.get(n).count + } + + #[inline(always)] + fn nb_bits(&self, n: i32) -> u32 { + u32::from(self.get(n).nb_bits) + } + + #[inline(always)] + fn set_count(&mut self, n: i32, count: u32) { + if let Some(node) = self.0.get_mut(Self::index(n)) { + node.count = count; + } + } + + #[inline(always)] + fn set_parent(&mut self, n: i32, parent: i32) { + if let Some(node) = self.0.get_mut(Self::index(n)) { + node.parent = parent as u16; + } + } + + #[inline(always)] + fn set_nb_bits(&mut self, n: i32, nb_bits: u32) { + if let Some(node) = self.0.get_mut(Self::index(n)) { + node.nb_bits = nb_bits as u8; + } + } + + /// The nodes from 0 on. + #[inline(always)] + fn leaves(&self) -> &[NodeElt] { + let [_, nodes @ ..] = &self.0; + nodes + } +} + +/// `HUF_setMaxHeight`: makes no code longer than `target_nb_bits`, which is not 0 and is enough for a code for every +/// symbol. `last_non_null` is the rarest symbol, which has the longest code. Returns the length of the longest code. +fn set_max_height( + huff_node: &mut HuffNodes, + last_non_null: u32, + target_nb_bits: u32, +) -> Result { + let largest_bits = huff_node.nb_bits(last_non_null as i32); + if largest_bits <= target_nb_bits { + return Ok(largest_bits); + } + + // The unit of cost is the share of the code space that a code of `largest_bits` takes. + let cost_of = |nb_bits: u32| 1u32.wrapping_shl(largest_bits.wrapping_sub(nb_bits)); + let mut total_cost = 0i32; + let base_cost = cost_of(target_nb_bits); + let mut n = last_non_null as i32; + + // Shorten the codes that are too long. `total_cost` is how much more than the whole code space is taken then. + while huff_node.nb_bits(n) > target_nb_bits { + let cost = base_cost.wrapping_sub(cost_of(huff_node.nb_bits(n))); + total_cost = total_cost.wrapping_add(cost as i32); + huff_node.set_nb_bits(n, target_nb_bits); + n -= 1; + } + // `n` ends at the rarest symbol with a code shorter than `target_nb_bits`, or at the sentinel. + while huff_node.nb_bits(n) == target_nb_bits { + n -= 1; + } + + // From here on the unit is the share of a code of `target_nb_bits`. + total_cost = total_cost.wrapping_shr(largest_bits - target_nb_bits); + + // Repay the cost by making other codes longer. + const NO_SYMBOL: u32 = 0xF0F0_F0F0; + // The rarest symbol whose code is that many bits shorter than `target_nb_bits`. + let mut rank_last = [NO_SYMBOL; HUF_TABLELOG_MAX as usize + 2]; + + let mut current_nb_bits = target_nb_bits; + for pos in (0..=n).rev() { + let nb_bits = huff_node.nb_bits(pos); + if nb_bits >= current_nb_bits { + continue; + } + current_nb_bits = nb_bits; + rank_last.put((target_nb_bits - current_nb_bits) as usize, pos as u32)?; + } + + while total_cost > 0 { + // One more bit on a code that is `n_bits_to_decrease` shorter repays `1 << (n_bits_to_decrease - 1)`. + let mut n_bits_to_decrease = highbit32(total_cost as u32) as usize + 1; + while n_bits_to_decrease > 1 { + let high_pos = rank_last.at(n_bits_to_decrease)?; + let low_pos = rank_last.at(n_bits_to_decrease - 1)?; + if high_pos != NO_SYMBOL { + if low_pos == NO_SYMBOL { + break; + } + // It is cheaper to lengthen this one than two of the rank below. + let high_total = huff_node.count(high_pos as i32); + let low_total = huff_node.count(low_pos as i32).wrapping_mul(2); + if high_total <= low_total { + break; + } + } + n_bits_to_decrease -= 1; + } + while n_bits_to_decrease <= HUF_TABLELOG_MAX as usize + && rank_last.at(n_bits_to_decrease)? == NO_SYMBOL + { + n_bits_to_decrease += 1; + } + let pos = rank_last.at(n_bits_to_decrease)?; + if pos == NO_SYMBOL { + return Err(BUG); + } + total_cost -= 1 << (n_bits_to_decrease - 1); + huff_node.set_nb_bits(pos as i32, huff_node.nb_bits(pos as i32) + 1); + + // The symbol is now of the rank below, where it is the rarest only if there was none. + let below = rank_last.at_mut(n_bits_to_decrease - 1)?; + if *below == NO_SYMBOL { + *below = pos; + } + // The next symbol takes its place, if it is of the same rank. + let same_rank = target_nb_bits.wrapping_sub(n_bits_to_decrease as u32); + let next = if pos != 0 && huff_node.nb_bits(pos as i32 - 1) == same_rank { + pos - 1 + } else { + NO_SYMBOL + }; + rank_last.put(n_bits_to_decrease, next)?; + } + + // Too much may have been repaid: shorten codes of `target_nb_bits`, the most frequent symbols first. + let [_, rank_last_1, ..] = &mut rank_last; + while total_cost < 0 { + if *rank_last_1 == NO_SYMBOL { + while huff_node.nb_bits(n) == target_nb_bits { + n -= 1; + } + *rank_last_1 = n as u32; + } + *rank_last_1 = rank_last_1.wrapping_add(1); + let pos = *rank_last_1 as i32; + huff_node.set_nb_bits(pos, huff_node.nb_bits(pos).wrapping_sub(1)); + total_cost += 1; + } + + Ok(target_nb_bits) +} + +/// `rankPos` +#[derive(Clone, Copy, Default)] +struct RankPos { + base: u16, + curr: u16, +} + +/// How many buckets [`sort`] has. +const RANK_POSITION_TABLE_SIZE: usize = 192; +const RANK_POSITION_MAX_COUNT_LOG: u32 = 32; +const RANK_POSITION_LOG_BUCKETS_BEGIN: u32 = + (RANK_POSITION_TABLE_SIZE as u32 - 1) - RANK_POSITION_MAX_COUNT_LOG - 1; +/// The counts below this one have a bucket each. The others share one with the counts of the same binary logarithm. +const RANK_POSITION_DISTINCT_COUNT_CUTOFF: u32 = + RANK_POSITION_LOG_BUCKETS_BEGIN + highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN); + +/// `HUF_getIndex`: the bucket of a count. +#[inline(always)] +fn get_index(count: u32) -> usize { + if count < RANK_POSITION_DISTINCT_COUNT_CUTOFF { + count as usize + } else { + (highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN) as usize + } +} + +/// `HUF_swapNodes` +#[inline(always)] +fn swap_nodes(arr: &mut HuffNodes, a: i32, b: i32) { + let tmp = arr.get(a); + arr.set(a, arr.get(b)); + arr.set(b, tmp); +} + +/// `HUF_insertionSort`: by decreasing count, from `low` to `high` included. +#[inline] +fn insertion_sort(huff_node: &mut HuffNodes, low: i32, high: i32) { + for i in low + 1..=high { + let key = huff_node.get(i); + let mut j = i - 1; + while j >= low && huff_node.count(j) < key.count { + huff_node.set(j + 1, huff_node.get(j)); + j -= 1; + } + huff_node.set(j + 1, key); + } +} + +/// `HUF_quickSortPartition`: the pivot is the last element. Returns where it ends up. +fn quick_sort_partition(arr: &mut HuffNodes, low: i32, high: i32) -> i32 { + let pivot = arr.count(high); + let mut i = low - 1; + for j in low..high { + if arr.count(j) > pivot { + i += 1; + swap_nodes(arr, i, j); + } + } + swap_nodes(arr, i + 1, high); + i + 1 +} + +/// `HUF_simpleQuickSort`: by decreasing count, from `low` to `high` included. It recurses into the smaller part only, +/// which bounds the depth. The order it leaves symbols of the same count in decides their codes. +fn simple_quick_sort(arr: &mut HuffNodes, mut low: i32, mut high: i32) { + const INSERTION_SORT_THRESHOLD: i32 = 8; + if high - low < INSERTION_SORT_THRESHOLD { + insertion_sort(arr, low, high); + return; + } + while low < high { + let idx = quick_sort_partition(arr, low, high); + if idx - low < high - idx { + simple_quick_sort(arr, low, idx - 1); + low = idx + 1; + } else { + simple_quick_sort(arr, idx + 1, high); + high = idx - 1; + } + } +} + +/// `HUF_sort`: makes the first nodes, which are all zeros, the symbols by decreasing count. At most 256 counts. +fn sort(huff_node: &mut HuffNodes, count: &[u32]) -> Result<()> { + let mut rank_position = [RankPos::default(); RANK_POSITION_TABLE_SIZE]; + + // A bucket starts after the symbols of all the buckets above it. + for &c in count { + rank_position.at_mut(get_index(c))?.base += 1; + } + let mut above = 0u16; + for rank in rank_position.iter_mut().rev() { + rank.base += above; + rank.curr = rank.base; + above = rank.base; + } + + for (n, &c) in count.iter().enumerate() { + let rank = rank_position.at_mut(get_index(c) + 1)?; + let node = NodeElt { + count: c, + byte: n as u8, + ..NodeElt::default() + }; + huff_node.set(i32::from(rank.curr), node); + rank.curr += 1; + } + + let shared = RANK_POSITION_DISTINCT_COUNT_CUTOFF as usize..RANK_POSITION_TABLE_SIZE - 1; + for rank in rank_position.sub(shared)? { + if rank.curr > rank.base + 1 { + simple_quick_sort(huff_node, i32::from(rank.base), i32::from(rank.curr) - 1); + } + } + Ok(()) +} + +/// `HUF_buildTree`: `huff_node` is sorted, and some count is not 0. Sets the depth of every leaf, and returns the +/// rarest symbol whose count is not 0. +fn build_tree(huff_node: &mut HuffNodes, max_symbol_value: u32) -> Result { + let mut symbols = (0..=max_symbol_value as i32).rev(); + let non_null_rank = symbols.find(|&n| huff_node.count(n) != 0).ok_or(BUG)?; + let mut low_s = non_null_rank; + let mut node_nb = STARTNODE; + let node_root = node_nb + low_s - 1; + let mut low_n = node_nb; + + let mut join = |huff_node: &mut HuffNodes, n1: i32, n2: i32| { + let count = huff_node.count(n1).wrapping_add(huff_node.count(n2)); + huff_node.set_count(node_nb, count); + huff_node.set_parent(n1, node_nb); + huff_node.set_parent(n2, node_nb); + node_nb += 1; + }; + + join(huff_node, low_s, low_s - 1); + low_s -= 2; + for n in STARTNODE + 1..=node_root { + huff_node.set_count(n, 1 << 30); + } + // The sentinel is never the smaller of the two. + huff_node.set_count(-1, 1 << 31); + + // Join the two nodes of the lowest counts, which are the next leaves or the next parents. + for _ in STARTNODE + 1..=node_root { + let mut lowest = |huff_node: &HuffNodes| { + if huff_node.count(low_s) < huff_node.count(low_n) { + low_s -= 1; + low_s + 1 + } else { + low_n += 1; + low_n - 1 + } + }; + let n1 = lowest(huff_node); + let n2 = lowest(huff_node); + join(huff_node, n1, n2); + } + + // The parents come after their children, so that this goes from the root down. + huff_node.set_nb_bits(node_root, 0); + for n in (STARTNODE..node_root).rev().chain(0..=non_null_rank) { + let parent = i32::from(huff_node.get(n).parent); + huff_node.set_nb_bits(n, huff_node.nb_bits(parent) + 1); + } + + Ok(non_null_rank) +} + +/// `HUF_buildCTableFromTree`: no leaf is deeper than `max_nb_bits`, which is at most 12. +fn build_ctable_from_tree( + ctable: &mut HufCTable, + huff_node: &[NodeElt], + non_null_rank: i32, + max_symbol_value: u32, + max_nb_bits: u32, +) -> Result<()> { + let alphabet_size = max_symbol_value as usize + 1; + let ct = ctable.elements_mut().sub_mut(..alphabet_size)?; + let mut nb_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 1]; + for node in huff_node.sub(..=non_null_rank as usize)? { + *nb_per_rank.at_mut(usize::from(node.nb_bits))? += 1; + } + for node in huff_node.sub(..alphabet_size)? { + ct.put(usize::from(node.byte), u64::from(node.nb_bits))?; + } + set_values(ct, &nb_per_rank, max_nb_bits)?; + ctable.write_header(max_nb_bits, max_symbol_value); + Ok(()) +} + +/// `HUF_buildCTable_wksp`: some count up to `max_symbol_value` is not 0, and `max_nb_bits` is enough for a code for +/// each that is not, or 0 for the default. Returns the length of the longest code. +pub(crate) fn build_ctable( + ctable: &mut HufCTable, + count: &[u32], + max_symbol_value: u32, + mut max_nb_bits: u32, +) -> Result { + if max_nb_bits == 0 { + max_nb_bits = HUF_TABLELOG_DEFAULT; + } + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return Err(ErrorCode::MaxSymbolValueTooLarge); + } + let mut huff_node = HuffNodes([NodeElt::default(); _]); + + sort(&mut huff_node, count.sub(..=max_symbol_value as usize)?)?; + let non_null_rank = build_tree(&mut huff_node, max_symbol_value)?; + max_nb_bits = set_max_height(&mut huff_node, non_null_rank as u32, max_nb_bits)?; + if max_nb_bits > HUF_TABLELOG_MAX { + return Err(ErrorCode::Generic); + } + build_ctable_from_tree( + ctable, + huff_node.leaves(), + non_null_rank, + max_symbol_value, + max_nb_bits, + )?; + Ok(max_nb_bits as usize) +} + +/// `HUF_estimateCompressedSize`: how many bytes the symbols counted in `count` take, without the last partial one. +pub(crate) fn estimate_compressed_size( + ctable: &HufCTable, + count: &[u32], + max_symbol_value: u32, +) -> usize { + let nb_bits: usize = (ctable.elements().iter().zip(count)) + .take(max_symbol_value as usize + 1) + .map(|(&elt, &count)| get_nb_bits(elt) as usize * count as usize) + .sum(); + nb_bits >> 3 +} + +/// `HUF_validateCTable`: whether the table has a code for every symbol whose count is not 0. +pub(crate) fn validate_ctable(ctable: &HufCTable, count: &[u32], max_symbol_value: u32) -> bool { + if u32::from(read_ctable_header(ctable).max_symbol_value) < max_symbol_value { + return false; + } + (ctable.elements().iter().zip(count)) + .take(max_symbol_value as usize + 1) + .all(|(&elt, &count)| count == 0 || get_nb_bits(elt) != 0) +} + +/// `HUF_compressBound` +pub(crate) const fn compress_bound(size: usize) -> usize { + HUF_CTABLEBOUND + size + (size >> 8) + 8 +} + +/// One `bitContainer` of `HUF_CStream_t` and its `bitPos`. +#[derive(Clone, Copy, Default)] +struct BitContainer { + /// The bits are at the top. The lowest bits can be dirty. + bits: u64, + /// The number of bits is in the lowest byte. The rest is dirty. + pos: u64, +} + +impl BitContainer { + /// `HUF_addBits`: there is room for the code of `elt`. With `FAST`, the lowest byte of `elt` is not cleared + /// first, which takes that there is room for 4 bits more. + #[inline(always)] + fn add_bits(&mut self, elt: u64) { + self.bits >>= get_nb_bits(elt) & 63; + self.bits |= if FAST { elt } else { elt & !0xFF }; + self.pos = self.pos.wrapping_add(elt); + } + + #[inline(always)] + const fn nb_bits(self) -> u64 { + self.pos & 0xFF + } +} + +/// `HUF_CStream_t`. Its second container is a local variable of the loop that uses it. +struct HufCStream<'a> { + container: BitContainer, + dst: &'a mut [u8], + pos: usize, + /// The last position a whole container can be written to. + end: usize, +} + +impl<'a> HufCStream<'a> { + /// `HUF_initCStream` + #[inline] + fn new(dst: &'a mut [u8]) -> Result { + if dst.len() <= size_of::() { + return Err(ErrorCode::DstSizeTooSmall); + } + let end = dst.len() - size_of::(); + Ok(Self { + container: BitContainer::default(), + dst, + pos: 0, + end, + }) + } + + /// `HUF_mergeIndex1`: there is room for the bits of `second`, which come after those the stream has. + #[inline(always)] + fn merge(&mut self, second: BitContainer) { + self.container.bits >>= second.nb_bits() & 63; + self.container.bits |= second.bits; + self.container.pos = self.container.pos.wrapping_add(second.pos); + } + + /// `HUF_flushBits`: writes the whole bytes. At the end of the output the stream stops advancing, which + /// [`Self::close`] reports. The C code has a variant without that limit for when the output is known to be large + /// enough, but here the limit is what spares a check on the write. + #[inline(always)] + fn flush_bits(&mut self) { + let nb_bits = self.container.nb_bits(); + let nb_bytes = (nb_bits >> 3) as usize; + let bits = self.container.bits >> (u64::from(u64::BITS).wrapping_sub(nb_bits) & 63); + self.container.pos &= 7; + let pos = self.pos.min(self.end); + if let Ok(out) = self.dst.array_at_mut(pos) { + *out = bits.to_le_bytes(); + } + self.pos = pos + nb_bytes; + } + + /// `HUF_closeCStream`: the size of the stream, or 0 if it did not fit. + #[inline] + fn close(mut self) -> usize { + /// `HUF_endMark`: one bit, set. + const END_MARK: u64 = 1 | (1 << 63); + self.container.add_bits::(END_MARK); + self.flush_bits(); + if self.pos >= self.end { + return 0; + } + self.pos + usize::from(self.container.nb_bits() > 0) + } +} + +type Elements = [u64; HUF_SYMBOLVALUE_MAX as usize + 1]; + +/// `HUF_encodeSymbol` +#[inline(always)] +fn encode_symbol(container: &mut BitContainer, symbol: u8, ct: &Elements) { + container.add_bits::(ct.get(usize::from(symbol)).copied().unwrap_or(0)); +} + +/// Encodes `symbols` from the last to the first. +#[inline(always)] +fn encode_symbols( + container: &mut BitContainer, + symbols: &[u8; UNROLL], + ct: &Elements, +) { + if let Some((&last, symbols)) = symbols.split_first() { + for &symbol in symbols.iter().rev() { + encode_symbol::(container, symbol, ct); + } + encode_symbol::(container, last, ct); + } +} + +/// `HUF_compress1X_usingCTable_internal_body_loop`: a container has room for `UNROLL` symbols on top of the 7 bits +/// that a flush can leave, and, with `LAST_FAST`, for 4 bits more. +#[inline(always)] +fn compress_1x_body_loop( + bit_c: &mut HufCStream<'_>, + src: &[u8], + ct: &Elements, +) { + let (groups, rem) = src.as_chunks::(); + if !rem.is_empty() { + for &symbol in rem.iter().rev() { + encode_symbol::(&mut bit_c.container, symbol, ct); + } + bit_c.flush_bits(); + } + let (pairs, odd) = groups.as_chunks::<2>(); + if let [group] = odd { + encode_symbols::(&mut bit_c.container, group, ct); + bit_c.flush_bits(); + } + for [second, first] in pairs.iter().rev() { + encode_symbols::(&mut bit_c.container, first, ct); + bit_c.flush_bits(); + // The second group does not depend on the flush, since it has a container of its own. + let mut container = BitContainer::default(); + encode_symbols::(&mut container, second, ct); + bit_c.merge(container); + bit_c.flush_bits(); + } +} + +/// `HUF_tightCompressBound`: no output is larger, given the longest code. +const fn tight_compress_bound(src_size: usize, table_log: usize) -> usize { + ((src_size * table_log) >> 3) + 8 +} + +/// `HUF_compress1X_usingCTable`: the table has a code for every symbol in `src`. Returns the size of the output, or 0 +/// if `dst` is too small. +pub(crate) fn compress_1x_using_ctable( + dst: &mut [u8], + src: &[u8], + ctable: &HufCTable, +) -> Result { + let table_log = read_ctable_header(ctable).table_log; + let ct = ctable.elements(); + let dst_size = dst.len(); + + let Ok(mut bit_c) = HufCStream::new(dst) else { + return Ok(0); + }; + let bit_c_ref = &mut bit_c; + + if dst_size < tight_compress_bound(src.len(), usize::from(table_log)) || table_log > 11 { + compress_1x_body_loop::<4, false>(bit_c_ref, src, ct); + } else { + match table_log { + 11 => compress_1x_body_loop::<5, false>(bit_c_ref, src, ct), + 10 => compress_1x_body_loop::<5, true>(bit_c_ref, src, ct), + 9 => compress_1x_body_loop::<6, false>(bit_c_ref, src, ct), + 8 => compress_1x_body_loop::<7, false>(bit_c_ref, src, ct), + 7 => compress_1x_body_loop::<8, false>(bit_c_ref, src, ct), + _ => compress_1x_body_loop::<9, true>(bit_c_ref, src, ct), + } + } + Ok(bit_c.close()) +} + +/// `HUF_compress4X_usingCTable`: four streams, after the sizes of the first three. The table has a code for every +/// symbol in `src`. Returns the size of the output, or 0 if `src` is too small for it, `dst` is, or a stream is too +/// large. +pub(crate) fn compress_4x_using_ctable( + dst: &mut [u8], + src: &[u8], + ctable: &HufCTable, +) -> Result { + let segment_size = src.len().div_ceil(4); + if dst.len() < 6 + 1 + 1 + 1 + 8 { + return Ok(0); + } + if src.len() < 12 { + return Ok(0); + } + let mut op = 6usize; + for (stream, segment) in src.chunks(segment_size).enumerate() { + let c_size = compress_1x_using_ctable(dst.sub_mut(op..)?, segment, ctable)?; + if c_size == 0 || c_size > 65535 { + return Ok(0); + } + if stream < 3 { + write_le16(dst, stream * 2, c_size as u16)?; + } + op += c_size; + } + Ok(op) +} + +/// `HUF_nbStreams_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum NbStreams { + Single, + Four, +} + +/// `HUF_compressCTable_internal`: writes from `op` on. Returns how much of `dst` is used, or 0 if that saves nothing. +fn compress_ctable_internal( + dst: &mut [u8], + op: usize, + src: &[u8], + nb_streams: NbStreams, + ctable: &HufCTable, +) -> Result { + let out = dst.sub_mut(op..)?; + let c_size = match nb_streams { + NbStreams::Single => compress_1x_using_ctable(out, src, ctable)?, + NbStreams::Four => compress_4x_using_ctable(out, src, ctable)?, + }; + if c_size == 0 { + return Ok(0); + } + if op + c_size >= src.len().wrapping_sub(1) { + return Ok(0); + } + Ok(op + c_size) +} + +const SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE: usize = 4096; +const SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO: usize = 10; + +/// `HUF_cardinality`: how many counts up to `max_symbol_value` are not 0. +pub(crate) fn cardinality(count: &[u32], max_symbol_value: u32) -> u32 { + let counts = count.iter().take(max_symbol_value as usize + 1); + counts.filter(|&&count| count != 0).count() as u32 +} + +/// `HUF_minTableLog`: the shortest that the longest code can be. `symbol_cardinality` is not 0. +pub(crate) fn min_table_log(symbol_cardinality: u32) -> u32 { + highbit32(symbol_cardinality) + 1 +} + +/// `HUF_optimalTableLog`: the limit to put on the length of the codes. `src_size` is more than 1. With +/// `flags.optimal_depth`, `table` is overwritten. +pub(crate) fn optimal_table_log( + max_table_log: u32, + src_size: usize, + max_symbol_value: u32, + table: &mut HufCTable, + count: &[u32], + flags: HufFlags, +) -> u32 { + if !flags.optimal_depth { + return fse::optimal_table_log_internal(max_table_log, src_size, max_symbol_value, 1); + } + + // Large enough that no step of `write_ctable` is short of room. + let mut dst = [0u8; 512]; + let symbol_cardinality = cardinality(count, max_symbol_value); + let min_table_log = min_table_log(symbol_cardinality); + let mut opt_size = usize::MAX - 1; + let mut opt_log = max_table_log; + + // Search until the size increases. + for opt_log_guess in min_table_log..=max_table_log { + let Ok(max_bits) = build_ctable(table, count, max_symbol_value, opt_log_guess) else { + continue; + }; + if max_bits < opt_log_guess as usize && opt_log_guess > min_table_log { + break; + } + let Ok(h_size) = write_ctable(&mut dst, table, max_symbol_value, max_bits as u32) else { + continue; + }; + let new_size = estimate_compressed_size(table, count, max_symbol_value) + h_size; + if new_size > opt_size + 1 { + break; + } + if new_size < opt_size { + opt_size = new_size; + opt_log = opt_log_guess; + } + } + opt_log +} + +/// `HUF_compress_internal` +fn compress_internal( + dst: &mut [u8], + src: &[u8], + mut max_symbol_value: u32, + mut huff_log: u32, + nb_streams: NbStreams, + old_huf_table: &mut HufCTable, + repeat: &mut HufRepeat, + flags: HufFlags, +) -> Result { + let src_size = src.len(); + let Some(&first_byte) = src.first() else { + return Ok(0); + }; + if dst.is_empty() { + return Ok(0); + } + if src_size > HUF_BLOCKSIZE_MAX { + return Err(ErrorCode::SrcSizeWrong); + } + if huff_log > HUF_TABLELOG_MAX { + return Err(ErrorCode::TableLogTooLarge); + } + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return Err(ErrorCode::MaxSymbolValueTooLarge); + } + if max_symbol_value == 0 { + max_symbol_value = HUF_SYMBOLVALUE_MAX; + } + if huff_log == 0 { + huff_log = HUF_TABLELOG_DEFAULT; + } + + if flags.prefer_repeat && *repeat == HufRepeat::Valid { + return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); + } + + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + + // If the input may well be incompressible, look at its start and its end before all of it. + if flags.suspect_uncompressible + && src_size >= SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO + { + let mut largest_total = 0usize; + for sample in [ + src.sub(..SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE)?, + src.sub(src_size - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE..)?, + ] { + let mut max_symbol_value_sample = max_symbol_value; + largest_total += + hist::count_simple(&mut count, &mut max_symbol_value_sample, sample)? as usize; + } + if largest_total <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7) + 4 { + return Ok(0); + } + } + + let largest = hist::count_wksp(&mut count, &mut max_symbol_value, src)?; + if largest == src_size { + // A single symbol: run-length encoding. + dst.put(0, first_byte)?; + return Ok(1); + } + if largest <= (src_size >> 7) + 4 { + // Probably not compressible enough. + return Ok(0); + } + + if *repeat == HufRepeat::Check && !validate_ctable(old_huf_table, &count, max_symbol_value) { + *repeat = HufRepeat::None; + } + if flags.prefer_repeat && *repeat != HufRepeat::None { + return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); + } + + let mut ctable = HufCTable::new(); + huff_log = optimal_table_log( + huff_log, + src_size, + max_symbol_value, + &mut ctable, + &count, + flags, + ); + huff_log = build_ctable(&mut ctable, &count, max_symbol_value, huff_log)? as u32; + + let h_size = write_ctable(dst, &ctable, max_symbol_value, huff_log)?; + // Is the previous table the better choice, since it takes no header? + if *repeat != HufRepeat::None { + let old_size = estimate_compressed_size(old_huf_table, &count, max_symbol_value); + let new_size = estimate_compressed_size(&ctable, &count, max_symbol_value); + if old_size <= h_size + new_size || h_size + 12 >= src_size { + return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); + } + } + + if h_size + 12 >= src_size { + return Ok(0); + } + *repeat = HufRepeat::None; + old_huf_table.clone_from(&ctable); + compress_ctable_internal(dst, h_size, src, nb_streams, &ctable) +} + +/// `HUF_compress1X_repeat`: one stream, after the header of its table, or without one if `huf_table` is used again, +/// which `repeat` says is allowed. If it is not used again, `repeat` becomes [`HufRepeat::None`] and `huf_table` the +/// new table. Returns the size of the output, 0 if `src` is not compressible, or 1 if it is one byte repeated, which +/// is then the output. +pub(crate) fn compress_1x_repeat( + dst: &mut [u8], + src: &[u8], + max_symbol_value: u32, + huff_log: u32, + huf_table: &mut HufCTable, + repeat: &mut HufRepeat, + flags: HufFlags, +) -> Result { + compress_internal( + dst, + src, + max_symbol_value, + huff_log, + NbStreams::Single, + huf_table, + repeat, + flags, + ) +} + +/// `HUF_compress4X_repeat`: as [`compress_1x_repeat`], with four streams. +pub(crate) fn compress_4x_repeat( + dst: &mut [u8], + src: &[u8], + max_symbol_value: u32, + huff_log: u32, + huf_table: &mut HufCTable, + repeat: &mut HufRepeat, + flags: HufFlags, +) -> Result { + compress_internal( + dst, + src, + max_symbol_value, + huff_log, + NbStreams::Four, + huf_table, + repeat, + flags, + ) +} diff --git a/src/zstd/wip/work/entropy/zstd/compress/internal.rs b/src/zstd/wip/work/entropy/zstd/compress/internal.rs new file mode 100644 index 000000000000..51157ea6c1ce --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/compress/internal.rs @@ -0,0 +1,1212 @@ +//! `zstd_compress_internal.h`: what the parts of the compressor share. + +use super::fse::{FseCTable, FseRepeat}; +use super::huf::{HufCTable, HufRepeat}; +use crate::common::{ + BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, + ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; + +pub(crate) const K_SEARCH_STRENGTH: u32 = 8; +pub(crate) const HASH_READ_SIZE: usize = 8; +/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. +pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; + +pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; +pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; + +pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; +pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; + +/// `ZSTD_strategy` +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub(crate) enum Strategy { + Fast = 1, + DFast = 2, + Greedy = 3, + Lazy = 4, + Lazy2 = 5, + BtLazy2 = 6, + BtOpt = 7, + BtUltra = 8, + BtUltra2 = 9, +} + +impl Strategy { + pub(crate) const fn from_int(value: i32) -> Option { + Some(match value { + 1 => Self::Fast, + 2 => Self::DFast, + 3 => Self::Greedy, + 4 => Self::Lazy, + 5 => Self::Lazy2, + 6 => Self::BtLazy2, + 7 => Self::BtOpt, + 8 => Self::BtUltra, + 9 => Self::BtUltra2, + _ => return None, + }) + } +} + +/// `ZSTD_ParamSwitch_e` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum ParamSwitch { + Auto = 0, + Enable = 1, + Disable = 2, +} + +/// `ZSTD_compressionParameters` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct CompressionParameters { + pub(crate) window_log: u32, + pub(crate) chain_log: u32, + pub(crate) hash_log: u32, + pub(crate) search_log: u32, + pub(crate) min_match: u32, + pub(crate) target_length: u32, + /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. + pub(crate) strategy: Option, +} + +impl CompressionParameters { + /// The strategy of parameters that are ready to be applied. + #[inline(always)] + pub(crate) fn strat(&self) -> Strategy { + self.strategy.unwrap_or(Strategy::Fast) + } +} + +/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum DictMode { + NoDict, + ExtDict, + DictMatchState, +} + +/// `ZSTD_dictTableLoadMethod_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DictTableLoadMethod { + Fast, + Full, +} + +/// `ZSTD_tableFillPurpose_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TableFillPurpose { + ForCCtx, + ForCDict, +} + +// ── entropy tables ──────────────────────────────────────────────────────── + +/// `ZSTD_hufCTables_t` +#[derive(Clone)] +pub(crate) struct HufCTables { + pub(crate) c_table: HufCTable, + pub(crate) repeat_mode: HufRepeat, +} + +/// `ZSTD_fseCTables_t` +#[derive(Clone)] +pub(crate) struct FseCTables { + pub(crate) offcode_c_table: FseCTable, + pub(crate) matchlength_c_table: FseCTable, + pub(crate) litlength_c_table: FseCTable, + pub(crate) offcode_repeat_mode: FseRepeat, + pub(crate) matchlength_repeat_mode: FseRepeat, + pub(crate) litlength_repeat_mode: FseRepeat, +} + +/// `ZSTD_entropyCTables_t` +#[derive(Clone)] +pub(crate) struct EntropyCTables { + pub(crate) huf: HufCTables, + pub(crate) fse: FseCTables, +} + +/// `ZSTD_compressedBlockState_t` +#[derive(Clone)] +pub(crate) struct CompressedBlockState { + pub(crate) entropy: EntropyCTables, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +impl CompressedBlockState { + /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. + pub(crate) fn new() -> Self { + Self { + entropy: EntropyCTables { + huf: HufCTables { + c_table: HufCTable::new(), + repeat_mode: HufRepeat::None, + }, + fse: FseCTables { + offcode_c_table: FseCTable::new(), + matchlength_c_table: FseCTable::new(), + litlength_c_table: FseCTable::new(), + offcode_repeat_mode: FseRepeat::None, + matchlength_repeat_mode: FseRepeat::None, + litlength_repeat_mode: FseRepeat::None, + }, + }, + rep: REP_START_VALUE, + } + } + + /// `ZSTD_reset_compressedBlockState` + pub(crate) fn reset(&mut self) { + self.rep = REP_START_VALUE; + self.entropy.huf.repeat_mode = HufRepeat::None; + self.entropy.fse.offcode_repeat_mode = FseRepeat::None; + self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; + self.entropy.fse.litlength_repeat_mode = FseRepeat::None; + } +} + +/// `ZSTD_hufCTablesMetadata_t` +pub(crate) struct HufCTablesMetadata { + pub(crate) h_type: SymbolEncodingType, + pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], + pub(crate) huf_des_size: usize, +} + +/// `ZSTD_fseCTablesMetadata_t` +pub(crate) struct FseCTablesMetadata { + pub(crate) ll_type: SymbolEncodingType, + pub(crate) of_type: SymbolEncodingType, + pub(crate) ml_type: SymbolEncodingType, + pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], + pub(crate) fse_tables_size: usize, + /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. + pub(crate) last_count_size: usize, +} + +/// `ZSTD_entropyCTablesMetadata_t` +pub(crate) struct EntropyCTablesMetadata { + pub(crate) huf_metadata: HufCTablesMetadata, + pub(crate) fse_metadata: FseCTablesMetadata, +} + +impl EntropyCTablesMetadata { + pub(crate) const fn new() -> Self { + Self { + huf_metadata: HufCTablesMetadata { + h_type: SymbolEncodingType::Basic, + huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], + huf_des_size: 0, + }, + fse_metadata: FseCTablesMetadata { + ll_type: SymbolEncodingType::Basic, + of_type: SymbolEncodingType::Basic, + ml_type: SymbolEncodingType::Basic, + fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], + fse_tables_size: 0, + last_count_size: 0, + }, + } + } +} + +// ── sequences ───────────────────────────────────────────────────────────── + +/// `SeqDef` +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct SeqDef { + /// A repeat code (1 to 3), or an offset plus 3. + pub(crate) off_base: u32, + pub(crate) lit_length: u16, + /// The match length minus [`MINMATCH`]. + pub(crate) ml_base: u16, +} + +/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum LongLengthType { + None, + LiteralLength, + MatchLength, +} + +/// `SeqStore_t`: the sequences and the literals of a block. +pub(crate) struct SeqStore { + /// Never grows beyond the capacity it is given by [`Self::reserve`]. + pub(crate) sequences: Vec, + /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. + pub(crate) lit_buffer: Vec, + /// How much of `lit_buffer` is literals. + pub(crate) lit_len: usize, + pub(crate) ll_code: Vec, + pub(crate) ml_code: Vec, + pub(crate) of_code: Vec, + pub(crate) max_nb_seq: usize, + pub(crate) max_nb_lit: usize, + pub(crate) long_length_type: LongLengthType, + /// Which sequence has the long length. + pub(crate) long_length_pos: u32, +} + +/// `ZSTD_SequenceLength` +#[derive(Clone, Copy)] +pub(crate) struct SequenceLength { + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +impl SeqStore { + pub(crate) const fn new() -> Self { + Self { + sequences: Vec::new(), + lit_buffer: Vec::new(), + lit_len: 0, + ll_code: Vec::new(), + ml_code: Vec::new(), + of_code: Vec::new(), + max_nb_seq: 0, + max_nb_lit: 0, + long_length_type: LongLengthType::None, + long_length_pos: 0, + } + } + + /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. + pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { + fn grow(vec: &mut Vec, len: usize) -> Result<()> { + vec.clear(); + vec.try_reserve_exact(len) + .map_err(|_| ErrorCode::MemoryAllocation)?; + vec.resize(len, T::default()); + Ok(()) + } + grow(&mut self.sequences, max_nb_seq)?; + grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; + grow(&mut self.ll_code, max_nb_seq)?; + grow(&mut self.ml_code, max_nb_seq)?; + grow(&mut self.of_code, max_nb_seq)?; + self.max_nb_seq = max_nb_seq; + self.max_nb_lit = max_nb_lit; + self.reset(); + Ok(()) + } + + /// `ZSTD_resetSeqStore` + #[inline] + pub(crate) fn reset(&mut self) { + self.lit_len = 0; + self.sequences.clear(); + self.long_length_type = LongLengthType::None; + } + + #[inline] + pub(crate) fn literals(&self) -> &[u8] { + self.lit_buffer.get(..self.lit_len).unwrap_or_default() + } + + /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. + #[inline(always)] + pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { + debug_assert!(self.sequences.len() < self.max_nb_seq); + if lit_length > 0xFFFF { + self.long_length_type = LongLengthType::LiteralLength; + self.long_length_pos = self.sequences.len() as u32; + } + let ml_base = match_length.wrapping_sub(MINMATCH as usize); + if ml_base > 0xFFFF { + self.long_length_type = LongLengthType::MatchLength; + self.long_length_pos = self.sequences.len() as u32; + } + self.sequences.push(SeqDef { + off_base, + lit_length: lit_length as u16, + ml_base: ml_base as u16, + }); + } + + /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. + #[inline(always)] + pub(crate) fn store_seq( + &mut self, + lit_length: usize, + buf: &[u8], + literals: usize, + off_base: u32, + match_length: usize, + ) -> Result<()> { + match ( + buf.array_at::<16>(literals), + self.lit_buffer.array_at_mut::<16>(self.lit_len), + ) { + (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, + _ => self + .lit_buffer + .write_at(self.lit_len, buf.span(literals, lit_length)?)?, + } + self.lit_len += lit_length; + self.store_seq_only(lit_length, off_base, match_length); + Ok(()) + } + + /// Appends the literals that follow the last sequence of a block. + #[inline] + pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { + self.lit_buffer.write_at(self.lit_len, literals)?; + self.lit_len += literals.len(); + Ok(()) + } +} + +pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; +pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; +pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; + +/// `OFFSET_TO_OFFBASE`: `offset` is not 0. +#[inline(always)] +pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { + offset.wrapping_add(ZSTD_REP_NUM as u32) +} + +/// `OFFBASE_IS_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { + off_base > ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_IS_REPCODE` +#[inline(always)] +pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { + 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_TO_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { + off_base.wrapping_sub(ZSTD_REP_NUM as u32) +} + +/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. +#[inline(always)] +pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { + let [rep0, rep1, rep2] = *rep; + if offbase_is_offset(off_base) { + *rep = [offbase_to_offset(off_base), rep0, rep1]; + } else { + match off_base - 1 + u32::from(ll0) { + 0 => {} + 1 => *rep = [rep1, rep0, rep2], + 2 => *rep = [rep2, rep0, rep1], + _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], + } + } +} + +/// `ZSTD_newRep` +#[inline(always)] +pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { + let mut new_reps = *rep; + update_rep(&mut new_reps, off_base, ll0); + new_reps +} + +/// `ZSTD_LLcode` +#[inline(always)] +pub(crate) fn ll_code(lit_length: u32) -> u32 { + #[rustfmt::skip] + const LL_CODE: [u8; 64] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + ]; + const LL_DELTA_CODE: u32 = 19; + match LL_CODE.get(lit_length as usize) { + Some(&code) => u32::from(code), + None => highbit32(lit_length) + LL_DELTA_CODE, + } +} + +/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. +#[inline(always)] +pub(crate) fn ml_code(ml_base: u32) -> u32 { + #[rustfmt::skip] + const ML_CODE: [u8; 128] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ]; + const ML_DELTA_CODE: u32 = 36; + match ML_CODE.get(ml_base as usize) { + Some(&code) => u32::from(code), + None => highbit32(ml_base) + ML_DELTA_CODE, + } +} + +/// `ZSTD_noCompressBlock` +pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { + let c_block_header24 = + u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); + if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header24)?; + dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; + Ok(ZSTD_BLOCKHEADERSIZE + src.len()) +} + +/// `ZSTD_rleCompressBlock` +pub(crate) fn rle_compress_block( + dst: &mut [u8], + byte: u8, + src_size: usize, + last_block: bool, +) -> Result { + let c_block_header = + u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); + if dst.len() < 4 { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header)?; + dst.put(3, byte)?; + Ok(4) +} + +/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. +#[inline] +pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { + let minlog = if strat >= Strategy::BtUltra { + strat as u32 - 1 + } else { + 6 + }; + (src_size >> minlog) + 2 +} + +// ── matching ────────────────────────────────────────────────────────────── + +/// How many bytes `a` and `b` have in common at their start. +#[inline(always)] +pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { + let (a_words, _) = a.as_chunks::<8>(); + let (b_words, _) = b.as_chunks::<8>(); + let mut len = 0usize; + for (a_word, b_word) in a_words.iter().zip(b_words) { + let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); + if diff != 0 { + return len + (diff.trailing_zeros() >> 3) as usize; + } + len += 8; + } + let a_tail = a.get(len..).unwrap_or_default(); + let b_tail = b.get(len..).unwrap_or_default(); + len + a_tail + .iter() + .zip(b_tail) + .take_while(|(a_byte, b_byte)| a_byte == b_byte) + .count() +} + +/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. +#[inline(always)] +pub(crate) fn count( + buf: &[u8], + p_in: usize, + match_buf: &[u8], + p_match: usize, + p_in_limit: usize, +) -> Result { + Ok(common_prefix( + buf.sub(p_in..p_in_limit)?, + match_buf.sub(p_match..)?, + )) +} + +/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at +/// `i_start` in `buf`. +#[inline(always)] +pub(crate) fn count_2segments( + buf: &[u8], + ip: usize, + match_buf: &[u8], + match_pos: usize, + i_end: usize, + m_end: usize, + i_start: usize, +) -> Result { + let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); + if match_pos + match_length != m_end { + return Ok(match_length); + } + Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) +} + +const PRIME_3_BYTES: u32 = 506_832_829; +const PRIME_4_BYTES: u32 = 2_654_435_761; +const PRIME_5_BYTES: u64 = 889_523_592_379; +const PRIME_6_BYTES: u64 = 227_718_039_650_203; +const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; +pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; + +/// `ZSTD_hash3` +#[inline(always)] +const fn hash3(u: u32, h: u32, s: u32) -> u32 { + (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash4(u: u32, h: u32, s: u32) -> u32 { + ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash5(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash6(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash7(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +/// `ZSTD_hash8` +#[inline(always)] +pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { + (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} + +/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. +#[inline(always)] +pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { + Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) +} + +/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, +/// else 8. +#[inline(always)] +pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { + hash_ptr_salted(buf, pos, h_bits, mls, 0) +} + +/// `ZSTD_hashPtrSalted` +#[inline(always)] +pub(crate) fn hash_ptr_salted( + buf: &[u8], + pos: usize, + h_bits: u32, + mls: u32, + hash_salt: u64, +) -> Result { + Ok(match mls { + 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), + 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), + 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), + 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), + _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, + }) +} + +// ── the window ──────────────────────────────────────────────────────────── + +/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are +/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the +/// others, and an address is turned into a position in the buffer only to read. +pub(crate) type Addr = usize; + +/// The buffers that a window can have a segment in. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum BufId { + /// Where an empty window points. + None = 0, + /// The context's own buffer of input. + InBuff = 1, + /// What the caller gave to the call in progress. + User = 2, + /// The content of the dictionary, or the prefix. + Dict = 3, +} + +impl BufId { + /// The address of the first byte of the buffer. + #[inline(always)] + pub(crate) const fn start(self) -> Addr { + (self as usize + 1) << 44 + } +} + +/// The buffers themselves, for as long as a call lasts. +#[derive(Clone, Copy)] +pub(crate) struct Buffers<'a> { + pub(crate) in_buff: &'a [u8], + pub(crate) user: &'a [u8], + pub(crate) dict: &'a [u8], +} + +impl<'a> Buffers<'a> { + #[inline] + pub(crate) fn get(&self, id: BufId) -> &'a [u8] { + match id { + BufId::None => &[], + BufId::InBuff => self.in_buff, + BufId::User => self.user, + BufId::Dict => self.dict, + } + } +} + +/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. +pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; + +/// The index past which indices are brought down. +pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; +/// The most input that can be indexed at once. +pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; + +/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the +/// index is less than `dict_limit`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Window { + /// Where input has to continue to be contiguous. + pub(crate) next_src: Addr, + pub(crate) base: Addr, + pub(crate) dict_base: Addr, + /// The buffer of the segment that starts at `dict_limit`. + pub(crate) base_buf: BufId, + /// The buffer of the segment from `low_limit` to `dict_limit`. + pub(crate) dict_buf: BufId, + pub(crate) dict_limit: u32, + /// Below this, there is nothing valid. + pub(crate) low_limit: u32, + pub(crate) nb_overflow_corrections: u32, +} + +impl Window { + /// `ZSTD_window_init` + pub(crate) const fn new() -> Self { + let base = BufId::None.start(); + Self { + next_src: base + ZSTD_WINDOW_START_INDEX as usize, + base, + dict_base: base, + base_buf: BufId::None, + dict_buf: BufId::None, + dict_limit: ZSTD_WINDOW_START_INDEX, + low_limit: ZSTD_WINDOW_START_INDEX, + nb_overflow_corrections: 0, + } + } + + /// The index of `addr`, which is in the current segment or at its end. + #[inline(always)] + pub(crate) const fn index_of(&self, addr: Addr) -> u32 { + addr.wrapping_sub(self.base) as u32 + } + + /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. + pub(crate) fn clear(&mut self) { + let end = self.index_of(self.next_src); + self.low_limit = end; + self.dict_limit = end; + } + + /// `ZSTD_window_isEmpty` + pub(crate) fn is_empty(&self) -> bool { + self.dict_limit == ZSTD_WINDOW_START_INDEX + && self.low_limit == ZSTD_WINDOW_START_INDEX + && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize + } + + /// `ZSTD_window_hasExtDict` + #[inline] + pub(crate) fn has_ext_dict(&self) -> bool { + self.low_limit < self.dict_limit + } + + /// `ZSTD_window_needOverflowCorrection` + #[inline] + pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { + self.index_of(src_end) > ZSTD_CURRENT_MAX + } + + /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns + /// what to subtract from every index that is stored. + pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { + let cycle_size = 1u32 << cycle_log; + let cycle_mask = cycle_size - 1; + let curr = self.index_of(src); + let current_cycle = curr & cycle_mask; + // Indices must not fall below the start index. + let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { + cycle_size.max(ZSTD_WINDOW_START_INDEX) + } else { + 0 + }; + let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); + let correction = curr.wrapping_sub(new_current); + + self.base = self.base.wrapping_add(correction as usize); + self.dict_base = self.dict_base.wrapping_add(correction as usize); + self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.low_limit - correction + }; + self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.dict_limit - correction + }; + self.nb_overflow_corrections += 1; + correction + } + + /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the + /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. + pub(crate) fn enforce_max_dist( + &mut self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { + let new_low_limit = block_end_idx - max_dist; + if self.low_limit < new_low_limit { + self.low_limit = new_low_limit; + } + if self.dict_limit < self.low_limit { + self.dict_limit = self.low_limit; + } + return true; + } + false + } + + /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not + /// right before the current segment, and is to be forgotten. + pub(crate) fn dict_is_invalid( + &self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit + } + + /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not + /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. + pub(crate) fn update( + &mut self, + src: Addr, + src_buf: BufId, + src_size: usize, + force_non_contiguous: bool, + ) -> bool { + let mut contiguous = true; + if src_size == 0 { + return contiguous; + } + if src != self.next_src || force_non_contiguous { + let distance_from_base = self.next_src.wrapping_sub(self.base); + self.low_limit = self.dict_limit; + self.dict_limit = distance_from_base as u32; + self.dict_base = self.base; + self.dict_buf = self.base_buf; + self.base = src.wrapping_sub(distance_from_base); + self.base_buf = src_buf; + if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { + // Too small to be of use. + self.low_limit = self.dict_limit; + } + contiguous = false; + } + self.next_src = src + src_size; + // Where the input overlaps the older segment, it has overwritten it. + if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) + & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) + { + let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); + self.low_limit = if high_input_idx > self.dict_limit as usize { + self.dict_limit + } else { + high_input_idx as u32 + }; + } + contiguous + } +} + +/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes +/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. +#[inline(always)] +pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { + prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 +} + +pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; +pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; + +/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. +#[inline(always)] +pub(crate) fn write_tagged_index( + hash_table: &mut [u32], + hash_and_tag: usize, + index: u32, +) -> Result<()> { + let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; + hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) +} + +/// `ZSTD_comparePackedTags` +#[inline(always)] +pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { + (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) + == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) +} + +// ── the state of the match finders ──────────────────────────────────────── + +/// `ZSTD_match_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Match { + /// As [`SeqDef::off_base`]. + pub(crate) off: u32, + pub(crate) len: u32, +} + +/// `rawSeq`: a sequence found by long distance matching. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct RawSeq { + pub(crate) offset: u32, + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +/// `RawSeqStore_t` +#[derive(Clone, Default)] +pub(crate) struct RawSeqStore { + /// `capacity` of them, of which `size` are in use. + pub(crate) seq: Vec, + /// The sequence that is being read. + pub(crate) pos: usize, + /// How many of its bytes, literals first, are read. + pub(crate) pos_in_sequence: usize, + pub(crate) size: usize, + pub(crate) capacity: usize, +} + +/// `ZSTD_optimal_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Optimal { + /// The cost of getting to this position. + pub(crate) price: i32, + pub(crate) off: u32, + pub(crate) mlen: u32, + pub(crate) litlen: u32, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +/// `ZSTD_OptPrice_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum OptPrice { + Dynamic, + Predef, +} + +/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. +pub(crate) struct OptState { + pub(crate) lit_freq: Vec, + pub(crate) lit_length_freq: Vec, + pub(crate) match_length_freq: Vec, + pub(crate) off_code_freq: Vec, + pub(crate) match_table: Vec, + pub(crate) price_table: Vec, + pub(crate) lit_sum: u32, + pub(crate) lit_length_sum: u32, + pub(crate) match_length_sum: u32, + pub(crate) off_code_sum: u32, + pub(crate) lit_sum_base_price: u32, + pub(crate) lit_length_sum_base_price: u32, + pub(crate) match_length_sum_base_price: u32, + pub(crate) off_code_sum_base_price: u32, + pub(crate) price_type: OptPrice, + pub(crate) literal_compression_mode: ParamSwitch, +} + +impl OptState { + pub(crate) const fn new() -> Self { + Self { + lit_freq: Vec::new(), + lit_length_freq: Vec::new(), + match_length_freq: Vec::new(), + off_code_freq: Vec::new(), + match_table: Vec::new(), + price_table: Vec::new(), + lit_sum: 0, + lit_length_sum: 0, + match_length_sum: 0, + off_code_sum: 0, + lit_sum_base_price: 0, + lit_length_sum_base_price: 0, + match_length_sum_base_price: 0, + off_code_sum_base_price: 0, + price_type: OptPrice::Dynamic, + literal_compression_mode: ParamSwitch::Auto, + } + } +} + +pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; + +/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures +/// own, are in [`BlockSrc`]. +pub(crate) struct MatchState { + pub(crate) window: Window, + /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. + pub(crate) loaded_dict_end: u32, + /// The index from which the tables are still to be filled. + pub(crate) next_to_update: u32, + /// 0 if there is no table of 3-byte hashes. + pub(crate) hash_log3: u32, + pub(crate) row_hash_log: u32, + /// For the row-based match finder: a byte of hash for each entry of `hash_table`. + pub(crate) tag_table: Vec, + pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], + /// Makes what a previous frame left in `tag_table` meaningless. + pub(crate) hash_salt: u64, + pub(crate) hash_salt_entropy: u32, + pub(crate) hash_table: Vec, + pub(crate) hash_table3: Vec, + pub(crate) chain_table: Vec, + pub(crate) force_non_contiguous: bool, + pub(crate) opt: OptState, + /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. + pub(crate) has_dict_match_state: bool, + pub(crate) c_params: CompressionParameters, + pub(crate) lazy_skipping: bool, +} + +impl MatchState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + loaded_dict_end: 0, + next_to_update: 0, + hash_log3: 0, + row_hash_log: 0, + tag_table: Vec::new(), + hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + hash_salt: 0, + hash_salt_entropy: 0, + hash_table: Vec::new(), + hash_table3: Vec::new(), + chain_table: Vec::new(), + force_non_contiguous: false, + opt: OptState::new(), + has_dict_match_state: false, + c_params: CompressionParameters { + window_log: 0, + chain_log: 0, + hash_log: 0, + search_log: 0, + min_match: 0, + target_length: 0, + strategy: None, + }, + lazy_skipping: false, + } + } + + /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. + pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { + if self.window.has_ext_dict() { + return DictMode::ExtDict; + } + if dms.is_some() { + DictMode::DictMatchState + } else { + DictMode::NoDict + } + } + + /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. + #[inline(always)] + pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.low_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + // A dictionary is valid as a whole until it is invalidated as a whole. + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } + + /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. + #[inline(always)] + pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.dict_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } +} + +/// The match state of a dictionary, and where its indices lead. +#[derive(Clone, Copy)] +pub(crate) struct DictMatchState<'a> { + pub(crate) ms: &'a MatchState, + /// The buffer its window is in, which is one segment. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// `window.nextSrc`, as a position in `buf`. + pub(crate) end: usize, +} + +/// The current segment of a window, for the functions that index a dictionary. +#[derive(Clone, Copy)] +pub(crate) struct Prefix<'a> { + /// The buffer that the segment is in. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, +} + +impl Prefix<'_> { + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. +#[derive(Clone, Copy)] +pub(crate) struct BlockSrc<'a> { + /// The buffer that the current segment of the window is in, and so the block. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// The buffer that the older segment of the window is in. + pub(crate) dict_buf: &'a [u8], + /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. + pub(crate) dict_base: usize, + /// Where the block starts in `buf`. + pub(crate) start: usize, + /// Where it ends. + pub(crate) end: usize, + pub(crate) dms: Option>, + /// `opt.symbolCosts`: the tables of the previous block. + pub(crate) symbol_costs: &'a EntropyCTables, + /// `ldmSeqStore`: matches that the optimal parser is to consider. + pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, +} + +impl<'a> BlockSrc<'a> { + #[inline(always)] + pub(crate) const fn prefix(&self) -> Prefix<'a> { + Prefix { + buf: self.buf, + base: self.base, + } + } + + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`, which is in the current segment. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } + + /// The position in `dict_buf` of `index`, which is in the older segment. + #[inline(always)] + pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { + self.dict_base.wrapping_add(index as usize) + } +} + +impl DictMatchState<'_> { + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +// ── long distance matching ──────────────────────────────────────────────── + +/// `ldmEntry_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct LdmEntry { + pub(crate) offset: u32, + pub(crate) checksum: u32, +} + +/// `ldmParams_t` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct LdmParams { + pub(crate) enable_ldm: ParamSwitch, + pub(crate) hash_log: u32, + pub(crate) bucket_size_log: u32, + pub(crate) min_match_length: u32, + pub(crate) hash_rate_log: u32, + pub(crate) window_log: u32, +} + +/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. +pub(crate) struct LdmState { + pub(crate) window: Window, + pub(crate) hash_table: Vec, + pub(crate) loaded_dict_end: u32, + /// The next entry to replace in each bucket. + pub(crate) bucket_offsets: Vec, +} + +impl LdmState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + hash_table: Vec::new(), + loaded_dict_end: 0, + bucket_offsets: Vec::new(), + } + } +} diff --git a/src/zstd/wip/work/entropy/zstd/compress/lazy.rs b/src/zstd/wip/work/entropy/zstd/compress/lazy.rs new file mode 100644 index 000000000000..9f653fd684f3 --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/compress/lazy.rs @@ -0,0 +1,34 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _ip: usize, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_row_update`: the same, in the rows of the row-based match finder. +pub(crate) fn row_update(_ms: &mut MatchState, _prefix: Prefix<'_>, _ip: usize) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, + _strategy: Strategy, + _use_row: bool, + _dict_mode: DictMode, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/entropy/zstd/compress/ldm.rs b/src/zstd/wip/work/entropy/zstd/compress/ldm.rs new file mode 100644 index 000000000000..2177fd945e10 --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/compress/ldm.rs @@ -0,0 +1 @@ +//! `zstd_ldm.c`: long distance matching. diff --git a/src/zstd/wip/work/entropy/zstd/compress/mod.rs b/src/zstd/wip/work/entropy/zstd/compress/mod.rs new file mode 100644 index 000000000000..13ddecdf82e4 --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/compress/mod.rs @@ -0,0 +1,68 @@ +//! `lib/compress` + +pub(crate) mod double_fast; +pub(crate) mod fast; +pub(crate) mod fse; +pub(crate) mod hist; +pub(crate) mod huf; +pub(crate) mod internal; +pub(crate) mod lazy; +pub(crate) mod ldm; +pub(crate) mod opt; +pub(crate) mod presplit; + +use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; + +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; + +/// `ZSTD_rowMatchFinderSupported` +pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { + strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 +} + +/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to +/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies +/// that support it look at. Returns how many bytes at the end of the block are in no sequence. +pub(crate) fn compress_block( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row_match_finder: bool, + dict_mode: DictMode, +) -> Result { + match strategy { + Strategy::Fast => match dict_mode { + DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), + DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), + DictMode::DictMatchState => { + fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::DFast => match dict_mode { + DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), + DictMode::ExtDict => { + double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { + lazy::compress_block_lazy( + ms, + seq_store, + rep, + src, + strategy, + use_row_match_finder && row_match_finder_supported(strategy), + dict_mode, + ) + } + Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { + opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) + } + } +} diff --git a/src/zstd/wip/work/entropy/zstd/compress/opt.rs b/src/zstd/wip/work/entropy/zstd/compress/opt.rs new file mode 100644 index 000000000000..d055a44cf982 --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/compress/opt.rs @@ -0,0 +1,29 @@ +//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. + +use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. +/// Matches are not compared past the position `iend`. +pub(crate) fn update_tree( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _ip: usize, + _iend: usize, +) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` +/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. +pub(crate) fn compress_block_opt( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, + _strategy: Strategy, + _dict_mode: DictMode, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/entropy/zstd/compress/presplit.rs b/src/zstd/wip/work/entropy/zstd/compress/presplit.rs new file mode 100644 index 000000000000..8b778e2989f9 --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/compress/presplit.rs @@ -0,0 +1 @@ +//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. diff --git a/src/zstd/wip/work/entropy/zstd/lib.rs b/src/zstd/wip/work/entropy/zstd/lib.rs new file mode 100644 index 000000000000..31c98c72e5e4 --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/lib.rs @@ -0,0 +1,385 @@ +//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the +//! same bytes in, in both directions, and the same errors. +#![forbid(unsafe_code)] +// Frames come from anywhere and a panic aborts the process. +#![deny( + clippy::indexing_slicing, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unreachable +)] +#![warn(unused_must_use)] + +mod common; +mod compress; +mod decompress; +mod error; +mod util; + +pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; +pub use decompress::{ + DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, + find_frame_compressed_size, get_frame_content_size, +}; +pub use error::ErrorCode; + +/// `ZSTD_inBuffer` +pub struct InBuffer<'a> { + pub src: &'a [u8], + /// How much of `src` is read. + pub pos: usize, +} + +/// `ZSTD_outBuffer` +pub struct OutBuffer<'a> { + pub dst: &'a mut [u8], + /// How much of `dst` is written. + pub pos: usize, +} + +/// `ZSTD_ResetDirective` +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ResetDirective { + SessionOnly = 1, + Parameters = 2, + SessionAndParameters = 3, +} + +// ----------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] +pub enum ZstdError { + /// The output, or the decoder state the frame's window size dictates, could not be allocated. + OutOfMemory, + InvalidZstdData, + DecompressionFailed, + ZstdDecompressionError, + ShortRead, +} + +impl core::fmt::Display for ZstdError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(<&'static str>::from(self)) + } +} +impl core::error::Error for ZstdError {} + +impl ZstdError { + /// The error for a failed decompression call; `other` is the non-allocation failure. + fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { + if code == ErrorCode::MemoryAllocation { + ZstdError::OutOfMemory + } else { + other + } + } +} + +/// `len` zeros for a decoder to write to. +fn output_buffer(len: usize) -> Result, ZstdError> { + util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) +} + +/// Decompress data, automatically allocating the output buffer. +/// Handles both frames with known and unknown content sizes. +/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. +/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. +pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { + let size = find_decompressed_size(src); + + const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit + + if size == ZSTD_CONTENTSIZE_ERROR { + return Err(ZstdError::InvalidZstdData); + } + + // Use streaming decompression if: + // 1. Content size is unknown, OR + // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) + if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { + let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { + // A frame's output is rarely smaller than its input. + src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) + } else { + // The header size is untrusted: reserve no more than the fast path below would. + MAX_PREALLOCATE_SIZE + }; + let mut list: Vec = Vec::new(); + list.try_reserve_exact(initial_capacity) + .map_err(|_| ZstdError::OutOfMemory)?; + StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; + return Ok(list); + } + + // Fast path: size is known and within reasonable limits + let mut output = output_buffer(size as usize)?; + let written = decompress(&mut output, src) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is +/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. +pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { + const MAX_SIZE: u64 = 16 * 1024 * 1024; + let size = find_decompressed_size(src); + // Also if it is unknown or invalid, which are the two largest values. + if size > MAX_SIZE { + return Err(ZstdError::InvalidZstdData); + } + let mut output = output_buffer(size as usize)?; + let written = DDict::prefix(prefix) + .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + Uninitialized, + Inflating, + End, + Error, +} + +/// Minimum spare output capacity offered to the decoder per call. +const STREAMING_OUTPUT_STEP: usize = 4096; + +// ────────────────────────────────────────────────────────────────────────── +// StreamingDecoder +// ────────────────────────────────────────────────────────────────────────── + +/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) +/// per call, so callers can hold the decoder across multiple body chunks. +pub struct StreamingDecoder { + stream: DCtx, + state: State, + /// The decoder filled its last window and may hold more. `max_output` can end a call there. + output_full: bool, +} + +impl Default for StreamingDecoder { + fn default() -> Self { + Self::new() + } +} + +impl StreamingDecoder { + pub fn new() -> Self { + Self { + stream: DCtx::new(), + state: State::Uninitialized, + output_full: false, + } + } + + #[inline] + pub fn is_inflating(&self) -> bool { + matches!(self.state, State::Inflating) + } + + /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is + /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. + /// Returns `ShortRead` when more input is required and `is_done` is false. + pub fn decompress( + &mut self, + input: &[u8], + out: &mut Vec, + max_output: usize, + is_done: bool, + ) -> Result { + if matches!(self.state, State::End | State::Error) { + return Ok(input.len()); + } + + let mut in_buf = InBuffer { src: input, pos: 0 }; + while matches!(self.state, State::Uninitialized | State::Inflating) { + let next_in_len = input.len() - in_buf.pos; + + // Call the decoder again with no input until it leaves the window short. + if next_in_len == 0 && !self.output_full { + if is_done { + if self.state == State::Inflating { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + self.state = State::End; + } + return Ok(in_buf.pos); + } + + if out.len() >= max_output { + return Ok(in_buf.pos); + } + + if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { + self.state = State::Error; + return Err(ZstdError::OutOfMemory); + } + // The decoder writes to initialized memory only. + let written_before = out.len(); + let window = (out.capacity() - written_before).min(max_output - written_before); + out.resize(written_before + window, 0); + let mut out_buf = OutBuffer { + dst: out.get_mut(written_before..).unwrap_or_default(), + pos: 0, + }; + let read_before = in_buf.pos; + + let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); + let bytes_written = out_buf.pos; + out.truncate(written_before + bytes_written); + let remaining = match result { + Ok(remaining) => remaining, + Err(code) => { + self.state = State::Error; + return Err(ZstdError::for_decompression( + code, + ZstdError::ZstdDecompressionError, + )); + } + }; + + self.output_full = bytes_written == window; + + if remaining == 0 { + // Frame complete, and fully flushed. + self.state = State::Uninitialized; + self.output_full = false; + if in_buf.pos >= input.len() { + if is_done { + self.state = State::End; + } + return Ok(in_buf.pos); + } + // More input available — reinitialize for the next frame. + self.stream.init_stream(); + continue; + } + + self.state = State::Inflating; + + if in_buf.pos - read_before == next_in_len { + if self.output_full { + continue; + } + if is_done { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + return Err(ZstdError::ShortRead); + } + } + Ok(in_buf.pos) + } +} + +// ── compressed embedded assets ──────────────────────────────────────────── + +/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes +/// Bun never executes or parses itself: the shell completion scripts and the +/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, +/// error overlay/page). Anything that runs inside Bun stays uncompressed. +/// +/// The exception is text that only `bun lint`, `bun fmt` and `bun check` +/// read. It may be compressed because it isn't on the path of running a program. +/// +/// Release (`bun_codegen_embed`) builds include the `.zst` twin that +/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to +/// `/compressed/.zst`; other builds evaluate `$fallback`. +#[macro_export] +macro_rules! embed_compressed { + // A codegen output (`/$sub`); debug builds read it at runtime. + (codegen $sub:literal) => { + $crate::embed_compressed!( + ("codegen/", $sub), + ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() + ) + }; + // Same, with a trailing NUL included in the slice (for C-string consumers). + (codegen_nul $sub:literal) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/codegen/", + $sub, + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); + &__COPY.get_or_init(|| { + let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); + let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); + copy.extend_from_slice(text); + copy.push(0); + copy.into_boxed_slice() + })[..] + } + }; + __bytes + }}; + // A file under `src/`; debug builds read it from the source tree at runtime. + (src $sub:literal) => { + $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) + }; + ($name:literal, $fallback:expr) => { + $crate::embed_compressed!(($name), $fallback) + }; + (($($name:literal),+), $fallback:expr) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/", + $($name,)+ + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + $fallback + } + }; + __bytes + }}; +} + +/// Cold, shared body of [`embed_compressed!`]'s release arm. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { + decompress_alloc(compressed) + .expect("embedded asset: invalid zstd frame") + .into_boxed_slice() +} + +/// [`inflate_embedded`] plus a trailing NUL byte. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { + let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); + inflated.reserve_exact(1); + inflated.push(0); + inflated.into_boxed_slice() +} +pub mod testing; diff --git a/src/zstd/wip/work/entropy/zstd/testing.rs b/src/zstd/wip/work/entropy/zstd/testing.rs new file mode 100644 index 000000000000..dd4e06c95a2d --- /dev/null +++ b/src/zstd/wip/work/entropy/zstd/testing.rs @@ -0,0 +1,457 @@ +//! Hooks for the differential tests. Only in scratch copies of the crate. +#![allow(clippy::all, clippy::indexing_slicing, dead_code, unreachable_pub)] + +use crate::ErrorCode; +use crate::compress::internal::*; +use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; + +fn bitmix(mut val: u64, len: u64) -> u64 { + val ^= val.rotate_right(49) ^ val.rotate_right(24); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^= (val >> 35).wrapping_add(len); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^ (val >> 28) +} + +/// The sequences of the first block of a frame, as (literal length, match length, offset), and then the literals +/// that are left as (length, 0, 0). +/// +/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, already adjusted to the sizes. +/// `dict` is a prefix (raw content, as `ZSTD_CCtx_refPrefix`), or empty. `src` is from 7 to 131071 bytes. +pub fn first_block( + cp: [u32; 7], + use_row: bool, + dict: &[u8], + src: &[u8], +) -> Result, ErrorCode> { + let strategy = Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?; + let c_params = CompressionParameters { + window_log: cp[0], + chain_log: cp[1], + hash_log: cp[2], + search_log: cp[3], + min_match: cp[4], + target_length: cp[5], + strategy: Some(strategy), + }; + let row_used = row_match_finder_supported(strategy) && use_row; + + // ZSTD_reset_matchState + let mut ms = MatchState::new(); + ms.hash_log3 = if c_params.min_match == 3 { + c_params.window_log.min(17) + } else { + 0 + }; + ms.window.clear(); + ms.next_to_update = ms.window.dict_limit; + ms.hash_table = vec![0; 1 << c_params.hash_log]; + ms.chain_table = vec![ + 0; + if strategy != Strategy::Fast && !row_used { + 1 << c_params.chain_log + } else { + 0 + } + ]; + ms.hash_table3 = vec![ + 0; + if ms.hash_log3 != 0 { + 1 << ms.hash_log3 + } else { + 0 + } + ]; + if row_used { + ms.tag_table = vec![0; 1 << c_params.hash_log]; + ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4); + ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); + } + if strategy >= Strategy::BtOpt { + ms.opt.lit_freq = vec![0; 256]; + ms.opt.lit_length_freq = vec![0; 36]; + ms.opt.match_length_freq = vec![0; 53]; + ms.opt.off_code_freq = vec![0; 32]; + ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; + ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; + } + ms.c_params = c_params; + + let block_size = (1usize << c_params.window_log).min(128 << 10); + let mut seq_store = SeqStore::new(); + seq_store.reserve( + block_size / if c_params.min_match == 3 { 3 } else { 4 }, + block_size, + )?; + + let bufs = Buffers { + in_buff: &[], + user: src, + dict, + }; + + // ZSTD_loadDictionaryContent(dtlm_fast, tfp_forCCtx) + if !dict.is_empty() { + let iend = dict.len(); + ms.window + .update(BufId::Dict.start(), BufId::Dict, dict.len(), false); + let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); + let ip = iend - dict.len().min(max_dict_size); + let prefix = Prefix { + buf: dict, + base: ms.window.base.wrapping_sub(BufId::Dict.start()), + }; + ms.next_to_update = prefix.index_of(ip); + ms.loaded_dict_end = prefix.index_of(iend); + if iend - ip > HASH_READ_SIZE { + match strategy { + Strategy::Fast => fast::fill_hash_table( + &mut ms, + prefix, + iend, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + )?, + Strategy::DFast => double_fast::fill_double_hash_table( + &mut ms, + prefix, + iend, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + )?, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { + if use_row { + ms.tag_table.fill(0); + lazy::row_update(&mut ms, prefix, iend - HASH_READ_SIZE)?; + } else { + lazy::insert_and_find_first_index(&mut ms, prefix, iend - HASH_READ_SIZE)?; + } + } + _ => opt::update_tree(&mut ms, prefix, iend - HASH_READ_SIZE, iend)?, + } + ms.next_to_update = prefix.index_of(iend); + } + } + + // ZSTD_compressContinue_internal + let src_addr = BufId::User.start(); + if !ms + .window + .update(src_addr, BufId::User, src.len(), ms.force_non_contiguous) + { + ms.force_non_contiguous = false; + ms.next_to_update = ms.window.dict_limit; + } + // ZSTD_compress_frameChunk + let max_dist = 1u32 << c_params.window_log; + if ms + .window + .dict_is_invalid(src_addr + src.len(), max_dist, ms.loaded_dict_end) + { + ms.loaded_dict_end = 0; + } + if ms + .window + .enforce_max_dist(src_addr, max_dist, ms.loaded_dict_end) + { + ms.loaded_dict_end = 0; + } + if ms.next_to_update < ms.window.low_limit { + ms.next_to_update = ms.window.low_limit; + } + // ZSTD_buildSeqStore + let prev = CompressedBlockState::new(); + let block = BlockSrc { + buf: bufs.get(ms.window.base_buf), + base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), + dict_buf: bufs.get(ms.window.dict_buf), + dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), + start: 0, + end: src.len(), + dms: None, + symbol_costs: &prev.entropy, + ldm_seq_store: None, + }; + let curr = block.index_of(0); + if curr > ms.next_to_update + 384 { + ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); + } + let dict_mode = ms.dict_mode(None); + let mut rep = prev.rep; + let last_ll = compress::compress_block( + &mut ms, + &mut seq_store, + &mut rep, + &block, + strategy, + use_row, + dict_mode, + )?; + + let mut out = Vec::new(); + let mut reps = prev.rep; + for (i, seq) in seq_store.sequences.iter().enumerate() { + let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); + if seq_store.long_length_pos as usize == i { + match seq_store.long_length_type { + LongLengthType::LiteralLength => ll += 0x10000, + LongLengthType::MatchLength => ml += 0x10000, + LongLengthType::None => {} + } + } + update_rep(&mut reps, seq.off_base, seq.lit_length == 0); + out.push((ll, ml, reps[0])); + } + out.push((last_ll as u32, 0, 0)); + assert_eq!( + reps, rep, + "the repeat offsets the block compressor leaves are those of its sequences" + ); + Ok(out) +} + +/// The entropy encoders. A `HufCTable` is passed as its array, which is what the C code has, and an `FseCTable` is +/// converted to the layout of the C code. +pub mod entropy { + use crate::ErrorCode; + use crate::common::bitstream::BitCStream; + use crate::compress::fse::{self, FseCState, FseCTable}; + use crate::compress::hist; + use crate::compress::huf::{self, HufCTable, HufFlags, HufRepeat}; + + type R = Result; + pub type Huf = [u64; 257]; + + pub fn hist_add(count: &mut [u32], src: &[u8]) -> R<()> { + hist::add(count, src) + } + pub fn hist_count_simple(count: &mut [u32], max: &mut u32, src: &[u8]) -> R { + hist::count_simple(count, max, src) + } + pub fn hist_count_fast_wksp(count: &mut [u32], max: &mut u32, src: &[u8]) -> R { + hist::count_fast_wksp(count, max, src) + } + pub fn hist_count_wksp(count: &mut [u32], max: &mut u32, src: &[u8]) -> R { + hist::count_wksp(count, max, src) + } + + pub struct Fse(FseCTable); + impl Fse { + pub fn new() -> Self { + Self(FseCTable::new()) + } + pub fn build(&mut self, norm: &[i16], max: u32, table_log: u32) -> R<()> { + self.0.build(norm, max, table_log) + } + pub fn build_rle(&mut self, symbol: u8) -> R<()> { + self.0.build_rle(symbol) + } + pub fn get_max_nb_bits(&self, symbol: u32) -> u32 { + self.0.get_max_nb_bits(symbol) + } + pub fn bit_cost(&self, symbol: u32, accuracy_log: u32) -> u32 { + self.0.bit_cost(symbol, accuracy_log) + } + /// The `U32`s of the C table, up to the last symbol's transform. + pub fn to_c(&self) -> Vec { + let t = &self.0; + let mut v = vec![t.table_log | t.max_symbol_value << 16]; + let states = if t.table_log != 0 { + 1usize << t.table_log + } else { + 2 + }; + for pair in t.state_table[..states].chunks(2) { + v.push(pair[0] as u32 | (pair[1] as u32) << 16); + } + for tt in &t.symbol_tt[..=t.max_symbol_value as usize] { + v.push(tt.delta_find_state as u32); + v.push(tt.delta_nb_bits); + } + v + } + pub fn compress(&self, dst: &mut [u8], src: &[u8]) -> usize { + fse::compress_using_ctable(dst, src, &self.0) + } + } + /// Three states on one stream, as the sequence encoder has. + pub fn fse_encode3(dst: &mut [u8], tables: [&Fse; 3], symbols: [&[u8]; 3]) -> R { + let n = symbols[0].len(); + let mut bit_c = BitCStream::new(dst)?; + let mut a = FseCState::new2(&tables[0].0, symbols[0][n - 1] as u32); + let mut b = FseCState::new2(&tables[1].0, symbols[1][n - 1] as u32); + let mut c = FseCState::new2(&tables[2].0, symbols[2][n - 1] as u32); + for i in (0..n - 1).rev() { + b.encode_symbol(&mut bit_c, symbols[1][i] as u32); + a.encode_symbol(&mut bit_c, symbols[0][i] as u32); + c.encode_symbol(&mut bit_c, symbols[2][i] as u32); + bit_c.flush_bits(); + } + a.flush(&mut bit_c); + b.flush(&mut bit_c); + c.flush(&mut bit_c); + let d = FseCState::new(&tables[0].0); + d.flush(&mut bit_c); + Ok(bit_c.close()) + } + pub const FSE_NCOUNTBOUND: usize = fse::FSE_NCOUNTBOUND; + pub fn fse_ncount_write_bound(max: u32, table_log: u32) -> usize { + fse::ncount_write_bound(max, table_log) + } + pub fn fse_write_ncount(dst: &mut [u8], norm: &[i16], max: u32, table_log: u32) -> R { + fse::write_ncount(dst, norm, max, table_log) + } + pub fn fse_optimal_table_log(max_log: u32, size: usize, max: u32) -> u32 { + fse::optimal_table_log(max_log, size, max) + } + pub fn fse_optimal_table_log_internal(max_log: u32, size: usize, max: u32, minus: u32) -> u32 { + fse::optimal_table_log_internal(max_log, size, max, minus) + } + pub fn fse_normalize_count( + norm: &mut [i16], + table_log: u32, + count: &[u32], + total: usize, + max: u32, + low_prob: bool, + ) -> R { + fse::normalize_count(norm, table_log, count, total, max, low_prob) + } + + fn flags(f: u32) -> HufFlags { + HufFlags { + optimal_depth: f & 2 != 0, + prefer_repeat: f & 4 != 0, + suspect_uncompressible: f & 8 != 0, + } + } + fn with(t: &mut Huf, f: impl FnOnce(&mut HufCTable) -> T) -> T { + let mut ct = HufCTable(*t); + let r = f(&mut ct); + *t = ct.0; + r + } + pub const HUF_CONSTANTS: (usize, usize, u32, u32) = ( + huf::HUF_WORKSPACE_SIZE, + huf::HUF_BLOCKSIZE_MAX, + huf::HUF_TABLELOG_DEFAULT, + huf::HUF_OPTIMAL_DEPTH_THRESHOLD as u32, + ); + pub fn huf_compress_bound(size: usize) -> usize { + huf::compress_bound(size) + } + pub fn huf_build_ctable(t: &mut Huf, count: &[u32], max: u32, max_nb_bits: u32) -> R { + with(t, |t| huf::build_ctable(t, count, max, max_nb_bits)) + } + pub fn huf_write_ctable(dst: &mut [u8], t: &Huf, max: u32, huff_log: u32) -> R { + huf::write_ctable(dst, &HufCTable(*t), max, huff_log) + } + /// (size, max symbol value, has zero weights) + pub fn huf_read_ctable(t: &mut Huf, max: u32, src: &[u8]) -> R<(usize, u32, bool)> { + with(t, |t| huf::read_ctable(t, max, src)) + .map(|s| (s.size, s.max_symbol_value, s.has_zero_weights)) + } + pub fn huf_estimate_compressed_size(t: &Huf, count: &[u32], max: u32) -> usize { + huf::estimate_compressed_size(&HufCTable(*t), count, max) + } + pub fn huf_validate_ctable(t: &Huf, count: &[u32], max: u32) -> bool { + huf::validate_ctable(&HufCTable(*t), count, max) + } + pub fn huf_optimal_table_log( + max_log: u32, + size: usize, + max: u32, + t: &mut Huf, + count: &[u32], + f: u32, + ) -> u32 { + with(t, |t| { + huf::optimal_table_log(max_log, size, max, t, count, flags(f)) + }) + } + pub fn huf_min_table_log(cardinality: u32) -> u32 { + huf::min_table_log(cardinality) + } + pub fn huf_cardinality(count: &[u32], max: u32) -> u32 { + huf::cardinality(count, max) + } + pub fn huf_get_nb_bits(t: &Huf, symbol: u32) -> u32 { + HufCTable(*t).get_nb_bits(symbol) + } + pub fn huf_read_ctable_header(t: &Huf) -> (u8, u8) { + let h = huf::read_ctable_header(&HufCTable(*t)); + (h.table_log, h.max_symbol_value) + } + pub fn huf_compress_1x_using_ctable(dst: &mut [u8], src: &[u8], t: &Huf) -> R { + huf::compress_1x_using_ctable(dst, src, &HufCTable(*t)) + } + pub fn huf_compress_4x_using_ctable(dst: &mut [u8], src: &[u8], t: &Huf) -> R { + huf::compress_4x_using_ctable(dst, src, &HufCTable(*t)) + } + /// `repeat` is 0, 1 or 2, as `HUF_repeat`. `f` has the bits of `HUF_flags_e`. + pub fn huf_compress_repeat( + four: bool, + dst: &mut [u8], + src: &[u8], + max: u32, + huff_log: u32, + t: &mut Huf, + repeat: &mut u32, + f: u32, + ) -> R { + let mut rep = match *repeat { + 0 => HufRepeat::None, + 1 => HufRepeat::Check, + _ => HufRepeat::Valid, + }; + let r = with(t, |t| { + if four { + huf::compress_4x_repeat(dst, src, max, huff_log, t, &mut rep, flags(f)) + } else { + huf::compress_1x_repeat(dst, src, max, huff_log, t, &mut rep, flags(f)) + } + }); + *repeat = match rep { + HufRepeat::None => 0, + HufRepeat::Check => 1, + HufRepeat::Valid => 2, + }; + r + } +} + +/// For timing: no copy of the table. +pub mod entropy_bench { + use crate::compress::huf::{self, HufCTable, HufFlags, HufRepeat}; + pub struct Table(HufCTable); + impl Table { + pub fn new(t: &[u64; 257]) -> Self { + Self(HufCTable(*t)) + } + pub fn build(&mut self, count: &[u32], max: u32, bits: u32) -> usize { + huf::build_ctable(&mut self.0, count, max, bits).unwrap_or(0) + } + pub fn write(&self, dst: &mut [u8], max: u32, log: u32) -> usize { + huf::write_ctable(dst, &self.0, max, log).unwrap_or(0) + } + pub fn compress4(&self, dst: &mut [u8], src: &[u8]) -> usize { + huf::compress_4x_using_ctable(dst, src, &self.0).unwrap_or(0) + } + pub fn repeat4(&mut self, dst: &mut [u8], src: &[u8], optimal_depth: bool) -> usize { + let mut rep = HufRepeat::None; + huf::compress_4x_repeat( + dst, + src, + 255, + 11, + &mut self.0, + &mut rep, + HufFlags { + optimal_depth, + ..Default::default() + }, + ) + .unwrap_or(0) + } + } +} diff --git a/src/zstd/wip/work/fast/harness/Cargo.toml b/src/zstd/wip/work/fast/harness/Cargo.toml new file mode 100644 index 000000000000..bba38b8e8cf6 --- /dev/null +++ b/src/zstd/wip/work/fast/harness/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "zstd_harness" +version = "0.0.0" +edition = "2024" + +[dependencies] +bun_zstd = { path = "/root/zstd-ref/work/fast/zstd" } + +[profile.release] +debug = "line-tables-only" +overflow-checks = true +debug-assertions = true +panic = "abort" + +[profile.bench] +inherits = "release" +overflow-checks = false +debug-assertions = false + +[workspace] diff --git a/src/zstd/wip/work/fast/harness/build.rs b/src/zstd/wip/work/fast/harness/build.rs new file mode 100644 index 000000000000..86bd64eee215 --- /dev/null +++ b/src/zstd/wip/work/fast/harness/build.rs @@ -0,0 +1,21 @@ +use std::process::Command; + +fn main() { + let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); + println!("cargo:rerun-if-env-changed=ZSTD_REF"); + println!("cargo:rerun-if-changed=c/shim.c"); + println!("cargo:rerun-if-changed=c/shim_cctx.inc"); + let out = std::env::var("OUT_DIR").unwrap(); + let ok = Command::new("clang-23") + .args(["-c", "-O2", "-g", "-fPIC", "-march=nehalem", "-DZSTD_MULTITHREAD", "-DZSTD_LEGACY_SUPPORT=0", "-DXXH_NAMESPACE=ZSTD_"]) + .args(["-I/root/zstd-ref/zstd/lib", "-I/root/zstd-ref/zstd/lib/common", "c/shim.c", "-o"]) + .arg(format!("{out}/shim.o")).status().unwrap().success(); + assert!(ok); + let _ = std::fs::remove_file(format!("{out}/libshim.a")); + let ok = Command::new("llvm-ar-23").arg("rcs").arg(format!("{out}/libshim.a")).arg(format!("{out}/shim.o")).status().unwrap().success(); + assert!(ok); + println!("cargo:rustc-link-search=native={out}"); + println!("cargo:rustc-link-lib=static=shim"); + println!("cargo:rustc-link-search=native=/root/zstd-ref/build/{variant}"); + println!("cargo:rustc-link-lib=static=zstd"); +} diff --git a/src/zstd/wip/work/fast/harness/c/shim.c b/src/zstd/wip/work/fast/harness/c/shim.c new file mode 100644 index 000000000000..fb4ef9f981d4 --- /dev/null +++ b/src/zstd/wip/work/fast/harness/c/shim.c @@ -0,0 +1,169 @@ +/* Drives the block compressors of zstd_fast.c and zstd_double_fast.c on match states made by hand. */ +#define ZSTD_STATIC_LINKING_ONLY +#include "compress/zstd_compress_internal.h" +#include "compress/zstd_fast.h" +#include "compress/zstd_double_fast.h" +#include +#include + +typedef struct { + ZSTD_MatchState_t ms; + SeqStore_t seqStore; +} Shim; + +typedef struct { + U64 base, dictBase, nextSrc; + U32 dictLimit, lowLimit, loadedDictEnd, nextToUpdate, dictMode, pad; +} ShimState; + +Shim* shim_new(const U32* cp, size_t maxBlock) +{ + Shim* const s = (Shim*)calloc(1, sizeof(*s)); + ZSTD_window_init(&s->ms.window); + s->ms.cParams.windowLog = cp[0]; + s->ms.cParams.chainLog = cp[1]; + s->ms.cParams.hashLog = cp[2]; + s->ms.cParams.searchLog = cp[3]; + s->ms.cParams.minMatch = cp[4]; + s->ms.cParams.targetLength = cp[5]; + s->ms.cParams.strategy = (ZSTD_strategy)cp[6]; + s->ms.hashTable = (U32*)calloc((size_t)1 << cp[2], sizeof(U32)); + s->ms.chainTable = (U32*)calloc((size_t)1 << cp[1], sizeof(U32)); + s->ms.nextToUpdate = s->ms.window.dictLimit; + s->seqStore.maxNbSeq = maxBlock / 3 + 1; + s->seqStore.maxNbLit = maxBlock; + s->seqStore.sequencesStart = (SeqDef*)malloc(s->seqStore.maxNbSeq * sizeof(SeqDef)); + s->seqStore.litStart = (BYTE*)malloc(maxBlock + WILDCOPY_OVERLENGTH); + ZSTD_resetSeqStore(&s->seqStore); + return s; +} + +void shim_free(Shim* s) +{ + free(s->ms.hashTable); free(s->ms.chainTable); + free(s->seqStore.sequencesStart); free(s->seqStore.litStart); + free(s); +} + +void shim_state(const Shim* s, ShimState* st) +{ + st->base = (U64)(size_t)s->ms.window.base; + st->dictBase = (U64)(size_t)s->ms.window.dictBase; + st->nextSrc = (U64)(size_t)s->ms.window.nextSrc; + st->dictLimit = s->ms.window.dictLimit; + st->lowLimit = s->ms.window.lowLimit; + st->loadedDictEnd = s->ms.loadedDictEnd; + st->nextToUpdate = s->ms.nextToUpdate; + st->dictMode = (U32)ZSTD_matchState_dictMode(&s->ms); + st->pad = 0; +} + +const U32* shim_hash_table(const Shim* s) { return s->ms.hashTable; } +const U32* shim_chain_table(const Shim* s) { return s->ms.chainTable; } + +/* Indices go on from `index`, as after many frames. */ +void shim_bump(Shim* s, U32 index) +{ + if (s->ms.window.dictLimit < index) { + s->ms.window.nextSrc = s->ms.window.base + index; + ZSTD_window_clear(&s->ms.window); + s->ms.nextToUpdate = s->ms.window.dictLimit; + } +} + +/* ZSTD_loadDictionaryContent. Returns whether the tables were filled; `before` is the state right before that. */ +int shim_load_dict(Shim* s, const void* src, size_t srcSize, int dtlm, int tfp, int forceNonContiguous, ShimState* before) +{ + ZSTD_MatchState_t* const ms = &s->ms; + const BYTE* ip = (const BYTE*)src; + const BYTE* const iend = ip + srcSize; + { U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; + if (tfp == ZSTD_tfp_forCDict) { + U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; + maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize); + } + if (srcSize > maxDictSize) { ip = iend - maxDictSize; src = ip; srcSize = maxDictSize; } + } + ZSTD_window_update(&ms->window, src, srcSize, 0); + { U32 maxDictSize = 1U << MIN(MAX(ms->cParams.hashLog + 3, ms->cParams.chainLog + 1), 31); + if (srcSize > maxDictSize) { ip = iend - maxDictSize; src = ip; srcSize = maxDictSize; } + } + ms->nextToUpdate = (U32)(ip - ms->window.base); + ms->loadedDictEnd = (U32)(iend - ms->window.base); + ms->forceNonContiguous = forceNonContiguous; + shim_state(s, before); + if (srcSize <= HASH_READ_SIZE) return 0; + if (ms->cParams.strategy == ZSTD_fast) + ZSTD_fillHashTable(ms, iend, (ZSTD_dictTableLoadMethod_e)dtlm, (ZSTD_tableFillPurpose_e)tfp); + else + ZSTD_fillDoubleHashTable(ms, iend, (ZSTD_dictTableLoadMethod_e)dtlm, (ZSTD_tableFillPurpose_e)tfp); + ms->nextToUpdate = (U32)(iend - ms->window.base); + return 1; +} + +/* The end of ZSTD_resetCCtx_byAttachingCDict. Returns whether the dictionary is attached. */ +int shim_attach(Shim* s, const Shim* dict) +{ + U32 const cdictEnd = (U32)(dict->ms.window.nextSrc - dict->ms.window.base); + U32 const cdictLen = cdictEnd - dict->ms.window.dictLimit; + if (cdictLen == 0) return 0; + s->ms.dictMatchState = &dict->ms; + if (s->ms.window.dictLimit < cdictEnd) { + s->ms.window.nextSrc = s->ms.window.base + cdictEnd; + ZSTD_window_clear(&s->ms.window); + } + s->ms.loadedDictEnd = s->ms.window.dictLimit; + return 1; +} + +/* The start of ZSTD_compressContinue_internal. Returns whether the input is contiguous. */ +int shim_update(Shim* s, const void* src, size_t srcSize) +{ + ZSTD_MatchState_t* const ms = &s->ms; + if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { + ms->forceNonContiguous = 0; + ms->nextToUpdate = ms->window.dictLimit; + return 0; + } + return 1; +} + +/* What ZSTD_compress_frameChunk does before each block. */ +void shim_pre_block(Shim* s, const void* src, size_t blockSize) +{ + ZSTD_MatchState_t* const ms = &s->ms; + const BYTE* const ip = (const BYTE*)src; + U32 const maxDist = (U32)1 << ms->cParams.windowLog; + ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; +} + +size_t shim_block(Shim* s, U32* rep, const void* src, size_t srcSize) +{ + ZSTD_MatchState_t* const ms = &s->ms; + ZSTD_dictMode_e const mode = ZSTD_matchState_dictMode(ms); + ZSTD_resetSeqStore(&s->seqStore); + if (ms->cParams.strategy == ZSTD_fast) { + switch (mode) { + case ZSTD_noDict: return ZSTD_compressBlock_fast(ms, &s->seqStore, rep, src, srcSize); + case ZSTD_extDict: return ZSTD_compressBlock_fast_extDict(ms, &s->seqStore, rep, src, srcSize); + default: return ZSTD_compressBlock_fast_dictMatchState(ms, &s->seqStore, rep, src, srcSize); + } + } + switch (mode) { + case ZSTD_noDict: return ZSTD_compressBlock_doubleFast(ms, &s->seqStore, rep, src, srcSize); + case ZSTD_extDict: return ZSTD_compressBlock_doubleFast_extDict(ms, &s->seqStore, rep, src, srcSize); + default: return ZSTD_compressBlock_doubleFast_dictMatchState(ms, &s->seqStore, rep, src, srcSize); + } +} + +size_t shim_nb_seq(const Shim* s) { return (size_t)(s->seqStore.sequences - s->seqStore.sequencesStart); } +const SeqDef* shim_seqs(const Shim* s) { return s->seqStore.sequencesStart; } +size_t shim_lit_len(const Shim* s) { return (size_t)(s->seqStore.lit - s->seqStore.litStart); } +const BYTE* shim_lits(const Shim* s) { return s->seqStore.litStart; } +U32 shim_long_type(const Shim* s) { return (U32)s->seqStore.longLengthType; } +U32 shim_long_pos(const Shim* s) { return s->seqStore.longLengthPos; } + +/* From the harness of the lazy strategies: the state of a whole CCtx. */ +#include "shim_cctx.inc" diff --git a/src/zstd/wip/work/fast/harness/c/shim_cctx.inc b/src/zstd/wip/work/fast/harness/c/shim_cctx.inc new file mode 100644 index 000000000000..25c2831c1206 --- /dev/null +++ b/src/zstd/wip/work/fast/harness/c/shim_cctx.inc @@ -0,0 +1,30 @@ + +typedef struct { + unsigned cp[7]; + unsigned useRow; + unsigned hasDms; + unsigned nextToUpdate, loadedDictEnd, lowLimit, dictLimit, rowHashLog, hashSaltEntropy, lazySkipping; + unsigned long long hashSalt; + unsigned hashCache[8]; + unsigned rep[3]; + const unsigned* hashTable; + const unsigned* chainTable; + const unsigned char* tagTable; +} ShimCCtxState; + +void shim_cctx_state(const ZSTD_CCtx* c, ShimCCtxState* s) { + const ZSTD_MatchState_t* ms = &c->blockState.matchState; + const ZSTD_compressionParameters* p = &c->appliedParams.cParams; + int i; + s->cp[0] = p->windowLog; s->cp[1] = p->chainLog; s->cp[2] = p->hashLog; s->cp[3] = p->searchLog; + s->cp[4] = p->minMatch; s->cp[5] = p->targetLength; s->cp[6] = (unsigned)p->strategy; + s->useRow = c->appliedParams.useRowMatchFinder == ZSTD_ps_enable; + s->hasDms = ms->dictMatchState != NULL; + s->nextToUpdate = ms->nextToUpdate; s->loadedDictEnd = ms->loadedDictEnd; + s->lowLimit = ms->window.lowLimit; s->dictLimit = ms->window.dictLimit; + s->rowHashLog = ms->rowHashLog; s->hashSaltEntropy = ms->hashSaltEntropy; s->lazySkipping = (unsigned)ms->lazySkipping; + s->hashSalt = ms->hashSalt; + for (i = 0; i < 8; i++) s->hashCache[i] = ms->hashCache[i]; + for (i = 0; i < 3; i++) s->rep[i] = c->blockState.prevCBlock->rep[i]; + s->hashTable = ms->hashTable; s->chainTable = ms->chainTable; s->tagTable = ms->tagTable; +} diff --git a/src/zstd/wip/work/fast/harness/rust-toolchain.toml b/src/zstd/wip/work/fast/harness/rust-toolchain.toml new file mode 100644 index 000000000000..c71778e8ccd3 --- /dev/null +++ b/src/zstd/wip/work/fast/harness/rust-toolchain.toml @@ -0,0 +1,35 @@ +[toolchain] +channel = "nightly-2026-09-15" +# rust-src is needed for -Zbuild-std (Tier 3 targets like +# aarch64-unknown-freebsd have no prebuilt std). miri is for +# `bun run rust:miri`. targets ensures the +# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain +# (CI's images install this same channel with these components and +# targets, from scripts/build/ci-images/spec.ts; a source lint keeps +# the two files in step). Note: rustup ignores this whole file when +# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree +# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` +# self-heals that path. Keep this list as a fallback for direct +# `cargo build`/`cargo check`/rust-analyzer use. +# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd +# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). +# darwin/windows triples build natively on their own agents, so the host std +# comes with the toolchain install regardless; they're listed here so a dev +# running `bun run rust:check-all` or `cargo check --target` from any host +# still gets prebuilt std for every Tier 1/2 triple. +# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at +# what it emits. The build does not use them. +components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] +targets = [ + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-musl", + "aarch64-linux-android", + "x86_64-linux-android", + "x86_64-unknown-freebsd", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-pc-windows-msvc", + "x86_64-pc-windows-msvc", +] diff --git a/src/zstd/wip/work/fast/harness/src/c.rs b/src/zstd/wip/work/fast/harness/src/c.rs new file mode 100644 index 000000000000..d5a82e1f6b78 --- /dev/null +++ b/src/zstd/wip/work/fast/harness/src/c.rs @@ -0,0 +1,104 @@ +//! The reference C library. +#![allow(non_camel_case_types, dead_code)] +use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; + +#[repr(C)] +pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } +#[repr(C)] +pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } +pub enum ZSTD_CCtx {} +pub enum ZSTD_DCtx {} +pub enum ZSTD_DDict {} + +unsafe extern "C" { + pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; + pub fn ZSTD_compressBound(size: usize) -> usize; + pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_isError(code: usize) -> c_uint; + pub fn ZSTD_getErrorCode(code: usize) -> c_uint; + pub fn ZSTD_getErrorName(code: usize) -> *const c_char; + pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; + pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; + pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; + pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; + pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; + pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; + pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; + pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; + pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; + pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; + pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; + pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; + pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; + pub fn ZSTD_defaultCLevel() -> c_int; + pub fn ZSTD_minCLevel() -> c_int; + pub fn ZSTD_maxCLevel() -> c_int; +} + +/// `Ok(value)` or `Err(error code)`. +pub fn res(code: usize) -> Result { + unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } +} + +pub struct CCtx(pub *mut ZSTD_CCtx); +impl CCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } + pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + /// Returns (result, out pos, in pos). + pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); + (r, o.pos, i.pos) + } +} +impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } + +pub struct DCtx(pub *mut ZSTD_DCtx); +impl DCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } + pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } + pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); + (r, o.pos, i.pos) + } +} +impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } + +pub fn compress(src: &[u8], level: i32) -> Vec { + let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; + let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); + dst.truncate(n); + dst +} +pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) +} diff --git a/src/zstd/wip/work/fast/harness/src/data.rs b/src/zstd/wip/work/fast/harness/src/data.rs new file mode 100644 index 000000000000..c48a645e1b5d --- /dev/null +++ b/src/zstd/wip/work/fast/harness/src/data.rs @@ -0,0 +1,65 @@ +//! Deterministic inputs. +pub struct Rng(pub u64); +impl Rng { + pub fn next(&mut self) -> u64 { + // splitmix64 + self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + } + pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } + pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } + pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } + pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } + /// Skewed towards small values. + pub fn size(&mut self, max: usize) -> usize { + let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); + (self.next() as usize & ((1usize << bits) - 1)).min(max) + } +} + +static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; + +/// One buffer of about `len` bytes, of a kind picked by `rng`. +pub fn data(rng: &mut Rng, len: usize) -> Vec { + let mut v = Vec::with_capacity(len + 64); + match rng.below(9) { + 0 => v.resize(len, rng.next() as u8), + 1 => while v.len() < len { v.push(rng.next() as u8) }, + 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, + 3 => { // few symbols, skewed + let n = rng.range(2, 40); + while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } + } + 4 => { // LZ-like: copies of earlier parts, at all distances + while v.len() < len { + if v.len() > 8 && rng.chance(3, 4) { + let dist = 1 + rng.size(v.len() - 1); + let n = 3 + rng.size(300); + for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } + } else { + for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } + } + } + } + 5 => { // periodic with noise + let period = rng.range(1, 300); + let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); + while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } + } + 6 => { // sections of different kinds + while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } + } + 7 => { // numbers as text + let mut x = rng.next() % 1000; + while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } + } + _ => { // long runs + while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } + } + } + v.truncate(len); + v +} diff --git a/src/zstd/wip/work/fast/harness/src/frames.rs b/src/zstd/wip/work/fast/harness/src/frames.rs new file mode 100644 index 000000000000..eeb73a560cae --- /dev/null +++ b/src/zstd/wip/work/fast/harness/src/frames.rs @@ -0,0 +1,178 @@ +//! A frame through the Rust block compressors and through the C library (`ZSTD_generateSequences`). +use crate::c; +use crate::data::{Rng, data}; +use bun_zstd::testing_frame::{Dict, State}; +use std::ffi::{c_uint, c_void}; + +#[repr(C)] +#[derive(Clone, Copy, Default, Debug)] +struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } +#[repr(C)] +#[derive(Clone, Copy, Debug)] +struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } +#[repr(C)] +#[derive(Clone, Copy)] +struct CustomMem { a: *const c_void, b: *const c_void, c: *const c_void } +#[repr(C)] +struct ShimState { + cp: [c_uint; 7], use_row: c_uint, has_dms: c_uint, + next_to_update: c_uint, loaded_dict_end: c_uint, low_limit: c_uint, dict_limit: c_uint, row_hash_log: c_uint, hash_salt_entropy: c_uint, lazy_skipping: c_uint, + hash_salt: u64, hash_cache: [c_uint; 8], rep: [c_uint; 3], + hash_table: *const u32, chain_table: *const u32, tag_table: *const u8, +} +enum CDict {} +enum CCtxParams {} +unsafe extern "C" { + fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; + fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; + fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; + fn ZSTD_createCCtxParams() -> *mut CCtxParams; + fn ZSTD_freeCCtxParams(p: *mut CCtxParams) -> usize; + fn ZSTD_CCtxParams_setParameter(p: *mut CCtxParams, param: c_uint, v: i32) -> usize; + fn ZSTD_createCDict_advanced2(d: *const c_void, n: usize, load: c_uint, ty: c_uint, p: *const CCtxParams, mem: CustomMem) -> *mut CDict; + fn ZSTD_freeCDict(d: *mut CDict) -> usize; + fn ZSTD_CCtx_refCDict(c: *mut c::ZSTD_CCtx, d: *const CDict) -> usize; + fn ZSTD_getCParamsFromCDict(d: *const CDict) -> CParams; + fn shim_cctx_state(c: *const c::ZSTD_CCtx, s: *mut ShimState); +} + +fn arr(p: CParams) -> [u32; 7] { [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy] } + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum Mode { NoDict, Prefix, CDict } + +struct COut { seqs: Vec<(u32, u32, u32, u32)>, cp: [u32; 7], use_row: bool, cdict_cp: [u32; 7], state: State } + +/// `None` if the C library refuses (a last block that is too small). +fn c_frame(cp: [u32; 7], use_row: bool, mode: Mode, dict: &[u8], src: &[u8]) -> Option { + let mut cctx = c::CCtx::new(); + let mut cdict = std::ptr::null_mut(); + let mut cdict_cp = [0; 7]; + cctx.set(1010, 2).unwrap(); // no block splitter + if mode == Mode::CDict { + unsafe { + let p = ZSTD_createCCtxParams(); + for (i, v) in cp.iter().enumerate() { c::res(ZSTD_CCtxParams_setParameter(p, 101 + i as u32, *v as i32)).unwrap(); } + c::res(ZSTD_CCtxParams_setParameter(p, 1011, if use_row { 1 } else { 2 })).unwrap(); + let null = std::ptr::null(); + cdict = ZSTD_createCDict_advanced2(dict.as_ptr().cast(), dict.len(), 1, 1, p, CustomMem { a: null, b: null, c: null }); + assert!(!cdict.is_null()); + ZSTD_freeCCtxParams(p); + c::res(ZSTD_CCtx_refCDict(cctx.0, cdict)).unwrap(); + cdict_cp = arr(ZSTD_getCParamsFromCDict(cdict)); + } + cctx.set(1001, 1).unwrap(); // force attach + cctx.set(101, cp[0] as i32).unwrap(); + } else { + for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } + cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder + if mode == Mode::Prefix { cctx.ref_prefix(dict).unwrap(); } + } + let mut out = vec![Sequence::default(); src.len() / 3 + src.len() / 1024 + 8]; + let n = c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }); + let r = n.ok().map(|n| { + let mut s: ShimState = unsafe { std::mem::zeroed() }; + unsafe { shim_cctx_state(cctx.0, &mut s) }; + let row_used = s.use_row != 0 && (3..=5).contains(&s.cp[6]); + let table = |p: *const u32, n: usize| if n == 0 { Vec::new() } else { unsafe { std::slice::from_raw_parts(p, n) }.to_vec() }; + let state = State { + has_dms: s.has_dms != 0, next_to_update: s.next_to_update, loaded_dict_end: s.loaded_dict_end, low_limit: s.low_limit, dict_limit: s.dict_limit, + hash_salt_entropy: s.hash_salt_entropy, lazy_skipping: s.lazy_skipping != 0, hash_cache: s.hash_cache, rep: s.rep, + hash_table: table(s.hash_table, 1 << s.cp[2]), + chain_table: table(s.chain_table, if s.cp[6] != 1 && !row_used { 1 << s.cp[1] } else { 0 }), + tag_table: if row_used { unsafe { std::slice::from_raw_parts(s.tag_table, 1 << s.cp[2]) }.to_vec() } else { Vec::new() }, + }; + COut { + seqs: out[..n].iter().map(|s| (s.lit_length, s.match_length, s.rep, if s.rep == 0 { s.offset } else { 0 })).collect(), + cp: s.cp, use_row: s.use_row != 0, cdict_cp, state, + } + }); + if !cdict.is_null() { unsafe { ZSTD_freeCDict(cdict) }; } + r +} + +fn first_diff(a: &[T], b: &[T]) -> Option { + if a.len() != b.len() { return Some(a.len().min(b.len())); } + a.iter().zip(b).position(|(a, b)| a != b) +} + +/// `strategies`: which of 1..=9 to test. `modes`: which kinds of dictionary. +pub fn run(seed: u64, iters: u64, strategies: &[u32], modes: &[Mode]) { + let mut skipped = 0u64; + let mut dms_blocks = 0u64; + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if i % 2000 == 0 { eprintln!("seqs iter {i} (seed {s})"); } + let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071, 400_000]); + let len = rng.range(7, max); + let mut src = data(&mut rng, len); + let mode = *rng.pick(modes); + let dict = if mode != Mode::NoDict { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; + let mode = if dict.is_empty() { Mode::NoDict } else { mode }; + if !dict.is_empty() && rng.chance(2, 3) { + for _ in 0..rng.range(1, 6) { + let n = rng.size(dict.len().min(src.len())); + let at = if rng.chance(1, 3) { dict.len() - n } else { rng.below(dict.len() - n + 1) }; + let to = if rng.chance(1, 4) { 0 } else { rng.below(src.len() - n + 1) }; + src[to..to + n].copy_from_slice(&dict[at..at + n]); + } + } + let strategy = *rng.pick(strategies); + let p = if rng.chance(1, 3) { + // the parameters of a level, with the strategy forced + let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; + p.strategy = strategy; + p + } else { + CParams { + window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, + search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, + target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, + } + }; + let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; + let use_row = rng.chance(1, 2); + let Some(c) = c_frame(arr(p), use_row, mode, &dict, &src) else { skipped += 1; continue }; + let d = match mode { Mode::NoDict => Dict::None, Mode::Prefix => Dict::Prefix(&dict), Mode::CDict => Dict::CDict(&dict, c.cdict_cp) }; + let what = format!("seed {s}: {mode:?} cp {:?} cdict cp {:?} row {} dict {} src {}", c.cp, c.cdict_cp, c.use_row, dict.len(), src.len()); + let (got, state) = bun_zstd::testing_frame::frame(c.cp, c.use_row, d, &src).unwrap_or_else(|e| panic!("{what}: error {e:?}")); + if mode == Mode::CDict && dict.len() >= 8 { dms_blocks += 1; } + if let Some(at) = first_diff(&got, &c.seqs) { + panic!("{what}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", got.get(at), c.seqs.get(at), got.len(), c.seqs.len()); + } + if state != c.state { + let (a, b) = (&state, &c.state); + panic!("{what}: state differs: hash table {:?} chain table {:?} tag table {:?}\n got {:?}\n exp {:?}", + first_diff(&a.hash_table, &b.hash_table), first_diff(&a.chain_table, &b.chain_table), first_diff(&a.tag_table, &b.tag_table), + (a.has_dms, a.next_to_update, a.loaded_dict_end, a.low_limit, a.dict_limit, a.hash_salt_entropy, a.lazy_skipping, a.hash_cache, a.rep), + (b.has_dms, b.next_to_update, b.loaded_dict_end, b.low_limit, b.dict_limit, b.hash_salt_entropy, b.lazy_skipping, b.hash_cache, b.rep)); + } + } + println!("seqs ok: {iters} iterations from seed {seed} ({skipped} skipped, {dms_blocks} with an attached CDict), strategies {strategies:?}, modes {modes:?}"); +} + +/// Speed of the Rust block compressors against the C ones, on a file. +pub fn bench(path: &str, mode: Mode) { + let all = std::fs::read(path).unwrap(); + let (dict, src) = if mode == Mode::NoDict { (&[][..], &all[..]) } else { all.split_at(100_000.min(all.len() / 2)) }; + for (level, strategy, use_row) in [(5, 3, false), (5, 3, true), (7, 4, false), (7, 4, true), (10, 5, false), (10, 5, true), (12, 5, true), (13, 6, false), (15, 6, false)] { + let mut p = unsafe { ZSTD_getCParams(level, if mode == Mode::CDict { 100_000 } else { src.len() as u64 }, dict.len()) }; + p.strategy = strategy; + let mut best = [f64::MAX; 2]; + let mut nseq = [0usize; 2]; + for _ in 0..3 { + let t = std::time::Instant::now(); + let c = c_frame(arr(p), use_row, mode, dict, src).unwrap(); + best[0] = best[0].min(t.elapsed().as_secs_f64()); + nseq[0] = c.seqs.len(); + let d = match mode { Mode::NoDict => Dict::None, Mode::Prefix => Dict::Prefix(dict), Mode::CDict => Dict::CDict(dict, c.cdict_cp) }; + let t = std::time::Instant::now(); + let (got, _) = bun_zstd::testing_frame::frame(c.cp, c.use_row, d, src).unwrap(); + best[1] = best[1].min(t.elapsed().as_secs_f64()); + nseq[1] = got.len(); + } + let mb = src.len() as f64 / 1e6; + println!("{mode:?} level {level} strategy {strategy} row {use_row}: C {:.1} MB/s, Rust {:.1} MB/s ({:.0}%) seqs {} {}", mb / best[0], mb / best[1], 100.0 * best[0] / best[1], nseq[0], nseq[1]); + } +} diff --git a/src/zstd/wip/work/fast/harness/src/main.rs b/src/zstd/wip/work/fast/harness/src/main.rs new file mode 100644 index 000000000000..9e62db4a65c6 --- /dev/null +++ b/src/zstd/wip/work/fast/harness/src/main.rs @@ -0,0 +1,44 @@ +mod c; +mod data; +mod frames; +mod seqs; +mod stream; + +fn main() { + let args: Vec = std::env::args().collect(); + let cmd = args.get(1).map(String::as_str).unwrap_or(""); + let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); + let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); + match cmd { + // seqs + "seqs" => { + let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); + seqs::run(seed, iters, &strategies, args.get(5).is_some_and(|s| s == "1")); + } + // frames <0: no dictionary, 1: a prefix or none, 2: an attached CDict or none> + "frames" => { + let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); + let modes: &[frames::Mode] = match args.get(5).map(String::as_str) { + Some("1") => &[frames::Mode::NoDict, frames::Mode::Prefix], + Some("2") => &[frames::Mode::CDict, frames::Mode::CDict, frames::Mode::CDict, frames::Mode::NoDict], + _ => &[frames::Mode::NoDict], + }; + frames::run(seed, iters, &strategies, modes); + } + // bench + "bench" => { + let cp: Vec = args[3].split(',').map(|s| s.parse().unwrap()).collect(); + stream::bench(&args[2], cp.try_into().unwrap()); + } + // stream + "stream" => { + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + eprintln!("stream seed {}", stream::CURRENT.load(std::sync::atomic::Ordering::Relaxed)); + hook(info); + })); + stream::run(seed, iters); + } + _ => eprintln!("unknown command"), + } +} diff --git a/src/zstd/wip/work/fast/harness/src/seqs.rs b/src/zstd/wip/work/fast/harness/src/seqs.rs new file mode 100644 index 000000000000..4a41a2c60adb --- /dev/null +++ b/src/zstd/wip/work/fast/harness/src/seqs.rs @@ -0,0 +1,78 @@ +//! One block through a Rust block compressor and through the C library (`ZSTD_generateSequences`). +use crate::c; +use crate::data::{Rng, data}; +use std::ffi::{c_uint, c_void}; + +#[repr(C)] +#[derive(Clone, Copy, Default, Debug)] +struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } +#[repr(C)] +#[derive(Clone, Copy, Debug)] +struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } +unsafe extern "C" { + fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; + fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; + fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; +} + +fn c_sequences(cp: [u32; 7], use_row: bool, dict: &[u8], src: &[u8]) -> Vec<(u32, u32, u32)> { + let mut cctx = c::CCtx::new(); + // windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy + for (id, v) in [101, 103, 102, 104, 105, 106, 107].iter().zip(cp) { cctx.set(*id, v as i32).unwrap(); } + cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder + cctx.set(1010, 2).unwrap(); // no block splitter + if !dict.is_empty() { cctx.ref_prefix(dict).unwrap(); } + let mut out = vec![Sequence::default(); src.len() / 3 + 8]; + let n = c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }).unwrap(); + out[..n].iter().map(|s| (s.lit_length, s.match_length, s.offset)).collect() +} + +/// `strategies`: which of 1..=9 to test. `with_dict`: also test with a prefix, which is the extDict variants. +pub fn run(seed: u64, iters: u64, strategies: &[u32], with_dict: bool) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if i % 500 == 0 { eprintln!("seqs iter {i} (seed {s})"); } + let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071]); + let len = rng.range(7, max); + let mut src = data(&mut rng, len); + let dict = if with_dict && rng.chance(1, 2) { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; + if !dict.is_empty() && rng.chance(2, 3) { + for _ in 0..rng.range(1, 6) { + let n = rng.size(dict.len().min(src.len())); + let at = rng.below(dict.len() - n + 1); + let to = rng.below(src.len() - n + 1); + src[to..to + n].copy_from_slice(&dict[at..at + n]); + } + } + let strategy = *rng.pick(strategies); + let p = if rng.chance(1, 3) { + // the parameters of a level, with the strategy forced + let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; + p.strategy = strategy; + p + } else { + CParams { + window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, + search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, + target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, + } + }; + let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; + // The C library refuses to give the sequences of a block of less than 7 bytes. + let block_size = (1usize << p.window_log).min(128 << 10); + if src.len() % block_size != 0 && src.len() % block_size < 7 { src.truncate(src.len() - src.len() % block_size); } + let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; + let cp = [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy]; + let use_row = rng.chance(1, 2); + let expected = c_sequences(cp, use_row, &dict, &src); + let got = bun_zstd::testing::first_block(cp, use_row, &dict, &src); + let got = got.unwrap_or_else(|e| panic!("seed {s}: error {e:?} (cp {cp:?} row {use_row} dict {} src {})", dict.len(), src.len())); + if got != expected { + let at = got.iter().zip(&expected).position(|(a, b)| a != b).unwrap_or(got.len().min(expected.len())); + panic!("seed {s}: cp {cp:?} row {use_row} dict {} src {}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", + dict.len(), src.len(), got.get(at), expected.get(at), got.len(), expected.len()); + } + } + println!("seqs ok: {iters} iterations from seed {seed}, strategies {strategies:?}, dict {with_dict}"); +} diff --git a/src/zstd/wip/work/fast/harness/src/stream.rs b/src/zstd/wip/work/fast/harness/src/stream.rs new file mode 100644 index 000000000000..e75619deaa4d --- /dev/null +++ b/src/zstd/wip/work/fast/harness/src/stream.rs @@ -0,0 +1,301 @@ +//! The block compressors and the table fillers of `fast` and `dfast`, call by call, against the C functions +//! (`c/shim.c`), on match states made by hand: dictionaries that are loaded, attached (dictMatchState) or none, input +//! in chunks that are contiguous or not or that overwrite earlier input, blocks of any size, indices that start high. +//! After each call the sequences, the literals, the repeat offsets and the tables are compared. +use crate::data::{Rng, data}; +use bun_zstd::testing::TestMs; +use std::ffi::c_void; + +pub enum Shim {} +#[repr(C)] +#[derive(Clone, Copy, Default, Debug)] +pub struct State { + base: u64, dict_base: u64, next_src: u64, + dict_limit: u32, low_limit: u32, loaded_dict_end: u32, next_to_update: u32, dict_mode: u32, pad: u32, +} +#[repr(C)] +#[derive(Clone, Copy, Debug)] +struct SeqDef { off_base: u32, lit_length: u16, ml_base: u16 } + +unsafe extern "C" { + fn shim_new(cp: *const u32, max_block: usize) -> *mut Shim; + fn shim_free(s: *mut Shim); + fn shim_state(s: *const Shim, st: *mut State); + fn shim_hash_table(s: *const Shim) -> *const u32; + fn shim_chain_table(s: *const Shim) -> *const u32; + fn shim_bump(s: *mut Shim, index: u32); + fn shim_load_dict(s: *mut Shim, src: *const c_void, size: usize, dtlm: i32, tfp: i32, force: i32, before: *mut State) -> i32; + fn shim_attach(s: *mut Shim, dict: *const Shim) -> i32; + fn shim_update(s: *mut Shim, src: *const c_void, size: usize) -> i32; + fn shim_pre_block(s: *mut Shim, src: *const c_void, size: usize); + fn shim_block(s: *mut Shim, rep: *mut u32, src: *const c_void, size: usize) -> usize; + fn shim_nb_seq(s: *const Shim) -> usize; + fn shim_seqs(s: *const Shim) -> *const SeqDef; + fn shim_lit_len(s: *const Shim) -> usize; + fn shim_lits(s: *const Shim) -> *const u8; + fn shim_long_type(s: *const Shim) -> u32; + fn shim_long_pos(s: *const Shim) -> u32; +} + +/// The same match state on both sides. +struct Pair { c: *mut Shim, r: TestMs, cp: [u32; 7] } +impl Pair { + fn new(cp: [u32; 7], max_block: usize) -> Self { Self { c: unsafe { shim_new(cp.as_ptr(), max_block) }, r: TestMs::new(cp, max_block), cp } } + fn state(&self) -> State { let mut st = State::default(); unsafe { shim_state(self.c, &mut st) }; st } + fn sync(&mut self, st: &State) { self.r.set_state(st.dict_limit, st.low_limit, st.loaded_dict_end, st.next_to_update); } + fn check_tables(&self, what: &str) { + let (hash, chain) = self.r.tables(); + let c_hash = unsafe { std::slice::from_raw_parts(shim_hash_table(self.c), 1 << self.cp[2]) }; + let c_chain = unsafe { std::slice::from_raw_parts(shim_chain_table(self.c), 1 << self.cp[1]) }; + if let Some(at) = hash.iter().zip(c_hash).position(|(a, b)| a != b) { panic!("{what}: hash table differs at {at}: got {}, expected {}", hash[at], c_hash[at]); } + if let Some(at) = chain.iter().zip(c_chain).position(|(a, b)| a != b) { panic!("{what}: chain table differs at {at}: got {}, expected {}", chain[at], c_chain[at]); } + } + /// Returns the state after the load. + fn load_dict(&mut self, dict: &[u8], full: bool, for_cdict: bool, force: bool) -> State { + let mut before = State::default(); + let filled = unsafe { shim_load_dict(self.c, dict.as_ptr().cast(), dict.len(), full as i32, for_cdict as i32, force as i32, &mut before) }; + if filled != 0 { + self.sync(&before); + let base = (before.base as usize).wrapping_sub(dict.as_ptr() as usize); + self.r.fill(dict, base, dict.len(), full, for_cdict).unwrap_or_else(|e| panic!("fill: error {e:?}")); + self.check_tables("fill"); + } + self.state() + } +} +impl Drop for Pair { fn drop(&mut self) { unsafe { shim_free(self.c) } } } + +#[derive(Default)] +pub struct Stats { pub blocks: [[u64; 3]; 2], pub seqs: [[u64; 3]; 2], pub fills: [[u64; 4]; 2] } + +fn splice(rng: &mut Rng, from: &[u8], to: &mut [u8], times: usize) { + if from.is_empty() || to.is_empty() { return; } + for _ in 0..times { + let n = rng.size(from.len().min(to.len())); + let at = rng.below(from.len() - n + 1); + let dst = rng.below(to.len() - n + 1); + to[dst..dst + n].copy_from_slice(&from[at..at + n]); + } +} + +fn one(rng: &mut Rng, stats: &mut Stats) { + let strategy = rng.range(1, 2) as u32; + let si = strategy as usize - 1; + let window_log = *rng.pick(&[10u32, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20]); + let log = |rng: &mut Rng| if rng.chance(1, 4) { rng.range(6, 9) } else { rng.range(8, 17) } as u32; + let min_match = rng.range(3, 7) as u32; + let target_length = *rng.pick(&[0u32, 0, 1, 2, 3, 4, 5, 6, 7, 8, 16, 32, 100, 999]); + let cp = [window_log, log(rng), log(rng), 1, min_match, target_length, strategy]; + let mut block_max = (1usize << window_log).min(128 << 10); + if rng.chance(1, 3) { block_max = block_max.min(1 << rng.range(10, 17)); } + + let total = { let m = *rng.pick(&[60usize, 600, 6000, 40_000, 40_000, 300_000]); rng.range(1, m) }; + let mut corpus = data(rng, total); + let kind = rng.below(5); // 0, 1: no dictionary. 2: loaded. 3, 4: attached. + let dict_len = if kind >= 2 { let m = *rng.pick(&[12usize, 100, 2000, 30_000, 200_000]); rng.range(1, m) } else { 0 }; + let mut dict = data(rng, dict_len); + if rng.chance(1, 2) { let t = rng.range(1, 6); splice(rng, &corpus, &mut dict, t); } + if rng.chance(3, 4) { let t = rng.range(1, 12); splice(rng, &dict, &mut corpus, t); } + if rng.chance(1, 2) { let t = rng.range(1, 8); let copy = corpus.clone(); splice(rng, ©, &mut corpus, t); } + + let mut pair = Pair::new(cp, block_max); + if rng.chance(1, 4) { + let index = *rng.pick(&[100u32, 70_000, 1 << 24, 1 << 31, 3_600_000_000]); + unsafe { shim_bump(pair.c, rng.range(2, index as usize) as u32) }; + } + + // Each region is a buffer of its own. `cur` is the one of the current segment of the window, `ext` of the older. + let mut regions: Vec> = Vec::new(); + let mut cur: Option = None; + let mut cur_end = 0usize; + let mut ext: Option = None; + let mut rep = [1u32, 4, 8]; + + let mut dict_pair: Option = None; + let mut dms_state = State::default(); + let mut attached = false; + if kind == 2 { + let room = if rng.chance(1, 2) { rng.size(total) + rng.below(20) } else { 0 }; + let mut mem = dict.clone(); + mem.resize(dict.len() + room, 0); + regions.push(mem); + let full = rng.chance(1, 3); + let force = rng.chance(1, 4); + let filled_before = pair.state(); + let _ = filled_before; + pair.load_dict(®ions[0][..dict.len()], full, false, force); + stats.fills[si][full as usize] += 1; + cur = Some(0); + cur_end = dict.len(); + } else if kind >= 3 { + let mut dcp = cp; + dcp[1] = log(rng); + dcp[2] = log(rng); + if rng.chance(1, 8) { dcp[4] = rng.range(3, 7) as u32; } + let mut d = Pair::new(dcp, 16); + let full = rng.chance(4, 5); + dms_state = d.load_dict(&dict, full, true, false); + stats.fills[si][2 + full as usize] += 1; + attached = unsafe { shim_attach(pair.c, d.c) } != 0; + dict_pair = Some(d); + if rng.chance(1, 3) { rep = [rng.range(1, dict.len()) as u32, rng.range(1, dict.len()) as u32, rng.range(1, dict.len()) as u32]; } + } + if kind < 3 && rng.chance(1, 4) { + let mut r = |rng: &mut Rng| { let m = *rng.pick(&[10usize, 1000, 100_000, u32::MAX as usize]); rng.range(1, m) as u32 }; + rep = [r(rng), r(rng), r(rng)]; + } + + let mut consumed = 0usize; + while consumed < corpus.len() { + let left = corpus.len() - consumed; + let n = if rng.chance(1, 3) { left } else { rng.range(1, left) }; + let chunk = &corpus[consumed..consumed + n]; + consumed += n; + + // Where the chunk goes. + let choice = rng.below(10); + let (region, at) = match cur { + Some(r) if choice < 6 && cur_end + n <= regions[r].len() => (r, cur_end), + Some(r) if choice >= 8 && n <= regions[r].len() => (r, if rng.chance(1, 2) { 0 } else { rng.below(regions[r].len() - n + 1) }), + _ => { + let room = if rng.chance(2, 3) { rng.size(left - n + 64) } else { 0 }; + let lead = if rng.chance(1, 4) { rng.below(40) } else { 0 }; + regions.push(vec![0; lead + n + room]); + (regions.len() - 1, lead) + } + }; + regions[region][at..at + n].copy_from_slice(chunk); + let ptr = regions[region].as_ptr(); + let contiguous = unsafe { shim_update(pair.c, ptr.add(at).cast(), n) } != 0; + if !contiguous { ext = cur; cur = Some(region); } + assert_eq!(cur, Some(region)); + cur_end = at + n; + + let mut pos = at; + while pos < cur_end { + let size = (cur_end - pos).min(if rng.chance(1, 4) { rng.range(1, block_max) } else { block_max }); + unsafe { shim_pre_block(pair.c, ptr.add(pos).cast(), size) }; + let st = pair.state(); + pair.sync(&st); + attached &= st.dict_mode == 2 || (st.dict_mode == 1 && st.loaded_dict_end != 0); + let buf = ®ions[region][..]; + let dict_buf: &[u8] = ext.map_or(&[], |r| ®ions[r][..]); + let base = (st.base as usize).wrapping_sub(buf.as_ptr() as usize); + let dict_base = (st.dict_base as usize).wrapping_sub(dict_buf.as_ptr() as usize); + let dms = if st.dict_mode == 2 { + let d = dict_pair.as_ref().unwrap(); + Some((&d.r, &dict[..], (dms_state.base as usize).wrapping_sub(dict.as_ptr() as usize), (dms_state.next_src as usize).wrapping_sub(dict.as_ptr() as usize))) + } else { None }; + + // Repeat offsets at the limit of what is valid. + if rng.chance(1, 5) { + let curr = (ptr as usize + pos).wrapping_sub(st.base as usize) as u32; + let reach = if st.dict_mode == 2 { + let dict_size = (dms_state.next_src - dms_state.base) as u32 - dms_state.dict_limit; + (curr - st.dict_limit + dict_size).saturating_sub(rng.below(3) as u32) + } else { + let lowest_valid = if rng.chance(1, 2) { st.low_limit } else { st.dict_limit }; + let from = if rng.chance(1, 2) { curr } else { curr + size as u32 }; + let low = if st.loaded_dict_end == 0 && from - lowest_valid > 1 << window_log { from - (1 << window_log) } else { lowest_valid }; + (curr - low + rng.size(64) as u32).saturating_sub(2) + }; + if reach != 0 { rep[rng.below(2)] = reach; } + } + let (mut rep_c, mut rep_r) = (rep, rep); + let expected = unsafe { shim_block(pair.c, rep_c.as_mut_ptr(), ptr.add(pos).cast(), size) }; + let what = format!("block {pos}..{} of region {region} (mode {}, {st:?})", pos + size, st.dict_mode); + let got = pair.r.block(&mut rep_r, st.dict_mode, buf, base, dict_buf, dict_base, pos, pos + size, dms) + .unwrap_or_else(|e| panic!("{what}: error {e:?}")); + + let c_seqs: Vec<(u32, u16, u16)> = unsafe { std::slice::from_raw_parts(shim_seqs(pair.c), shim_nb_seq(pair.c)) } + .iter().map(|s| (s.off_base, s.lit_length, s.ml_base)).collect(); + let r_seqs = pair.r.sequences(); + if r_seqs != c_seqs { + let i = r_seqs.iter().zip(&c_seqs).position(|(a, b)| a != b).unwrap_or(r_seqs.len().min(c_seqs.len())); + panic!("{what}: sequence {i} differs: got {:?}, expected {:?} ({} vs {})", r_seqs.get(i), c_seqs.get(i), r_seqs.len(), c_seqs.len()); + } + assert_eq!(got, expected, "{what}: last literals"); + assert_eq!(rep_r, rep_c, "{what}: rep"); + let c_lits = unsafe { std::slice::from_raw_parts(shim_lits(pair.c), shim_lit_len(pair.c)) }; + assert!(pair.r.literals() == c_lits, "{what}: literals"); + let long = unsafe { (shim_long_type(pair.c), shim_long_pos(pair.c)) }; + if long.0 != 0 || pair.r.long_length().0 != 0 { assert_eq!(pair.r.long_length(), long, "{what}: long length"); } + pair.check_tables(&what); + assert_eq!(pair.state().next_to_update, st.next_to_update); + + stats.blocks[si][st.dict_mode as usize] += 1; + stats.seqs[si][st.dict_mode as usize] += c_seqs.len() as u64; + if rng.chance(7, 8) { rep = rep_c; } + pos += size; + } + } + let _ = attached; +} + +pub fn run(seed: u64, iters: u64) { + let mut stats = Stats::default(); + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + if i % 2000 == 0 { eprintln!("stream iter {i} (seed {s})"); } + if std::env::var("ONLY").is_ok_and(|o| o != s.to_string()) { continue; } + CURRENT.store(s, std::sync::atomic::Ordering::Relaxed); + one(&mut Rng(s), &mut stats); + } + println!("stream ok: {iters} iterations from seed {seed}"); + for (si, name) in ["fast", "dfast"].iter().enumerate() { + println!(" {name}: blocks noDict/extDict/dictMatchState {:?}, sequences {:?}, fills CCtx-fast/CCtx-full/CDict-fast/CDict-full {:?}", + stats.blocks[si], stats.seqs[si], stats.fills[si]); + } +} + +pub static CURRENT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// bench : the time both sides take to find the sequences of the file, in blocks of 128 KB, with the +/// parameters of the level (which has to be of the `fast` or `dfast` strategy), without and with an attached dictionary. +pub fn bench(path: &str, cp: [u32; 7]) { + use std::time::{Duration, Instant}; + let all = std::fs::read(path).unwrap(); + for with_dms in [false, true] { + let (dict, src) = if with_dms { all.split_at(100_000.min(all.len() / 2)) } else { all.split_at(0) }; + let mut cp = cp; + if with_dms { cp[0] = 27; } + let (mut best_c, mut best_r) = (Duration::MAX, Duration::MAX); + let mut nb_seq = 0usize; + for _ in 0..5 { + let mut pair = Pair::new(cp, 128 << 10); + let mut d = Pair::new(cp, 16); + let mut dms_state = State::default(); + if with_dms { + dms_state = d.load_dict(dict, true, true, false); + unsafe { shim_attach(pair.c, d.c) }; + } + unsafe { shim_update(pair.c, src.as_ptr().cast(), src.len()) }; + let (mut time_c, mut time_r) = (Duration::ZERO, Duration::ZERO); + let (mut rep_c, mut rep_r) = ([1u32, 4, 8], [1u32, 4, 8]); + nb_seq = 0; + let mut pos = 0; + while pos < src.len() { + let size = (src.len() - pos).min(128 << 10); + unsafe { shim_pre_block(pair.c, src.as_ptr().add(pos).cast(), size) }; + let st = pair.state(); + pair.sync(&st); + let base = (st.base as usize).wrapping_sub(src.as_ptr() as usize); + let dms = (st.dict_mode == 2).then(|| (&d.r, dict, (dms_state.base as usize).wrapping_sub(dict.as_ptr() as usize), (dms_state.next_src as usize).wrapping_sub(dict.as_ptr() as usize))); + let t = Instant::now(); + let a = unsafe { shim_block(pair.c, rep_c.as_mut_ptr(), src.as_ptr().add(pos).cast(), size) }; + time_c += t.elapsed(); + let t = Instant::now(); + let b = pair.r.block(&mut rep_r, st.dict_mode, src, base, &[], 0, pos, pos + size, dms).unwrap(); + time_r += t.elapsed(); + assert_eq!(a, b); + nb_seq += unsafe { shim_nb_seq(pair.c) }; + pos += size; + } + best_c = best_c.min(time_c); + best_r = best_r.min(time_r); + } + let mbs = |d: Duration| src.len() as f64 / 1e6 / d.as_secs_f64(); + println!("cp {cp:?} dms {with_dms}: {} bytes, {nb_seq} sequences: C {:.0} MB/s, Rust {:.0} MB/s ({:.2}x the time)", + src.len(), mbs(best_c), mbs(best_r), best_r.as_secs_f64() / best_c.as_secs_f64()); + } +} diff --git a/src/zstd/wip/work/fast/zstd/compress/double_fast.rs b/src/zstd/wip/work/fast/zstd/compress/double_fast.rs new file mode 100644 index 000000000000..741c372c3cb6 --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/compress/double_fast.rs @@ -0,0 +1,785 @@ +//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. + +use super::fast::{FAST_HASH_FILL_STEP, Located, count_backwards, restore_rep}; +use super::internal::{ + BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, + write_tagged_index, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, read_le32, read_le64}; + +/// `ZSTD_fillDoubleHashTableForCDict` +fn fill_double_hash_table_for_cdict( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits_l = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let h_bits_s = ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; + let mls = ms.c_params.min_match; + let hash_large = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + // The positions in between only go in the table of long hashes, where there is nothing yet. + while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { + let curr = prefix.index_of(ip); + for i in 0..FAST_HASH_FILL_STEP { + let sm_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; + let lg_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; + if i == 0 { + write_tagged_index(hash_small, sm_hash_and_tag, curr + i)?; + } + if i == 0 || hash_large.at(lg_hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { + write_tagged_index(hash_large, lg_hash_and_tag, curr + i)?; + } + if dtlm == DictTableLoadMethod::Fast { + break; + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillDoubleHashTableForCCtx` +fn fill_double_hash_table_for_cctx( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let mls = ms.c_params.min_match; + let hash_large = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + // The positions in between only go in the table of long hashes, where there is nothing yet. + while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { + let curr = prefix.index_of(ip); + for i in 0..FAST_HASH_FILL_STEP { + let sm_hash = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; + let lg_hash = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; + if i == 0 { + hash_small.put(sm_hash, curr + i)?; + } + if i == 0 || hash_large.at(lg_hash)? == 0 { + hash_large.put(lg_hash, curr + i)?; + } + if dtlm == DictTableLoadMethod::Fast { + break; + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_double_hash_table( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, + tfp: TableFillPurpose, +) -> Result<()> { + match tfp { + TableFillPurpose::ForCDict => fill_double_hash_table_for_cdict(ms, prefix, end, dtlm), + TableFillPurpose::ForCCtx => fill_double_hash_table_for_cctx(ms, prefix, end, dtlm), + } +} + +/// `ZSTD_compressBlock_doubleFast_noDict_generic` +fn compress_block_double_fast_no_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; + + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let window_log = ms.c_params.window_log; + let buf = src.buf.sub(..src.end)?; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, window_log); + let prefix_lowest = src.pos_of(prefix_lowest_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip = istart + usize::from(istart == prefix_lowest); + let [mut offset_1, mut offset_2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let current = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(current, window_log); + let max_rep = current.wrapping_sub(window_low); + if offset_2 > max_rep { + offset_saved2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved1 = offset_1; + offset_1 = 0; + } + } + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + // One turn per match. + 'start: loop { + let mut step = 1usize; + let mut next_step = ip + K_STEP_INCR; + let mut ip1 = ip + step; + if ip1 > ilimit { + break 'start; + } + + let mut hl0 = hash_ptr(buf, ip, h_bits_l, 8)?; + let mut idxl0 = hash_long.at(hl0)?; + let mut curr; + + let m_length = 'match_stored: { + // One turn per position. Ends with the offset and the length of a match, which `ip` is moved to the start + // of. + let (offset, m_length, hl1) = loop { + let hs0 = hash_ptr(buf, ip, h_bits_s, MLS)?; + let idxs0 = hash_small.at(hs0)?; + curr = src.index_of(ip); + hash_long.put(hl0, curr)?; + hash_small.put(hs0, curr)?; + + if (offset_1 > 0) + & (read_le32(buf, (ip + 1).wrapping_sub(offset_1 as usize))? + == read_le32(buf, ip + 1)?) + { + let m_length = count( + buf, + ip + 1 + 4, + buf, + (ip + 1 + 4).wrapping_sub(offset_1 as usize), + iend, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break 'match_stored m_length; + } + + let hl1 = hash_ptr(buf, ip1, h_bits_l, 8)?; + + let matchl0 = src.pos_of(idxl0); + if idxl0 >= prefix_lowest_index && read_le64(buf, matchl0)? == read_le64(buf, ip)? { + let m_length = count(buf, ip + 8, buf, matchl0 + 8, iend)? + 8; + let offset = ip.wrapping_sub(matchl0) as u32; + let back = count_backwards(buf, ip, anchor, buf, matchl0, prefix_lowest); + ip -= back; + break (offset, m_length + back, hl1); + } + + let idxl1 = hash_long.at(hl1)?; + + let mut matchs0 = src.pos_of(idxs0); + if idxs0 >= prefix_lowest_index && read_le32(buf, matchs0)? == read_le32(buf, ip)? { + // A short match: a longer one at the next position is preferred. + let mut m_length = count(buf, ip + 4, buf, matchs0 + 4, iend)? + 4; + let mut offset = ip.wrapping_sub(matchs0) as u32; + let matchl1 = src.pos_of(idxl1); + if idxl1 > prefix_lowest_index + && read_le64(buf, matchl1)? == read_le64(buf, ip1)? + { + let l1len = count(buf, ip1 + 8, buf, matchl1 + 8, iend)? + 8; + if l1len > m_length { + ip = ip1; + m_length = l1len; + offset = ip.wrapping_sub(matchl1) as u32; + matchs0 = matchl1; + } + } + let back = count_backwards(buf, ip, anchor, buf, matchs0, prefix_lowest); + ip -= back; + break (offset, m_length + back, hl1); + } + + if ip1 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + ip = ip1; + ip1 += step; + + hl0 = hl1; + idxl0 = idxl1; + if ip1 > ilimit { + break 'start; + } + }; + + offset_2 = offset_1; + offset_1 = offset; + // With a larger step, ip1 may be past the end of the match. + if step < 4 { + hash_long.put(hl1, src.index_of(ip1))?; + } + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + m_length + }; + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit + && (offset_2 > 0) + & (read_le32(buf, ip)? == read_le32(buf, ip.wrapping_sub(offset_2 as usize))?) + { + let r_length = count( + buf, + ip + 4, + buf, + (ip + 4).wrapping_sub(offset_2 as usize), + iend, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, src.index_of(ip))?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, src.index_of(ip))?; + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; + ip += r_length; + anchor = ip; + } + } + } + + restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` +fn compress_block_double_fast_dict_match_state_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let buf = src.buf.sub(..src.end)?; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, ms.c_params.window_log); + let prefix_lowest = src.pos_of(prefix_lowest_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + let [mut offset_1, mut offset_2, _] = *rep; + + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_hash_long = dms.ms.hash_table.as_slice(); + let dict_hash_small = dms.ms.chain_table.as_slice(); + let dict_start_index = dms.ms.window.dict_limit; + let dict_start = dms.pos_of(dict_start_index); + let dict_end = dms.end; + // What to subtract from an index below prefix_lowest_index to get the index in the dictionary. + let dict_index_delta = prefix_lowest_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); + let dict_h_bits_l = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let dict_h_bits_s = dms.ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; + let dict_and_prefix_length = istart + .wrapping_sub(prefix_lowest) + .wrapping_add(dict_end.wrapping_sub(dict_start)) as u32; + + let locate = |index: u32| { + if index < prefix_lowest_index { + Located { + buf: dms.buf, + pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_lowest, + end: iend, + } + } + }; + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip = istart + usize::from(dict_and_prefix_length == 0); + + 'search: while ip < ilimit { + let h2 = hash_ptr(buf, ip, h_bits_l, 8)?; + let h = hash_ptr(buf, ip, h_bits_s, MLS)?; + let dict_hash_and_tag_l = hash_ptr(buf, ip, dict_h_bits_l, 8)?; + let dict_hash_and_tag_s = hash_ptr(buf, ip, dict_h_bits_s, MLS)?; + let dict_match_index_and_tag_l = + dict_hash_long.at(dict_hash_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_match_index_and_tag_s = + dict_hash_small.at(dict_hash_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_tags_match_l = + compare_packed_tags(dict_match_index_and_tag_l as usize, dict_hash_and_tag_l); + let dict_tags_match_s = + compare_packed_tags(dict_match_index_and_tag_s as usize, dict_hash_and_tag_s); + let curr = src.index_of(ip); + let match_index_l = hash_long.at(h2)?; + let mut match_index_s = hash_small.at(h)?; + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + hash_long.put(h2, curr)?; + hash_small.put(h, curr)?; + + let m_length = 'match_stored: { + if index_overlap_check(prefix_lowest_index, rep_index) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? + { + let m_length = count_2segments( + buf, + ip + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_lowest, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break 'match_stored m_length; + } + + // The offset and the length of a match, which `ip` is moved to the start of. + let (offset, m_length) = 'match_found: { + let match_long = src.pos_of(match_index_l); + if match_index_l >= prefix_lowest_index + && read_le64(buf, match_long)? == read_le64(buf, ip)? + { + let m_length = count(buf, ip + 8, buf, match_long + 8, iend)? + 8; + let offset = ip.wrapping_sub(match_long) as u32; + let back = count_backwards(buf, ip, anchor, buf, match_long, prefix_lowest); + ip -= back; + break 'match_found (offset, m_length + back); + } else if dict_tags_match_l { + let dict_match_index_l = + dict_match_index_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match_l = dms.pos_of(dict_match_index_l); + if dict_match_index_l > dict_start_index + && read_le64(dms.buf, dict_match_l)? == read_le64(buf, ip)? + { + let m_length = count_2segments( + buf, + ip + 8, + dms.buf, + dict_match_l + 8, + iend, + dict_end, + prefix_lowest, + )? + 8; + let offset = curr + .wrapping_sub(dict_match_index_l) + .wrapping_sub(dict_index_delta); + let back = + count_backwards(buf, ip, anchor, dms.buf, dict_match_l, dict_start); + ip -= back; + break 'match_found (offset, m_length + back); + } + } + + let short_found = if match_index_s > prefix_lowest_index { + read_le32(buf, src.pos_of(match_index_s))? == read_le32(buf, ip)? + } else if dict_tags_match_s { + let dict_match_index_s = + dict_match_index_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS; + match_index_s = dict_match_index_s.wrapping_add(dict_index_delta); + dict_match_index_s > dict_start_index + && read_le32(dms.buf, dms.pos_of(dict_match_index_s))? + == read_le32(buf, ip)? + } else { + false + }; + if !short_found { + ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; + continue 'search; + } + + // A short match: a long one at the next position is preferred. + let hl3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; + let dict_hash_and_tag_l3 = hash_ptr(buf, ip + 1, dict_h_bits_l, 8)?; + let match_index_l3 = hash_long.at(hl3)?; + let dict_match_index_and_tag_l3 = + dict_hash_long.at(dict_hash_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_tags_match_l3 = + compare_packed_tags(dict_match_index_and_tag_l3 as usize, dict_hash_and_tag_l3); + let match_l3 = src.pos_of(match_index_l3); + hash_long.put(hl3, curr + 1)?; + + if match_index_l3 >= prefix_lowest_index + && read_le64(buf, match_l3)? == read_le64(buf, ip + 1)? + { + let m_length = count(buf, ip + 9, buf, match_l3 + 8, iend)? + 8; + ip += 1; + let offset = ip.wrapping_sub(match_l3) as u32; + let back = count_backwards(buf, ip, anchor, buf, match_l3, prefix_lowest); + ip -= back; + break 'match_found (offset, m_length + back); + } else if dict_tags_match_l3 { + let dict_match_index_l3 = + dict_match_index_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match_l3 = dms.pos_of(dict_match_index_l3); + if dict_match_index_l3 > dict_start_index + && read_le64(dms.buf, dict_match_l3)? == read_le64(buf, ip + 1)? + { + let m_length = count_2segments( + buf, + ip + 1 + 8, + dms.buf, + dict_match_l3 + 8, + iend, + dict_end, + prefix_lowest, + )? + 8; + ip += 1; + let offset = curr + .wrapping_add(1) + .wrapping_sub(dict_match_index_l3) + .wrapping_sub(dict_index_delta); + let back = + count_backwards(buf, ip, anchor, dms.buf, dict_match_l3, dict_start); + ip -= back; + break 'match_found (offset, m_length + back); + } + } + + let match_s = locate(match_index_s); + let m_length = count_2segments( + buf, + ip + 4, + match_s.buf, + match_s.pos + 4, + iend, + match_s.end, + prefix_lowest, + )? + 4; + let offset = curr.wrapping_sub(match_index_s); + let back = + count_backwards(buf, ip, anchor, match_s.buf, match_s.pos, match_s.start); + ip -= back; + (offset, m_length + back) + }; + + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + m_length + }; + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit { + let current2 = src.index_of(ip); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !(index_overlap_check(prefix_lowest_index, rep_index2) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_lowest, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; + ip += rep_length2; + anchor = ip; + } + } + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast` +pub(crate) fn compress_block_double_fast( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_no_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_no_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_no_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_no_dict_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState` +pub(crate) fn compress_block_double_fast_dict_match_state( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_doubleFast_extDict_generic` +fn compress_block_double_fast_ext_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let buf = src.buf.sub(..src.end)?; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); + let dict_start_index = low_limit; + let dict_limit = ms.window.dict_limit; + let prefix_start_index = dict_limit.max(low_limit); + let prefix_start = src.pos_of(prefix_start_index); + let dict_start = src.dict_pos_of(dict_start_index); + let dict_end = src.dict_pos_of(prefix_start_index); + let [mut offset_1, mut offset_2, _] = *rep; + + // Nothing of the older segment is within reach. + if prefix_start_index == dict_start_index { + return compress_block_double_fast(ms, seq_store, rep, src); + } + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: src.dict_buf, + pos: src.dict_pos_of(index), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip = istart; + + while ip < ilimit { + let h_small = hash_ptr(buf, ip, h_bits_s, MLS)?; + let match_index = hash_small.at(h_small)?; + let match_s = locate(match_index); + + let h_long = hash_ptr(buf, ip, h_bits_l, 8)?; + let match_long_index = hash_long.at(h_long)?; + let match_long = locate(match_long_index); + + let curr = src.index_of(ip); + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + hash_long.put(h_long, curr)?; + hash_small.put(h_small, curr)?; + + let m_length; + if (index_overlap_check(prefix_start_index, rep_index) + & (offset_1 <= curr.wrapping_add(1).wrapping_sub(dict_start_index))) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? + { + m_length = count_2segments( + buf, + ip + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_start, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + } else { + // The match, how much of it is known, and its offset. `ip` is moved to where it is for. + let (found, known, offset) = if match_long_index > dict_start_index + && read_le64(match_long.buf, match_long.pos)? == read_le64(buf, ip)? + { + (match_long, 8, curr.wrapping_sub(match_long_index)) + } else if match_index > dict_start_index + && read_le32(match_s.buf, match_s.pos)? == read_le32(buf, ip)? + { + // A short match: a long one at the next position is preferred. + let h3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; + let match_index3 = hash_long.at(h3)?; + let match3 = locate(match_index3); + hash_long.put(h3, curr + 1)?; + if match_index3 > dict_start_index + && read_le64(match3.buf, match3.pos)? == read_le64(buf, ip + 1)? + { + ip += 1; + (match3, 8, curr.wrapping_add(1).wrapping_sub(match_index3)) + } else { + (match_s, 4, curr.wrapping_sub(match_index)) + } + } else { + ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; + continue; + }; + let forwards = count_2segments( + buf, + ip + known, + found.buf, + found.pos + known, + iend, + found.end, + prefix_start, + )? + known; + let back = count_backwards(buf, ip, anchor, found.buf, found.pos, found.start); + ip -= back; + m_length = forwards + back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + } + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit { + let current2 = src.index_of(ip); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !((index_overlap_check(prefix_start_index, rep_index2) + & (offset_2 <= current2.wrapping_sub(dict_start_index))) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; + ip += rep_length2; + anchor = ip; + } + } + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast_extDict` +pub(crate) fn compress_block_double_fast_ext_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), + } +} diff --git a/src/zstd/wip/work/fast/zstd/compress/fast.rs b/src/zstd/wip/work/fast/zstd/compress/fast.rs new file mode 100644 index 000000000000..f543cad9ae16 --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/compress/fast.rs @@ -0,0 +1,829 @@ +//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, + write_tagged_index, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, read_le32}; + +pub(super) const FAST_HASH_FILL_STEP: u32 = 3; + +/// Where an index leads, when it can be in one of two segments. +#[derive(Clone, Copy)] +pub(super) struct Located<'a> { + /// The buffer that the segment is in. + pub(super) buf: &'a [u8], + /// The position of the index in `buf`. + pub(super) pos: usize, + /// Where the segment starts in `buf`. + pub(super) start: usize, + /// Where it ends. + pub(super) end: usize, +} + +/// How many bytes before `ip`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. +#[inline(always)] +pub(super) fn count_backwards( + buf: &[u8], + ip: usize, + anchor: usize, + match_buf: &[u8], + match_pos: usize, + match_low: usize, +) -> usize { + let before_ip = buf.get(anchor..ip).unwrap_or_default(); + let before_match = match_buf.get(match_low..match_pos).unwrap_or_default(); + before_ip + .iter() + .rev() + .zip(before_match.iter().rev()) + .take_while(|(a, b)| a == b) + .count() +} + +/// The repeat offsets to leave at the end of a block, at the start of which those that reached too far were saved and +/// set to 0. +#[inline(always)] +pub(super) fn restore_rep( + rep: &mut [u32; ZSTD_REP_NUM], + offset_1: u32, + offset_2: u32, + offset_saved1: u32, + offset_saved2: u32, +) { + // If offset_1 started invalid and became valid, offset_saved1 is what rotates into the second place. + let offset_saved2 = if offset_saved1 != 0 && offset_1 != 0 { + offset_saved1 + } else { + offset_saved2 + }; + let [rep0, rep1, _] = rep; + *rep0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved1 + }; + *rep1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved2 + }; +} + +/// `ZSTD_fillHashTableForCDict` +fn fill_hash_table_for_cdict( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let mls = ms.c_params.min_match; + let hash_table = ms.hash_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { + let curr = prefix.index_of(ip); + write_tagged_index(hash_table, hash_ptr(buf, ip, h_bits, mls)?, curr)?; + if dtlm == DictTableLoadMethod::Full { + // The positions in between only go where there is nothing yet. + for p in 1..FAST_HASH_FILL_STEP { + let hash_and_tag = hash_ptr(buf, ip + p as usize, h_bits, mls)?; + if hash_table.at(hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { + write_tagged_index(hash_table, hash_and_tag, curr + p)?; + } + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillHashTableForCCtx` +fn fill_hash_table_for_cctx( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits = ms.c_params.hash_log; + let mls = ms.c_params.min_match; + let hash_table = ms.hash_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { + let curr = prefix.index_of(ip); + hash_table.put(hash_ptr(buf, ip, h_bits, mls)?, curr)?; + if dtlm == DictTableLoadMethod::Full { + // The positions in between only go where there is nothing yet. + for p in 1..FAST_HASH_FILL_STEP { + let hash = hash_ptr(buf, ip + p as usize, h_bits, mls)?; + if hash_table.at(hash)? == 0 { + hash_table.put(hash, curr + p)?; + } + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_hash_table( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, + tfp: TableFillPurpose, +) -> Result<()> { + match tfp { + TableFillPurpose::ForCDict => fill_hash_table_for_cdict(ms, prefix, end, dtlm), + TableFillPurpose::ForCCtx => fill_hash_table_for_cctx(ms, prefix, end, dtlm), + } +} + +/// `ZSTD_match4Found_branch`, and `ZSTD_match4Found_cmov`, which only differs in how it avoids reading at an index that +/// is not valid. +#[inline(always)] +fn match4_found( + buf: &[u8], + current: usize, + match_pos: usize, + match_idx: u32, + idx_low_limit: u32, +) -> Result { + Ok(match_idx >= idx_low_limit && read_le32(buf, current)? == read_le32(buf, match_pos)?) +} + +/// `ZSTD_compressBlock_fast_noDict_generic` +/// +/// Each turn of the inner loop looks at two positions, `ip0` and `ip1`, for a match, and at `ip2` for a repeat. The +/// hash of a position is computed, and the table is read, one position ahead of the comparison. +fn compress_block_fast_no_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); + + let hlog = ms.c_params.hash_log; + let window_log = ms.c_params.window_log; + let target_length = ms.c_params.target_length; + // The least step is 2, as two positions are looked at per turn. + let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; + let buf = src.buf.sub(..src.end)?; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_start_index = ms.get_lowest_prefix_index(end_index, window_log); + let prefix_start = src.pos_of(prefix_start_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip0 = istart + usize::from(istart == prefix_start); + let [mut rep_offset1, mut rep_offset2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let curr = src.index_of(ip0); + let window_low = ms.get_lowest_prefix_index(curr, window_log); + let max_rep = curr.wrapping_sub(window_low); + if rep_offset2 > max_rep { + offset_saved2 = rep_offset2; + rep_offset2 = 0; + } + if rep_offset1 > max_rep { + offset_saved1 = rep_offset1; + rep_offset1 = 0; + } + } + let hash_table = ms.hash_table.as_mut_slice(); + + 'start: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let mut ip1 = ip0 + 1; + let mut ip2 = ip0 + step; + let mut ip3 = ip2 + 1; + if ip3 >= ilimit { + break 'start; + } + + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let mut match_idx = hash_table.at(hash0)?; + let mut current0; + + // Ends with the start and the length so far of a repeat match, or with `None` for a match at `match_idx`. + let rep_match = loop { + let rval = read_le32(buf, ip2.wrapping_sub(rep_offset1 as usize))?; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if (read_le32(buf, ip2)? == rval) & (rep_offset1 > 0) { + ip0 = ip2; + let match0 = ip0.wrapping_sub(rep_offset1 as usize); + let back = usize::from(buf.at(ip0 - 1)? == buf.at(match0.wrapping_sub(1))?); + ip0 -= back; + // The hash of ip1 is there, and ip1 is before the end of the match. + hash_table.put(hash1, src.index_of(ip1))?; + break Some((match0.wrapping_sub(back), 4 + back)); + } + + if match4_found( + buf, + ip0, + src.pos_of(match_idx), + match_idx, + prefix_start_index, + )? { + hash_table.put(hash1, src.index_of(ip1))?; + break None; + } + + match_idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if match4_found( + buf, + ip0, + src.pos_of(match_idx), + match_idx, + prefix_start_index, + )? { + // ip1 is certain to be before the end of the match only if the step is small. + if step <= 4 { + hash_table.put(hash1, src.index_of(ip1))?; + } + break None; + } + + match_idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + if ip2 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + if ip3 >= ilimit { + break 'start; + } + }; + + let (offcode, match0, mut m_length) = match rep_match { + Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), + None => { + let match0 = src.pos_of(match_idx); + rep_offset2 = rep_offset1; + rep_offset1 = ip0.wrapping_sub(match0) as u32; + let back = count_backwards(buf, ip0, anchor, buf, match0, prefix_start); + ip0 -= back; + (offset_to_offbase(rep_offset1), match0 - back, 4 + back) + } + }; + + m_length += count(buf, ip0 + m_length, buf, match0 + m_length, iend)?; + seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; + ip0 += m_length; + anchor = ip0; + + if ip0 <= ilimit { + hash_table.put( + hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, + current0 + 2, + )?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + if rep_offset2 > 0 { + while ip0 <= ilimit + && read_le32(buf, ip0)? + == read_le32(buf, ip0.wrapping_sub(rep_offset2 as usize))? + { + let r_length = count( + buf, + ip0 + 4, + buf, + (ip0 + 4).wrapping_sub(rep_offset2 as usize), + iend, + )? + 4; + core::mem::swap(&mut rep_offset1, &mut rep_offset2); + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; + ip0 += r_length; + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; + anchor = ip0; + } + } + } + } + + restore_rep(rep, rep_offset1, rep_offset2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast` +pub(crate) fn compress_block_fast( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_no_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_no_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_no_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_no_dict_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_fast_dictMatchState_generic` +fn compress_block_fast_dict_match_state_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; + + let hlog = ms.c_params.hash_log; + let target_length = ms.c_params.target_length; + let step_size = (target_length + u32::from(target_length == 0)) as usize; + let buf = src.buf.sub(..src.end)?; + let istart = src.start; + let iend = src.end; + let prefix_start_index = ms.window.dict_limit; + let prefix_start = src.pos_of(prefix_start_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + let [mut offset_1, mut offset_2, _] = *rep; + + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_hash_table = dms.ms.hash_table.as_slice(); + let dict_start_index = dms.ms.window.dict_limit; + let dict_start = dms.pos_of(dict_start_index); + let dict_end = dms.end; + // What to subtract from an index below prefix_start_index to get the index in the dictionary. + let dict_index_delta = prefix_start_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); + let dict_and_prefix_length = istart + .wrapping_sub(prefix_start) + .wrapping_add(dict_end) + .wrapping_sub(dict_start) as u32; + let dict_h_bits = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: dms.buf, + pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + let hash_table = ms.hash_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip0 = istart + usize::from(dict_and_prefix_length == 0); + let mut ip1 = istart + step_size; + + 'search: while ip1 <= ilimit { + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + + let dict_hash_and_tag0 = hash_ptr(buf, ip0, dict_h_bits, MLS)?; + let mut dict_match_index_and_tag = + dict_hash_table.at(dict_hash_and_tag0 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let mut dict_tags_match = + compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag0); + + let mut match_index = hash_table.at(hash0)?; + let mut curr = src.index_of(ip0); + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let m_length = loop { + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + let hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let dict_hash_and_tag1 = hash_ptr(buf, ip1, dict_h_bits, MLS)?; + hash_table.put(hash0, curr)?; + + if index_overlap_check(prefix_start_index, rep_index) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip0 + 1)? + { + let m_length = count_2segments( + buf, + ip0 + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_start, + )? + 4; + ip0 += 1; + seq_store.store_seq(ip0 - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break m_length; + } + + if dict_tags_match { + let dict_match_index = dict_match_index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match = dms.pos_of(dict_match_index); + // A match in the dictionary is only taken if there is no candidate in the prefix. + if dict_match_index > dict_start_index + && read_le32(dms.buf, dict_match)? == read_le32(buf, ip0)? + && match_index <= prefix_start_index + { + let offset = curr + .wrapping_sub(dict_match_index) + .wrapping_sub(dict_index_delta); + let mut m_length = count_2segments( + buf, + ip0 + 4, + dms.buf, + dict_match + 4, + iend, + dict_end, + prefix_start, + )? + 4; + let back = count_backwards(buf, ip0, anchor, dms.buf, dict_match, dict_start); + ip0 -= back; + m_length += back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip0 - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + break m_length; + } + } + + let match_pos = src.pos_of(match_index); + if match4_found(buf, ip0, match_pos, match_index, prefix_start_index)? { + let offset = ip0.wrapping_sub(match_pos) as u32; + let mut m_length = count(buf, ip0 + 4, buf, match_pos + 4, iend)? + 4; + let back = count_backwards(buf, ip0, anchor, buf, match_pos, prefix_start); + ip0 -= back; + m_length += back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip0 - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + break m_length; + } + + dict_match_index_and_tag = + dict_hash_table.at(dict_hash_and_tag1 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + dict_tags_match = + compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag1); + match_index = hash_table.at(hash1)?; + + if ip1 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + ip0 = ip1; + ip1 += step; + if ip1 > ilimit { + break 'search; + } + + curr = src.index_of(ip0); + hash0 = hash1; + }; + + ip0 += m_length; + anchor = ip0; + + if ip0 <= ilimit { + hash_table.put(hash_ptr(buf, src.pos_of(curr + 2), hlog, MLS)?, curr + 2)?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + while ip0 <= ilimit { + let current2 = src.index_of(ip0); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !(index_overlap_check(prefix_start_index, rep_index2) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip0 + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, current2)?; + ip0 += rep_length2; + anchor = ip0; + } + } + + ip1 = ip0 + step_size; + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast_dictMatchState` +pub(crate) fn compress_block_fast_dict_match_state( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_fast_extDict_generic` +fn compress_block_fast_ext_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); + + let hlog = ms.c_params.hash_log; + let target_length = ms.c_params.target_length; + let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; + let buf = src.buf.sub(..src.end)?; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); + let dict_start_index = low_limit; + let dict_start = src.dict_pos_of(dict_start_index); + let dict_limit = ms.window.dict_limit; + let prefix_start_index = dict_limit.max(low_limit); + let prefix_start = src.pos_of(prefix_start_index); + let dict_end = src.dict_pos_of(prefix_start_index); + + // Nothing of the older segment is within reach. + if prefix_start_index == dict_start_index { + return compress_block_fast(ms, seq_store, rep, src); + } + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip0 = istart; + let [mut offset_1, mut offset_2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let curr = src.index_of(ip0); + let max_rep = curr.wrapping_sub(dict_start_index); + if offset_2 >= max_rep { + offset_saved2 = offset_2; + offset_2 = 0; + } + if offset_1 >= max_rep { + offset_saved1 = offset_1; + offset_1 = 0; + } + } + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: src.dict_buf, + pos: src.dict_pos_of(index), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + // Whether there is a match for the position `ip` at `idx`, which is from the table. + let match_found = |ip: usize, idx: u32| -> Result { + let candidate = locate(idx); + Ok(idx >= dict_start_index + && read_le32(candidate.buf, candidate.pos)? == read_le32(buf, ip)?) + }; + let hash_table = ms.hash_table.as_mut_slice(); + + 'start: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let mut ip1 = ip0 + 1; + let mut ip2 = ip0 + step; + let mut ip3 = ip2 + 1; + if ip3 >= ilimit { + break 'start; + } + + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let mut idx = hash_table.at(hash0)?; + let mut current0; + + // Ends with the start and the length so far of a repeat match, or with `None` for a match at `idx`. + let rep_match = loop { + let current2 = src.index_of(ip2); + let rep_index = current2.wrapping_sub(offset_1); + let mut rep_match = locate(rep_index); + // The 4 bytes have to be in one segment, with a byte before them. + let rep_found = ((prefix_start_index.wrapping_sub(rep_index) >= 4) & (offset_1 > 0)) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip2)?; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if rep_found { + ip0 = ip2; + let back = usize::from( + buf.at(ip0 - 1)? == rep_match.buf.at(rep_match.pos.wrapping_sub(1))?, + ); + ip0 -= back; + rep_match.pos = rep_match.pos.wrapping_sub(back); + break Some((rep_match, 4 + back)); + } + + if match_found(ip0, idx)? { + break None; + } + + idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if match_found(ip0, idx)? { + break None; + } + + idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + if ip2 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + if ip3 >= ilimit { + break 'start; + } + }; + + let (offcode, match0, mut m_length) = match rep_match { + Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), + None => { + let offset = current0.wrapping_sub(idx); + let mut match0 = locate(idx); + offset_2 = offset_1; + offset_1 = offset; + let back = count_backwards(buf, ip0, anchor, match0.buf, match0.pos, match0.start); + ip0 -= back; + match0.pos -= back; + (offset_to_offbase(offset), match0, 4 + back) + } + }; + + m_length += count_2segments( + buf, + ip0 + m_length, + match0.buf, + match0.pos + m_length, + iend, + match0.end, + prefix_start, + )?; + seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; + ip0 += m_length; + anchor = ip0; + + if ip1 < ip0 { + hash_table.put(hash1, src.index_of(ip1))?; + } + + if ip0 <= ilimit { + hash_table.put( + hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, + current0 + 2, + )?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + while ip0 <= ilimit { + let rep_index2 = src.index_of(ip0).wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !((index_overlap_check(prefix_start_index, rep_index2) & (offset_2 > 0)) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip0 + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; + ip0 += rep_length2; + anchor = ip0; + } + } + } + + restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast_extDict` +pub(crate) fn compress_block_fast_ext_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), + } +} diff --git a/src/zstd/wip/work/fast/zstd/compress/fse.rs b/src/zstd/wip/work/fast/zstd/compress/fse.rs new file mode 100644 index 000000000000..1b8f7cbf3173 --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/compress/fse.rs @@ -0,0 +1,70 @@ +//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. + +use crate::common::{MAX_SEQ, ML_FSE_LOG}; +use crate::util::SliceExt; + +/// `FSE_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum FseRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `FSE_symbolCompressionTransform` +#[derive(Clone, Copy, Default)] +pub(crate) struct FseSymbolTransform { + pub(crate) delta_find_state: i32, + pub(crate) delta_nb_bits: u32, +} + +/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. +#[derive(Clone)] +pub(crate) struct FseCTable { + pub(crate) table_log: u32, + pub(crate) max_symbol_value: u32, + pub(crate) state_table: [u16; 1 << ML_FSE_LOG], + pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], +} + +impl FseCTable { + pub(crate) const fn new() -> Self { + Self { + table_log: 0, + max_symbol_value: 0, + state_table: [0; 1 << ML_FSE_LOG], + symbol_tt: [FseSymbolTransform { + delta_find_state: 0, + delta_nb_bits: 0, + }; MAX_SEQ as usize + 1], + } + } + + /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log + /// if the table has no code for it. + #[inline(always)] + pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 + } + + /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. + /// The table has a code for it. + #[inline(always)] + pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + let min_nb_bits = transform.delta_nb_bits >> 16; + let threshold = (min_nb_bits + 1) << 16; + let table_size = 1u32 << self.table_log; + let delta_from_threshold = + threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); + // A linear interpolation, which is very approximate. + let normalized_delta_from_threshold = + (delta_from_threshold << accuracy_log) >> self.table_log; + let bit_multiplier = 1u32 << accuracy_log; + ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) + } +} diff --git a/src/zstd/wip/work/fast/zstd/compress/hist.rs b/src/zstd/wip/work/fast/zstd/compress/hist.rs new file mode 100644 index 000000000000..becf42516867 --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/compress/hist.rs @@ -0,0 +1,33 @@ +//! `hist.c`: how many times each byte value occurs. + +use crate::error::Result; +use crate::util::SliceExt; + +/// `HIST_add`: `count` has room for every value in `src`. +pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { + for &byte in src { + *count.at_mut(usize::from(byte))? += 1; + } + Ok(()) +} + +/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. +/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. +pub(crate) fn count_simple( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + let mut max_symbol = *max_symbol_value as usize; + count.sub_mut(..=max_symbol)?.fill(0); + if src.is_empty() { + *max_symbol_value = 0; + return Ok(0); + } + add(count, src)?; + while count.at(max_symbol)? == 0 { + max_symbol -= 1; + } + *max_symbol_value = max_symbol as u32; + Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) +} diff --git a/src/zstd/wip/work/fast/zstd/compress/huf.rs b/src/zstd/wip/work/fast/zstd/compress/huf.rs new file mode 100644 index 000000000000..78373a89ebf0 --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/compress/huf.rs @@ -0,0 +1,38 @@ +//! `huf_compress.c`: Huffman coding of literals. + +use crate::common::HUF_SYMBOLVALUE_MAX; +use crate::util::SliceExt; + +/// `HUF_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum HufRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the +/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in +/// its highest bits. +#[derive(Clone)] +pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); + +impl HufCTable { + pub(crate) const fn new() -> Self { + Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) + } + + /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. + #[inline] + pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { + let [header, elements @ ..] = &self.0; + let max_symbol_value = (header >> 8) as u8; + if symbol_value > u32::from(max_symbol_value) { + return 0; + } + (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 + } +} diff --git a/src/zstd/wip/work/fast/zstd/compress/internal.rs b/src/zstd/wip/work/fast/zstd/compress/internal.rs new file mode 100644 index 000000000000..51157ea6c1ce --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/compress/internal.rs @@ -0,0 +1,1212 @@ +//! `zstd_compress_internal.h`: what the parts of the compressor share. + +use super::fse::{FseCTable, FseRepeat}; +use super::huf::{HufCTable, HufRepeat}; +use crate::common::{ + BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, + ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; + +pub(crate) const K_SEARCH_STRENGTH: u32 = 8; +pub(crate) const HASH_READ_SIZE: usize = 8; +/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. +pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; + +pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; +pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; + +pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; +pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; + +/// `ZSTD_strategy` +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub(crate) enum Strategy { + Fast = 1, + DFast = 2, + Greedy = 3, + Lazy = 4, + Lazy2 = 5, + BtLazy2 = 6, + BtOpt = 7, + BtUltra = 8, + BtUltra2 = 9, +} + +impl Strategy { + pub(crate) const fn from_int(value: i32) -> Option { + Some(match value { + 1 => Self::Fast, + 2 => Self::DFast, + 3 => Self::Greedy, + 4 => Self::Lazy, + 5 => Self::Lazy2, + 6 => Self::BtLazy2, + 7 => Self::BtOpt, + 8 => Self::BtUltra, + 9 => Self::BtUltra2, + _ => return None, + }) + } +} + +/// `ZSTD_ParamSwitch_e` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum ParamSwitch { + Auto = 0, + Enable = 1, + Disable = 2, +} + +/// `ZSTD_compressionParameters` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct CompressionParameters { + pub(crate) window_log: u32, + pub(crate) chain_log: u32, + pub(crate) hash_log: u32, + pub(crate) search_log: u32, + pub(crate) min_match: u32, + pub(crate) target_length: u32, + /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. + pub(crate) strategy: Option, +} + +impl CompressionParameters { + /// The strategy of parameters that are ready to be applied. + #[inline(always)] + pub(crate) fn strat(&self) -> Strategy { + self.strategy.unwrap_or(Strategy::Fast) + } +} + +/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum DictMode { + NoDict, + ExtDict, + DictMatchState, +} + +/// `ZSTD_dictTableLoadMethod_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DictTableLoadMethod { + Fast, + Full, +} + +/// `ZSTD_tableFillPurpose_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TableFillPurpose { + ForCCtx, + ForCDict, +} + +// ── entropy tables ──────────────────────────────────────────────────────── + +/// `ZSTD_hufCTables_t` +#[derive(Clone)] +pub(crate) struct HufCTables { + pub(crate) c_table: HufCTable, + pub(crate) repeat_mode: HufRepeat, +} + +/// `ZSTD_fseCTables_t` +#[derive(Clone)] +pub(crate) struct FseCTables { + pub(crate) offcode_c_table: FseCTable, + pub(crate) matchlength_c_table: FseCTable, + pub(crate) litlength_c_table: FseCTable, + pub(crate) offcode_repeat_mode: FseRepeat, + pub(crate) matchlength_repeat_mode: FseRepeat, + pub(crate) litlength_repeat_mode: FseRepeat, +} + +/// `ZSTD_entropyCTables_t` +#[derive(Clone)] +pub(crate) struct EntropyCTables { + pub(crate) huf: HufCTables, + pub(crate) fse: FseCTables, +} + +/// `ZSTD_compressedBlockState_t` +#[derive(Clone)] +pub(crate) struct CompressedBlockState { + pub(crate) entropy: EntropyCTables, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +impl CompressedBlockState { + /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. + pub(crate) fn new() -> Self { + Self { + entropy: EntropyCTables { + huf: HufCTables { + c_table: HufCTable::new(), + repeat_mode: HufRepeat::None, + }, + fse: FseCTables { + offcode_c_table: FseCTable::new(), + matchlength_c_table: FseCTable::new(), + litlength_c_table: FseCTable::new(), + offcode_repeat_mode: FseRepeat::None, + matchlength_repeat_mode: FseRepeat::None, + litlength_repeat_mode: FseRepeat::None, + }, + }, + rep: REP_START_VALUE, + } + } + + /// `ZSTD_reset_compressedBlockState` + pub(crate) fn reset(&mut self) { + self.rep = REP_START_VALUE; + self.entropy.huf.repeat_mode = HufRepeat::None; + self.entropy.fse.offcode_repeat_mode = FseRepeat::None; + self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; + self.entropy.fse.litlength_repeat_mode = FseRepeat::None; + } +} + +/// `ZSTD_hufCTablesMetadata_t` +pub(crate) struct HufCTablesMetadata { + pub(crate) h_type: SymbolEncodingType, + pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], + pub(crate) huf_des_size: usize, +} + +/// `ZSTD_fseCTablesMetadata_t` +pub(crate) struct FseCTablesMetadata { + pub(crate) ll_type: SymbolEncodingType, + pub(crate) of_type: SymbolEncodingType, + pub(crate) ml_type: SymbolEncodingType, + pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], + pub(crate) fse_tables_size: usize, + /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. + pub(crate) last_count_size: usize, +} + +/// `ZSTD_entropyCTablesMetadata_t` +pub(crate) struct EntropyCTablesMetadata { + pub(crate) huf_metadata: HufCTablesMetadata, + pub(crate) fse_metadata: FseCTablesMetadata, +} + +impl EntropyCTablesMetadata { + pub(crate) const fn new() -> Self { + Self { + huf_metadata: HufCTablesMetadata { + h_type: SymbolEncodingType::Basic, + huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], + huf_des_size: 0, + }, + fse_metadata: FseCTablesMetadata { + ll_type: SymbolEncodingType::Basic, + of_type: SymbolEncodingType::Basic, + ml_type: SymbolEncodingType::Basic, + fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], + fse_tables_size: 0, + last_count_size: 0, + }, + } + } +} + +// ── sequences ───────────────────────────────────────────────────────────── + +/// `SeqDef` +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct SeqDef { + /// A repeat code (1 to 3), or an offset plus 3. + pub(crate) off_base: u32, + pub(crate) lit_length: u16, + /// The match length minus [`MINMATCH`]. + pub(crate) ml_base: u16, +} + +/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum LongLengthType { + None, + LiteralLength, + MatchLength, +} + +/// `SeqStore_t`: the sequences and the literals of a block. +pub(crate) struct SeqStore { + /// Never grows beyond the capacity it is given by [`Self::reserve`]. + pub(crate) sequences: Vec, + /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. + pub(crate) lit_buffer: Vec, + /// How much of `lit_buffer` is literals. + pub(crate) lit_len: usize, + pub(crate) ll_code: Vec, + pub(crate) ml_code: Vec, + pub(crate) of_code: Vec, + pub(crate) max_nb_seq: usize, + pub(crate) max_nb_lit: usize, + pub(crate) long_length_type: LongLengthType, + /// Which sequence has the long length. + pub(crate) long_length_pos: u32, +} + +/// `ZSTD_SequenceLength` +#[derive(Clone, Copy)] +pub(crate) struct SequenceLength { + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +impl SeqStore { + pub(crate) const fn new() -> Self { + Self { + sequences: Vec::new(), + lit_buffer: Vec::new(), + lit_len: 0, + ll_code: Vec::new(), + ml_code: Vec::new(), + of_code: Vec::new(), + max_nb_seq: 0, + max_nb_lit: 0, + long_length_type: LongLengthType::None, + long_length_pos: 0, + } + } + + /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. + pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { + fn grow(vec: &mut Vec, len: usize) -> Result<()> { + vec.clear(); + vec.try_reserve_exact(len) + .map_err(|_| ErrorCode::MemoryAllocation)?; + vec.resize(len, T::default()); + Ok(()) + } + grow(&mut self.sequences, max_nb_seq)?; + grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; + grow(&mut self.ll_code, max_nb_seq)?; + grow(&mut self.ml_code, max_nb_seq)?; + grow(&mut self.of_code, max_nb_seq)?; + self.max_nb_seq = max_nb_seq; + self.max_nb_lit = max_nb_lit; + self.reset(); + Ok(()) + } + + /// `ZSTD_resetSeqStore` + #[inline] + pub(crate) fn reset(&mut self) { + self.lit_len = 0; + self.sequences.clear(); + self.long_length_type = LongLengthType::None; + } + + #[inline] + pub(crate) fn literals(&self) -> &[u8] { + self.lit_buffer.get(..self.lit_len).unwrap_or_default() + } + + /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. + #[inline(always)] + pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { + debug_assert!(self.sequences.len() < self.max_nb_seq); + if lit_length > 0xFFFF { + self.long_length_type = LongLengthType::LiteralLength; + self.long_length_pos = self.sequences.len() as u32; + } + let ml_base = match_length.wrapping_sub(MINMATCH as usize); + if ml_base > 0xFFFF { + self.long_length_type = LongLengthType::MatchLength; + self.long_length_pos = self.sequences.len() as u32; + } + self.sequences.push(SeqDef { + off_base, + lit_length: lit_length as u16, + ml_base: ml_base as u16, + }); + } + + /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. + #[inline(always)] + pub(crate) fn store_seq( + &mut self, + lit_length: usize, + buf: &[u8], + literals: usize, + off_base: u32, + match_length: usize, + ) -> Result<()> { + match ( + buf.array_at::<16>(literals), + self.lit_buffer.array_at_mut::<16>(self.lit_len), + ) { + (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, + _ => self + .lit_buffer + .write_at(self.lit_len, buf.span(literals, lit_length)?)?, + } + self.lit_len += lit_length; + self.store_seq_only(lit_length, off_base, match_length); + Ok(()) + } + + /// Appends the literals that follow the last sequence of a block. + #[inline] + pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { + self.lit_buffer.write_at(self.lit_len, literals)?; + self.lit_len += literals.len(); + Ok(()) + } +} + +pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; +pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; +pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; + +/// `OFFSET_TO_OFFBASE`: `offset` is not 0. +#[inline(always)] +pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { + offset.wrapping_add(ZSTD_REP_NUM as u32) +} + +/// `OFFBASE_IS_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { + off_base > ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_IS_REPCODE` +#[inline(always)] +pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { + 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_TO_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { + off_base.wrapping_sub(ZSTD_REP_NUM as u32) +} + +/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. +#[inline(always)] +pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { + let [rep0, rep1, rep2] = *rep; + if offbase_is_offset(off_base) { + *rep = [offbase_to_offset(off_base), rep0, rep1]; + } else { + match off_base - 1 + u32::from(ll0) { + 0 => {} + 1 => *rep = [rep1, rep0, rep2], + 2 => *rep = [rep2, rep0, rep1], + _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], + } + } +} + +/// `ZSTD_newRep` +#[inline(always)] +pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { + let mut new_reps = *rep; + update_rep(&mut new_reps, off_base, ll0); + new_reps +} + +/// `ZSTD_LLcode` +#[inline(always)] +pub(crate) fn ll_code(lit_length: u32) -> u32 { + #[rustfmt::skip] + const LL_CODE: [u8; 64] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + ]; + const LL_DELTA_CODE: u32 = 19; + match LL_CODE.get(lit_length as usize) { + Some(&code) => u32::from(code), + None => highbit32(lit_length) + LL_DELTA_CODE, + } +} + +/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. +#[inline(always)] +pub(crate) fn ml_code(ml_base: u32) -> u32 { + #[rustfmt::skip] + const ML_CODE: [u8; 128] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ]; + const ML_DELTA_CODE: u32 = 36; + match ML_CODE.get(ml_base as usize) { + Some(&code) => u32::from(code), + None => highbit32(ml_base) + ML_DELTA_CODE, + } +} + +/// `ZSTD_noCompressBlock` +pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { + let c_block_header24 = + u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); + if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header24)?; + dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; + Ok(ZSTD_BLOCKHEADERSIZE + src.len()) +} + +/// `ZSTD_rleCompressBlock` +pub(crate) fn rle_compress_block( + dst: &mut [u8], + byte: u8, + src_size: usize, + last_block: bool, +) -> Result { + let c_block_header = + u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); + if dst.len() < 4 { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header)?; + dst.put(3, byte)?; + Ok(4) +} + +/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. +#[inline] +pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { + let minlog = if strat >= Strategy::BtUltra { + strat as u32 - 1 + } else { + 6 + }; + (src_size >> minlog) + 2 +} + +// ── matching ────────────────────────────────────────────────────────────── + +/// How many bytes `a` and `b` have in common at their start. +#[inline(always)] +pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { + let (a_words, _) = a.as_chunks::<8>(); + let (b_words, _) = b.as_chunks::<8>(); + let mut len = 0usize; + for (a_word, b_word) in a_words.iter().zip(b_words) { + let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); + if diff != 0 { + return len + (diff.trailing_zeros() >> 3) as usize; + } + len += 8; + } + let a_tail = a.get(len..).unwrap_or_default(); + let b_tail = b.get(len..).unwrap_or_default(); + len + a_tail + .iter() + .zip(b_tail) + .take_while(|(a_byte, b_byte)| a_byte == b_byte) + .count() +} + +/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. +#[inline(always)] +pub(crate) fn count( + buf: &[u8], + p_in: usize, + match_buf: &[u8], + p_match: usize, + p_in_limit: usize, +) -> Result { + Ok(common_prefix( + buf.sub(p_in..p_in_limit)?, + match_buf.sub(p_match..)?, + )) +} + +/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at +/// `i_start` in `buf`. +#[inline(always)] +pub(crate) fn count_2segments( + buf: &[u8], + ip: usize, + match_buf: &[u8], + match_pos: usize, + i_end: usize, + m_end: usize, + i_start: usize, +) -> Result { + let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); + if match_pos + match_length != m_end { + return Ok(match_length); + } + Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) +} + +const PRIME_3_BYTES: u32 = 506_832_829; +const PRIME_4_BYTES: u32 = 2_654_435_761; +const PRIME_5_BYTES: u64 = 889_523_592_379; +const PRIME_6_BYTES: u64 = 227_718_039_650_203; +const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; +pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; + +/// `ZSTD_hash3` +#[inline(always)] +const fn hash3(u: u32, h: u32, s: u32) -> u32 { + (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash4(u: u32, h: u32, s: u32) -> u32 { + ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash5(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash6(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash7(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +/// `ZSTD_hash8` +#[inline(always)] +pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { + (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} + +/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. +#[inline(always)] +pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { + Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) +} + +/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, +/// else 8. +#[inline(always)] +pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { + hash_ptr_salted(buf, pos, h_bits, mls, 0) +} + +/// `ZSTD_hashPtrSalted` +#[inline(always)] +pub(crate) fn hash_ptr_salted( + buf: &[u8], + pos: usize, + h_bits: u32, + mls: u32, + hash_salt: u64, +) -> Result { + Ok(match mls { + 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), + 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), + 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), + 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), + _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, + }) +} + +// ── the window ──────────────────────────────────────────────────────────── + +/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are +/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the +/// others, and an address is turned into a position in the buffer only to read. +pub(crate) type Addr = usize; + +/// The buffers that a window can have a segment in. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum BufId { + /// Where an empty window points. + None = 0, + /// The context's own buffer of input. + InBuff = 1, + /// What the caller gave to the call in progress. + User = 2, + /// The content of the dictionary, or the prefix. + Dict = 3, +} + +impl BufId { + /// The address of the first byte of the buffer. + #[inline(always)] + pub(crate) const fn start(self) -> Addr { + (self as usize + 1) << 44 + } +} + +/// The buffers themselves, for as long as a call lasts. +#[derive(Clone, Copy)] +pub(crate) struct Buffers<'a> { + pub(crate) in_buff: &'a [u8], + pub(crate) user: &'a [u8], + pub(crate) dict: &'a [u8], +} + +impl<'a> Buffers<'a> { + #[inline] + pub(crate) fn get(&self, id: BufId) -> &'a [u8] { + match id { + BufId::None => &[], + BufId::InBuff => self.in_buff, + BufId::User => self.user, + BufId::Dict => self.dict, + } + } +} + +/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. +pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; + +/// The index past which indices are brought down. +pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; +/// The most input that can be indexed at once. +pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; + +/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the +/// index is less than `dict_limit`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Window { + /// Where input has to continue to be contiguous. + pub(crate) next_src: Addr, + pub(crate) base: Addr, + pub(crate) dict_base: Addr, + /// The buffer of the segment that starts at `dict_limit`. + pub(crate) base_buf: BufId, + /// The buffer of the segment from `low_limit` to `dict_limit`. + pub(crate) dict_buf: BufId, + pub(crate) dict_limit: u32, + /// Below this, there is nothing valid. + pub(crate) low_limit: u32, + pub(crate) nb_overflow_corrections: u32, +} + +impl Window { + /// `ZSTD_window_init` + pub(crate) const fn new() -> Self { + let base = BufId::None.start(); + Self { + next_src: base + ZSTD_WINDOW_START_INDEX as usize, + base, + dict_base: base, + base_buf: BufId::None, + dict_buf: BufId::None, + dict_limit: ZSTD_WINDOW_START_INDEX, + low_limit: ZSTD_WINDOW_START_INDEX, + nb_overflow_corrections: 0, + } + } + + /// The index of `addr`, which is in the current segment or at its end. + #[inline(always)] + pub(crate) const fn index_of(&self, addr: Addr) -> u32 { + addr.wrapping_sub(self.base) as u32 + } + + /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. + pub(crate) fn clear(&mut self) { + let end = self.index_of(self.next_src); + self.low_limit = end; + self.dict_limit = end; + } + + /// `ZSTD_window_isEmpty` + pub(crate) fn is_empty(&self) -> bool { + self.dict_limit == ZSTD_WINDOW_START_INDEX + && self.low_limit == ZSTD_WINDOW_START_INDEX + && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize + } + + /// `ZSTD_window_hasExtDict` + #[inline] + pub(crate) fn has_ext_dict(&self) -> bool { + self.low_limit < self.dict_limit + } + + /// `ZSTD_window_needOverflowCorrection` + #[inline] + pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { + self.index_of(src_end) > ZSTD_CURRENT_MAX + } + + /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns + /// what to subtract from every index that is stored. + pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { + let cycle_size = 1u32 << cycle_log; + let cycle_mask = cycle_size - 1; + let curr = self.index_of(src); + let current_cycle = curr & cycle_mask; + // Indices must not fall below the start index. + let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { + cycle_size.max(ZSTD_WINDOW_START_INDEX) + } else { + 0 + }; + let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); + let correction = curr.wrapping_sub(new_current); + + self.base = self.base.wrapping_add(correction as usize); + self.dict_base = self.dict_base.wrapping_add(correction as usize); + self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.low_limit - correction + }; + self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.dict_limit - correction + }; + self.nb_overflow_corrections += 1; + correction + } + + /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the + /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. + pub(crate) fn enforce_max_dist( + &mut self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { + let new_low_limit = block_end_idx - max_dist; + if self.low_limit < new_low_limit { + self.low_limit = new_low_limit; + } + if self.dict_limit < self.low_limit { + self.dict_limit = self.low_limit; + } + return true; + } + false + } + + /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not + /// right before the current segment, and is to be forgotten. + pub(crate) fn dict_is_invalid( + &self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit + } + + /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not + /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. + pub(crate) fn update( + &mut self, + src: Addr, + src_buf: BufId, + src_size: usize, + force_non_contiguous: bool, + ) -> bool { + let mut contiguous = true; + if src_size == 0 { + return contiguous; + } + if src != self.next_src || force_non_contiguous { + let distance_from_base = self.next_src.wrapping_sub(self.base); + self.low_limit = self.dict_limit; + self.dict_limit = distance_from_base as u32; + self.dict_base = self.base; + self.dict_buf = self.base_buf; + self.base = src.wrapping_sub(distance_from_base); + self.base_buf = src_buf; + if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { + // Too small to be of use. + self.low_limit = self.dict_limit; + } + contiguous = false; + } + self.next_src = src + src_size; + // Where the input overlaps the older segment, it has overwritten it. + if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) + & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) + { + let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); + self.low_limit = if high_input_idx > self.dict_limit as usize { + self.dict_limit + } else { + high_input_idx as u32 + }; + } + contiguous + } +} + +/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes +/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. +#[inline(always)] +pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { + prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 +} + +pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; +pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; + +/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. +#[inline(always)] +pub(crate) fn write_tagged_index( + hash_table: &mut [u32], + hash_and_tag: usize, + index: u32, +) -> Result<()> { + let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; + hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) +} + +/// `ZSTD_comparePackedTags` +#[inline(always)] +pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { + (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) + == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) +} + +// ── the state of the match finders ──────────────────────────────────────── + +/// `ZSTD_match_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Match { + /// As [`SeqDef::off_base`]. + pub(crate) off: u32, + pub(crate) len: u32, +} + +/// `rawSeq`: a sequence found by long distance matching. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct RawSeq { + pub(crate) offset: u32, + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +/// `RawSeqStore_t` +#[derive(Clone, Default)] +pub(crate) struct RawSeqStore { + /// `capacity` of them, of which `size` are in use. + pub(crate) seq: Vec, + /// The sequence that is being read. + pub(crate) pos: usize, + /// How many of its bytes, literals first, are read. + pub(crate) pos_in_sequence: usize, + pub(crate) size: usize, + pub(crate) capacity: usize, +} + +/// `ZSTD_optimal_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Optimal { + /// The cost of getting to this position. + pub(crate) price: i32, + pub(crate) off: u32, + pub(crate) mlen: u32, + pub(crate) litlen: u32, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +/// `ZSTD_OptPrice_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum OptPrice { + Dynamic, + Predef, +} + +/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. +pub(crate) struct OptState { + pub(crate) lit_freq: Vec, + pub(crate) lit_length_freq: Vec, + pub(crate) match_length_freq: Vec, + pub(crate) off_code_freq: Vec, + pub(crate) match_table: Vec, + pub(crate) price_table: Vec, + pub(crate) lit_sum: u32, + pub(crate) lit_length_sum: u32, + pub(crate) match_length_sum: u32, + pub(crate) off_code_sum: u32, + pub(crate) lit_sum_base_price: u32, + pub(crate) lit_length_sum_base_price: u32, + pub(crate) match_length_sum_base_price: u32, + pub(crate) off_code_sum_base_price: u32, + pub(crate) price_type: OptPrice, + pub(crate) literal_compression_mode: ParamSwitch, +} + +impl OptState { + pub(crate) const fn new() -> Self { + Self { + lit_freq: Vec::new(), + lit_length_freq: Vec::new(), + match_length_freq: Vec::new(), + off_code_freq: Vec::new(), + match_table: Vec::new(), + price_table: Vec::new(), + lit_sum: 0, + lit_length_sum: 0, + match_length_sum: 0, + off_code_sum: 0, + lit_sum_base_price: 0, + lit_length_sum_base_price: 0, + match_length_sum_base_price: 0, + off_code_sum_base_price: 0, + price_type: OptPrice::Dynamic, + literal_compression_mode: ParamSwitch::Auto, + } + } +} + +pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; + +/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures +/// own, are in [`BlockSrc`]. +pub(crate) struct MatchState { + pub(crate) window: Window, + /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. + pub(crate) loaded_dict_end: u32, + /// The index from which the tables are still to be filled. + pub(crate) next_to_update: u32, + /// 0 if there is no table of 3-byte hashes. + pub(crate) hash_log3: u32, + pub(crate) row_hash_log: u32, + /// For the row-based match finder: a byte of hash for each entry of `hash_table`. + pub(crate) tag_table: Vec, + pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], + /// Makes what a previous frame left in `tag_table` meaningless. + pub(crate) hash_salt: u64, + pub(crate) hash_salt_entropy: u32, + pub(crate) hash_table: Vec, + pub(crate) hash_table3: Vec, + pub(crate) chain_table: Vec, + pub(crate) force_non_contiguous: bool, + pub(crate) opt: OptState, + /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. + pub(crate) has_dict_match_state: bool, + pub(crate) c_params: CompressionParameters, + pub(crate) lazy_skipping: bool, +} + +impl MatchState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + loaded_dict_end: 0, + next_to_update: 0, + hash_log3: 0, + row_hash_log: 0, + tag_table: Vec::new(), + hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + hash_salt: 0, + hash_salt_entropy: 0, + hash_table: Vec::new(), + hash_table3: Vec::new(), + chain_table: Vec::new(), + force_non_contiguous: false, + opt: OptState::new(), + has_dict_match_state: false, + c_params: CompressionParameters { + window_log: 0, + chain_log: 0, + hash_log: 0, + search_log: 0, + min_match: 0, + target_length: 0, + strategy: None, + }, + lazy_skipping: false, + } + } + + /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. + pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { + if self.window.has_ext_dict() { + return DictMode::ExtDict; + } + if dms.is_some() { + DictMode::DictMatchState + } else { + DictMode::NoDict + } + } + + /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. + #[inline(always)] + pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.low_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + // A dictionary is valid as a whole until it is invalidated as a whole. + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } + + /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. + #[inline(always)] + pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.dict_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } +} + +/// The match state of a dictionary, and where its indices lead. +#[derive(Clone, Copy)] +pub(crate) struct DictMatchState<'a> { + pub(crate) ms: &'a MatchState, + /// The buffer its window is in, which is one segment. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// `window.nextSrc`, as a position in `buf`. + pub(crate) end: usize, +} + +/// The current segment of a window, for the functions that index a dictionary. +#[derive(Clone, Copy)] +pub(crate) struct Prefix<'a> { + /// The buffer that the segment is in. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, +} + +impl Prefix<'_> { + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. +#[derive(Clone, Copy)] +pub(crate) struct BlockSrc<'a> { + /// The buffer that the current segment of the window is in, and so the block. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// The buffer that the older segment of the window is in. + pub(crate) dict_buf: &'a [u8], + /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. + pub(crate) dict_base: usize, + /// Where the block starts in `buf`. + pub(crate) start: usize, + /// Where it ends. + pub(crate) end: usize, + pub(crate) dms: Option>, + /// `opt.symbolCosts`: the tables of the previous block. + pub(crate) symbol_costs: &'a EntropyCTables, + /// `ldmSeqStore`: matches that the optimal parser is to consider. + pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, +} + +impl<'a> BlockSrc<'a> { + #[inline(always)] + pub(crate) const fn prefix(&self) -> Prefix<'a> { + Prefix { + buf: self.buf, + base: self.base, + } + } + + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`, which is in the current segment. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } + + /// The position in `dict_buf` of `index`, which is in the older segment. + #[inline(always)] + pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { + self.dict_base.wrapping_add(index as usize) + } +} + +impl DictMatchState<'_> { + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +// ── long distance matching ──────────────────────────────────────────────── + +/// `ldmEntry_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct LdmEntry { + pub(crate) offset: u32, + pub(crate) checksum: u32, +} + +/// `ldmParams_t` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct LdmParams { + pub(crate) enable_ldm: ParamSwitch, + pub(crate) hash_log: u32, + pub(crate) bucket_size_log: u32, + pub(crate) min_match_length: u32, + pub(crate) hash_rate_log: u32, + pub(crate) window_log: u32, +} + +/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. +pub(crate) struct LdmState { + pub(crate) window: Window, + pub(crate) hash_table: Vec, + pub(crate) loaded_dict_end: u32, + /// The next entry to replace in each bucket. + pub(crate) bucket_offsets: Vec, +} + +impl LdmState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + hash_table: Vec::new(), + loaded_dict_end: 0, + bucket_offsets: Vec::new(), + } + } +} diff --git a/src/zstd/wip/work/fast/zstd/compress/lazy.rs b/src/zstd/wip/work/fast/zstd/compress/lazy.rs new file mode 100644 index 000000000000..9f653fd684f3 --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/compress/lazy.rs @@ -0,0 +1,34 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _ip: usize, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_row_update`: the same, in the rows of the row-based match finder. +pub(crate) fn row_update(_ms: &mut MatchState, _prefix: Prefix<'_>, _ip: usize) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, + _strategy: Strategy, + _use_row: bool, + _dict_mode: DictMode, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/fast/zstd/compress/ldm.rs b/src/zstd/wip/work/fast/zstd/compress/ldm.rs new file mode 100644 index 000000000000..2177fd945e10 --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/compress/ldm.rs @@ -0,0 +1 @@ +//! `zstd_ldm.c`: long distance matching. diff --git a/src/zstd/wip/work/fast/zstd/compress/mod.rs b/src/zstd/wip/work/fast/zstd/compress/mod.rs new file mode 100644 index 000000000000..13ddecdf82e4 --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/compress/mod.rs @@ -0,0 +1,68 @@ +//! `lib/compress` + +pub(crate) mod double_fast; +pub(crate) mod fast; +pub(crate) mod fse; +pub(crate) mod hist; +pub(crate) mod huf; +pub(crate) mod internal; +pub(crate) mod lazy; +pub(crate) mod ldm; +pub(crate) mod opt; +pub(crate) mod presplit; + +use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; + +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; + +/// `ZSTD_rowMatchFinderSupported` +pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { + strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 +} + +/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to +/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies +/// that support it look at. Returns how many bytes at the end of the block are in no sequence. +pub(crate) fn compress_block( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row_match_finder: bool, + dict_mode: DictMode, +) -> Result { + match strategy { + Strategy::Fast => match dict_mode { + DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), + DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), + DictMode::DictMatchState => { + fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::DFast => match dict_mode { + DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), + DictMode::ExtDict => { + double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { + lazy::compress_block_lazy( + ms, + seq_store, + rep, + src, + strategy, + use_row_match_finder && row_match_finder_supported(strategy), + dict_mode, + ) + } + Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { + opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) + } + } +} diff --git a/src/zstd/wip/work/fast/zstd/compress/opt.rs b/src/zstd/wip/work/fast/zstd/compress/opt.rs new file mode 100644 index 000000000000..d055a44cf982 --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/compress/opt.rs @@ -0,0 +1,29 @@ +//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. + +use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. +/// Matches are not compared past the position `iend`. +pub(crate) fn update_tree( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _ip: usize, + _iend: usize, +) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` +/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. +pub(crate) fn compress_block_opt( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, + _strategy: Strategy, + _dict_mode: DictMode, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/fast/zstd/compress/presplit.rs b/src/zstd/wip/work/fast/zstd/compress/presplit.rs new file mode 100644 index 000000000000..8b778e2989f9 --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/compress/presplit.rs @@ -0,0 +1 @@ +//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. diff --git a/src/zstd/wip/work/fast/zstd/lib.rs b/src/zstd/wip/work/fast/zstd/lib.rs new file mode 100644 index 000000000000..c658d011170d --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/lib.rs @@ -0,0 +1,386 @@ +//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the +//! same bytes in, in both directions, and the same errors. +#![forbid(unsafe_code)] +// Frames come from anywhere and a panic aborts the process. +#![deny( + clippy::indexing_slicing, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unreachable +)] +#![warn(unused_must_use)] + +mod common; +mod compress; +mod decompress; +mod error; +mod util; + +pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; +pub use decompress::{ + DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, + find_frame_compressed_size, get_frame_content_size, +}; +pub use error::ErrorCode; + +/// `ZSTD_inBuffer` +pub struct InBuffer<'a> { + pub src: &'a [u8], + /// How much of `src` is read. + pub pos: usize, +} + +/// `ZSTD_outBuffer` +pub struct OutBuffer<'a> { + pub dst: &'a mut [u8], + /// How much of `dst` is written. + pub pos: usize, +} + +/// `ZSTD_ResetDirective` +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ResetDirective { + SessionOnly = 1, + Parameters = 2, + SessionAndParameters = 3, +} + +// ----------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] +pub enum ZstdError { + /// The output, or the decoder state the frame's window size dictates, could not be allocated. + OutOfMemory, + InvalidZstdData, + DecompressionFailed, + ZstdDecompressionError, + ShortRead, +} + +impl core::fmt::Display for ZstdError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(<&'static str>::from(self)) + } +} +impl core::error::Error for ZstdError {} + +impl ZstdError { + /// The error for a failed decompression call; `other` is the non-allocation failure. + fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { + if code == ErrorCode::MemoryAllocation { + ZstdError::OutOfMemory + } else { + other + } + } +} + +/// `len` zeros for a decoder to write to. +fn output_buffer(len: usize) -> Result, ZstdError> { + util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) +} + +/// Decompress data, automatically allocating the output buffer. +/// Handles both frames with known and unknown content sizes. +/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. +/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. +pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { + let size = find_decompressed_size(src); + + const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit + + if size == ZSTD_CONTENTSIZE_ERROR { + return Err(ZstdError::InvalidZstdData); + } + + // Use streaming decompression if: + // 1. Content size is unknown, OR + // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) + if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { + let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { + // A frame's output is rarely smaller than its input. + src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) + } else { + // The header size is untrusted: reserve no more than the fast path below would. + MAX_PREALLOCATE_SIZE + }; + let mut list: Vec = Vec::new(); + list.try_reserve_exact(initial_capacity) + .map_err(|_| ZstdError::OutOfMemory)?; + StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; + return Ok(list); + } + + // Fast path: size is known and within reasonable limits + let mut output = output_buffer(size as usize)?; + let written = decompress(&mut output, src) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is +/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. +pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { + const MAX_SIZE: u64 = 16 * 1024 * 1024; + let size = find_decompressed_size(src); + // Also if it is unknown or invalid, which are the two largest values. + if size > MAX_SIZE { + return Err(ZstdError::InvalidZstdData); + } + let mut output = output_buffer(size as usize)?; + let written = DDict::prefix(prefix) + .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + Uninitialized, + Inflating, + End, + Error, +} + +/// Minimum spare output capacity offered to the decoder per call. +const STREAMING_OUTPUT_STEP: usize = 4096; + +// ────────────────────────────────────────────────────────────────────────── +// StreamingDecoder +// ────────────────────────────────────────────────────────────────────────── + +/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) +/// per call, so callers can hold the decoder across multiple body chunks. +pub struct StreamingDecoder { + stream: DCtx, + state: State, + /// The decoder filled its last window and may hold more. `max_output` can end a call there. + output_full: bool, +} + +impl Default for StreamingDecoder { + fn default() -> Self { + Self::new() + } +} + +impl StreamingDecoder { + pub fn new() -> Self { + Self { + stream: DCtx::new(), + state: State::Uninitialized, + output_full: false, + } + } + + #[inline] + pub fn is_inflating(&self) -> bool { + matches!(self.state, State::Inflating) + } + + /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is + /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. + /// Returns `ShortRead` when more input is required and `is_done` is false. + pub fn decompress( + &mut self, + input: &[u8], + out: &mut Vec, + max_output: usize, + is_done: bool, + ) -> Result { + if matches!(self.state, State::End | State::Error) { + return Ok(input.len()); + } + + let mut in_buf = InBuffer { src: input, pos: 0 }; + while matches!(self.state, State::Uninitialized | State::Inflating) { + let next_in_len = input.len() - in_buf.pos; + + // Call the decoder again with no input until it leaves the window short. + if next_in_len == 0 && !self.output_full { + if is_done { + if self.state == State::Inflating { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + self.state = State::End; + } + return Ok(in_buf.pos); + } + + if out.len() >= max_output { + return Ok(in_buf.pos); + } + + if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { + self.state = State::Error; + return Err(ZstdError::OutOfMemory); + } + // The decoder writes to initialized memory only. + let written_before = out.len(); + let window = (out.capacity() - written_before).min(max_output - written_before); + out.resize(written_before + window, 0); + let mut out_buf = OutBuffer { + dst: out.get_mut(written_before..).unwrap_or_default(), + pos: 0, + }; + let read_before = in_buf.pos; + + let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); + let bytes_written = out_buf.pos; + out.truncate(written_before + bytes_written); + let remaining = match result { + Ok(remaining) => remaining, + Err(code) => { + self.state = State::Error; + return Err(ZstdError::for_decompression( + code, + ZstdError::ZstdDecompressionError, + )); + } + }; + + self.output_full = bytes_written == window; + + if remaining == 0 { + // Frame complete, and fully flushed. + self.state = State::Uninitialized; + self.output_full = false; + if in_buf.pos >= input.len() { + if is_done { + self.state = State::End; + } + return Ok(in_buf.pos); + } + // More input available — reinitialize for the next frame. + self.stream.init_stream(); + continue; + } + + self.state = State::Inflating; + + if in_buf.pos - read_before == next_in_len { + if self.output_full { + continue; + } + if is_done { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + return Err(ZstdError::ShortRead); + } + } + Ok(in_buf.pos) + } +} + +// ── compressed embedded assets ──────────────────────────────────────────── + +/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes +/// Bun never executes or parses itself: the shell completion scripts and the +/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, +/// error overlay/page). Anything that runs inside Bun stays uncompressed. +/// +/// The exception is text that only `bun lint`, `bun fmt` and `bun check` +/// read. It may be compressed because it isn't on the path of running a program. +/// +/// Release (`bun_codegen_embed`) builds include the `.zst` twin that +/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to +/// `/compressed/.zst`; other builds evaluate `$fallback`. +#[macro_export] +macro_rules! embed_compressed { + // A codegen output (`/$sub`); debug builds read it at runtime. + (codegen $sub:literal) => { + $crate::embed_compressed!( + ("codegen/", $sub), + ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() + ) + }; + // Same, with a trailing NUL included in the slice (for C-string consumers). + (codegen_nul $sub:literal) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/codegen/", + $sub, + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); + &__COPY.get_or_init(|| { + let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); + let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); + copy.extend_from_slice(text); + copy.push(0); + copy.into_boxed_slice() + })[..] + } + }; + __bytes + }}; + // A file under `src/`; debug builds read it from the source tree at runtime. + (src $sub:literal) => { + $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) + }; + ($name:literal, $fallback:expr) => { + $crate::embed_compressed!(($name), $fallback) + }; + (($($name:literal),+), $fallback:expr) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/", + $($name,)+ + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + $fallback + } + }; + __bytes + }}; +} + +/// Cold, shared body of [`embed_compressed!`]'s release arm. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { + decompress_alloc(compressed) + .expect("embedded asset: invalid zstd frame") + .into_boxed_slice() +} + +/// [`inflate_embedded`] plus a trailing NUL byte. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { + let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); + inflated.reserve_exact(1); + inflated.push(0); + inflated.into_boxed_slice() +} +pub mod testing; +pub mod testing_frame; diff --git a/src/zstd/wip/work/fast/zstd/testing.rs b/src/zstd/wip/work/fast/zstd/testing.rs new file mode 100644 index 000000000000..9f6051d0ef68 --- /dev/null +++ b/src/zstd/wip/work/fast/zstd/testing.rs @@ -0,0 +1,364 @@ +//! Hooks for the differential tests. Only in scratch copies of the crate. +#![allow(clippy::all, dead_code, unreachable_pub)] + +use crate::ErrorCode; +use crate::compress::internal::*; +use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; + +fn bitmix(mut val: u64, len: u64) -> u64 { + val ^= val.rotate_right(49) ^ val.rotate_right(24); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^= (val >> 35).wrapping_add(len); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^ (val >> 28) +} + +/// The sequences of the first block of a frame, as (literal length, match length, offset), and then the literals +/// that are left as (length, 0, 0). +/// +/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, already adjusted to the sizes. +/// `dict` is a prefix (raw content, as `ZSTD_CCtx_refPrefix`), or empty. `src` is from 7 to 131071 bytes. +pub fn first_block( + cp: [u32; 7], + use_row: bool, + dict: &[u8], + src: &[u8], +) -> Result, ErrorCode> { + let strategy = Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?; + let c_params = CompressionParameters { + window_log: cp[0], + chain_log: cp[1], + hash_log: cp[2], + search_log: cp[3], + min_match: cp[4], + target_length: cp[5], + strategy: Some(strategy), + }; + let row_used = row_match_finder_supported(strategy) && use_row; + + // ZSTD_reset_matchState + let mut ms = MatchState::new(); + ms.hash_log3 = if c_params.min_match == 3 { + c_params.window_log.min(17) + } else { + 0 + }; + ms.window.clear(); + ms.next_to_update = ms.window.dict_limit; + ms.hash_table = vec![0; 1 << c_params.hash_log]; + ms.chain_table = vec![ + 0; + if strategy != Strategy::Fast && !row_used { + 1 << c_params.chain_log + } else { + 0 + } + ]; + ms.hash_table3 = vec![ + 0; + if ms.hash_log3 != 0 { + 1 << ms.hash_log3 + } else { + 0 + } + ]; + if row_used { + ms.tag_table = vec![0; 1 << c_params.hash_log]; + ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4); + ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); + } + if strategy >= Strategy::BtOpt { + ms.opt.lit_freq = vec![0; 256]; + ms.opt.lit_length_freq = vec![0; 36]; + ms.opt.match_length_freq = vec![0; 53]; + ms.opt.off_code_freq = vec![0; 32]; + ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; + ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; + } + ms.c_params = c_params; + + let block_size = (1usize << c_params.window_log).min(128 << 10); + let mut seq_store = SeqStore::new(); + seq_store.reserve( + block_size / if c_params.min_match == 3 { 3 } else { 4 }, + block_size, + )?; + + let bufs = Buffers { + in_buff: &[], + user: src, + dict, + }; + + // ZSTD_loadDictionaryContent(dtlm_fast, tfp_forCCtx) + if !dict.is_empty() { + let iend = dict.len(); + ms.window + .update(BufId::Dict.start(), BufId::Dict, dict.len(), false); + let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); + let ip = iend - dict.len().min(max_dict_size); + let prefix = Prefix { + buf: dict, + base: ms.window.base.wrapping_sub(BufId::Dict.start()), + }; + ms.next_to_update = prefix.index_of(ip); + ms.loaded_dict_end = prefix.index_of(iend); + if iend - ip > HASH_READ_SIZE { + match strategy { + Strategy::Fast => fast::fill_hash_table( + &mut ms, + prefix, + iend, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + )?, + Strategy::DFast => double_fast::fill_double_hash_table( + &mut ms, + prefix, + iend, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + )?, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { + if use_row { + ms.tag_table.fill(0); + lazy::row_update(&mut ms, prefix, iend - HASH_READ_SIZE)?; + } else { + lazy::insert_and_find_first_index(&mut ms, prefix, iend - HASH_READ_SIZE)?; + } + } + _ => opt::update_tree(&mut ms, prefix, iend - HASH_READ_SIZE, iend)?, + } + ms.next_to_update = prefix.index_of(iend); + } + } + + // ZSTD_compressContinue_internal + let src_addr = BufId::User.start(); + if !ms + .window + .update(src_addr, BufId::User, src.len(), ms.force_non_contiguous) + { + ms.force_non_contiguous = false; + ms.next_to_update = ms.window.dict_limit; + } + // ZSTD_compress_frameChunk + let max_dist = 1u32 << c_params.window_log; + let mut prev = CompressedBlockState::new(); + let mut out = Vec::new(); + let mut start = 0usize; + while start < src.len() { + let end = src.len().min(start + block_size); + if ms + .window + .dict_is_invalid(src_addr + end, max_dist, ms.loaded_dict_end) + { + ms.loaded_dict_end = 0; + } + if ms + .window + .enforce_max_dist(src_addr + start, max_dist, ms.loaded_dict_end) + { + ms.loaded_dict_end = 0; + } + if ms.next_to_update < ms.window.low_limit { + ms.next_to_update = ms.window.low_limit; + } + // ZSTD_buildSeqStore + seq_store.reset(); + let block = BlockSrc { + buf: bufs.get(ms.window.base_buf), + base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), + dict_buf: bufs.get(ms.window.dict_buf), + dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), + start, + end, + dms: None, + symbol_costs: &prev.entropy, + ldm_seq_store: None, + }; + let curr = block.index_of(start); + if curr > ms.next_to_update + 384 { + ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); + } + let dict_mode = ms.dict_mode(None); + if std::env::var("DBG").is_ok() { + eprintln!( + "block {start}..{end} mode {dict_mode:?} window {:?} lde {} ntu {}", + ms.window, ms.loaded_dict_end, ms.next_to_update + ); + } + let mut rep = prev.rep; + let last_ll = compress::compress_block( + &mut ms, + &mut seq_store, + &mut rep, + &block, + strategy, + use_row, + dict_mode, + )?; + + let mut reps = prev.rep; + for (i, seq) in seq_store.sequences.iter().enumerate() { + let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); + if seq_store.long_length_pos as usize == i { + match seq_store.long_length_type { + LongLengthType::LiteralLength => ll += 0x10000, + LongLengthType::MatchLength => ml += 0x10000, + LongLengthType::None => {} + } + } + update_rep(&mut reps, seq.off_base, ll == 0); + out.push((ll, ml, reps[0])); + } + out.push((last_ll as u32, 0, 0)); + assert_eq!( + reps[..2], + rep[..2], + "the repeat offsets the block compressor leaves are those of its sequences" + ); + prev.rep = rep; + start = end; + } + Ok(out) +} + +/// A match state whose window is set by hand, to run the block compressors of `fast` and `dfast` one call at a time. +pub struct TestMs { + ms: MatchState, + seq_store: SeqStore, + prev: CompressedBlockState, +} + +/// The dictionary's match state, its buffer, `window.base` and `window.nextSrc` as positions in it. +pub type TestDms<'a> = (&'a TestMs, &'a [u8], usize, usize); + +impl TestMs { + /// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy. + pub fn new(cp: [u32; 7], max_block: usize) -> Self { + let mut ms = MatchState::new(); + ms.c_params = CompressionParameters { + window_log: cp[0], + chain_log: cp[1], + hash_log: cp[2], + search_log: cp[3], + min_match: cp[4], + target_length: cp[5], + strategy: Strategy::from_int(cp[6] as i32), + }; + ms.hash_table = vec![0; 1 << cp[2]]; + ms.chain_table = vec![0; 1 << cp[1]]; + let mut seq_store = SeqStore::new(); + seq_store.reserve(max_block / 3 + 1, max_block).unwrap(); + Self { + ms, + seq_store, + prev: CompressedBlockState::new(), + } + } + + pub fn set_state( + &mut self, + dict_limit: u32, + low_limit: u32, + loaded_dict_end: u32, + next_to_update: u32, + ) { + self.ms.window.dict_limit = dict_limit; + self.ms.window.low_limit = low_limit; + self.ms.loaded_dict_end = loaded_dict_end; + self.ms.next_to_update = next_to_update; + } + + pub fn fill( + &mut self, + buf: &[u8], + base: usize, + end: usize, + full: bool, + for_cdict: bool, + ) -> Result<(), ErrorCode> { + let prefix = Prefix { buf, base }; + let dtlm = if full { + DictTableLoadMethod::Full + } else { + DictTableLoadMethod::Fast + }; + let tfp = if for_cdict { + TableFillPurpose::ForCDict + } else { + TableFillPurpose::ForCCtx + }; + match self.ms.c_params.strat() { + Strategy::Fast => fast::fill_hash_table(&mut self.ms, prefix, end, dtlm, tfp), + _ => double_fast::fill_double_hash_table(&mut self.ms, prefix, end, dtlm, tfp), + } + } + + /// `mode` is a `ZSTD_dictMode_e`. + pub fn block( + &mut self, + rep: &mut [u32; 3], + mode: u32, + buf: &[u8], + base: usize, + dict_buf: &[u8], + dict_base: usize, + start: usize, + end: usize, + dms: Option>, + ) -> Result { + self.seq_store.reset(); + let dms = dms.map(|(dict, buf, base, end)| DictMatchState { + ms: &dict.ms, + buf, + base, + end, + }); + let block = BlockSrc { + buf, + base, + dict_buf, + dict_base, + start, + end, + dms, + symbol_costs: &self.prev.entropy, + ldm_seq_store: None, + }; + let dict_mode = self.ms.dict_mode(dms.as_ref()); + assert_eq!(dict_mode as u32, mode); + let strategy = self.ms.c_params.strat(); + compress::compress_block( + &mut self.ms, + &mut self.seq_store, + rep, + &block, + strategy, + false, + dict_mode, + ) + } + + /// offBase, litLength, mlBase + pub fn sequences(&self) -> Vec<(u32, u16, u16)> { + self.seq_store + .sequences + .iter() + .map(|s| (s.off_base, s.lit_length, s.ml_base)) + .collect() + } + pub fn literals(&self) -> &[u8] { + self.seq_store.literals() + } + pub fn long_length(&self) -> (u32, u32) { + ( + self.seq_store.long_length_type as u32, + self.seq_store.long_length_pos, + ) + } + pub fn tables(&self) -> (&[u32], &[u32]) { + (&self.ms.hash_table, &self.ms.chain_table) + } +} diff --git a/src/zstd/wip/work/lazy/harness/Cargo.toml b/src/zstd/wip/work/lazy/harness/Cargo.toml new file mode 100644 index 000000000000..ac7a85d4b3bf --- /dev/null +++ b/src/zstd/wip/work/lazy/harness/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "zstd_harness" +version = "0.0.0" +edition = "2024" + +[dependencies] +bun_zstd = { path = "/root/zstd-ref/work/lazy/zstd" } + +[profile.release] +debug = "line-tables-only" +overflow-checks = true +debug-assertions = true +panic = "abort" + +[profile.bench] +inherits = "release" +overflow-checks = false +debug-assertions = false + +[workspace] diff --git a/src/zstd/wip/work/lazy/harness/build.rs b/src/zstd/wip/work/lazy/harness/build.rs new file mode 100644 index 000000000000..5dd59e7fcdfd --- /dev/null +++ b/src/zstd/wip/work/lazy/harness/build.rs @@ -0,0 +1,18 @@ +use std::process::Command; +fn main() { + let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); + println!("cargo:rerun-if-env-changed=ZSTD_REF"); + println!("cargo:rerun-if-changed=shim.c"); + let out = std::env::var("OUT_DIR").unwrap(); + let ok = Command::new("clang-23") + .args(["-c", "-O2", "-fPIC", "-march=nehalem", "-DZSTD_MULTITHREAD", "-DZSTD_LEGACY_SUPPORT=0", "-DXXH_NAMESPACE=ZSTD_", + "-I/root/zstd-ref/zstd/lib", "-I/root/zstd-ref/zstd/lib/common", "shim.c", "-o"]) + .arg(format!("{out}/shim.o")).status().unwrap().success(); + assert!(ok); + let _ = std::fs::remove_file(format!("{out}/libshim.a")); + assert!(Command::new("llvm-ar-23").args(["rcs", &format!("{out}/libshim.a"), &format!("{out}/shim.o")]).status().unwrap().success()); + println!("cargo:rustc-link-search=native={out}"); + println!("cargo:rustc-link-lib=static=shim"); + println!("cargo:rustc-link-search=native={}", if variant.starts_with("/") { variant.clone() } else { format!("/root/zstd-ref/build/{variant}") }); + println!("cargo:rustc-link-lib=static=zstd"); +} diff --git a/src/zstd/wip/work/lazy/harness/rust-toolchain.toml b/src/zstd/wip/work/lazy/harness/rust-toolchain.toml new file mode 100644 index 000000000000..c71778e8ccd3 --- /dev/null +++ b/src/zstd/wip/work/lazy/harness/rust-toolchain.toml @@ -0,0 +1,35 @@ +[toolchain] +channel = "nightly-2026-09-15" +# rust-src is needed for -Zbuild-std (Tier 3 targets like +# aarch64-unknown-freebsd have no prebuilt std). miri is for +# `bun run rust:miri`. targets ensures the +# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain +# (CI's images install this same channel with these components and +# targets, from scripts/build/ci-images/spec.ts; a source lint keeps +# the two files in step). Note: rustup ignores this whole file when +# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree +# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` +# self-heals that path. Keep this list as a fallback for direct +# `cargo build`/`cargo check`/rust-analyzer use. +# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd +# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). +# darwin/windows triples build natively on their own agents, so the host std +# comes with the toolchain install regardless; they're listed here so a dev +# running `bun run rust:check-all` or `cargo check --target` from any host +# still gets prebuilt std for every Tier 1/2 triple. +# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at +# what it emits. The build does not use them. +components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] +targets = [ + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-musl", + "aarch64-linux-android", + "x86_64-linux-android", + "x86_64-unknown-freebsd", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-pc-windows-msvc", + "x86_64-pc-windows-msvc", +] diff --git a/src/zstd/wip/work/lazy/harness/shim.c b/src/zstd/wip/work/lazy/harness/shim.c new file mode 100644 index 000000000000..08a216c4ed2f --- /dev/null +++ b/src/zstd/wip/work/lazy/harness/shim.c @@ -0,0 +1,41 @@ +// Exposes the internal state of the C library to the differential tests. +#define ZSTD_STATIC_LINKING_ONLY +#include "compress/zstd_compress_internal.h" + +typedef struct { + unsigned cp[7]; + unsigned useRow; + unsigned hasDms; + unsigned nextToUpdate, loadedDictEnd, lowLimit, dictLimit, rowHashLog, hashSaltEntropy, lazySkipping; + unsigned long long hashSalt; + unsigned hashCache[8]; + unsigned rep[3]; + const unsigned* hashTable; + const unsigned* chainTable; + const unsigned char* tagTable; +} ShimState; + +void shim_state(const ZSTD_CCtx* c, ShimState* s) { + const ZSTD_MatchState_t* ms = &c->blockState.matchState; + const ZSTD_compressionParameters* p = &c->appliedParams.cParams; + int i; + s->cp[0] = p->windowLog; s->cp[1] = p->chainLog; s->cp[2] = p->hashLog; s->cp[3] = p->searchLog; + s->cp[4] = p->minMatch; s->cp[5] = p->targetLength; s->cp[6] = (unsigned)p->strategy; + s->useRow = c->appliedParams.useRowMatchFinder == ZSTD_ps_enable; + s->hasDms = ms->dictMatchState != NULL; + s->nextToUpdate = ms->nextToUpdate; s->loadedDictEnd = ms->loadedDictEnd; + s->lowLimit = ms->window.lowLimit; s->dictLimit = ms->window.dictLimit; + s->rowHashLog = ms->rowHashLog; s->hashSaltEntropy = ms->hashSaltEntropy; s->lazySkipping = (unsigned)ms->lazySkipping; + s->hashSalt = ms->hashSalt; + for (i = 0; i < 8; i++) s->hashCache[i] = ms->hashCache[i]; + for (i = 0; i < 3; i++) s->rep[i] = c->blockState.prevCBlock->rep[i]; + s->hashTable = ms->hashTable; s->chainTable = ms->chainTable; s->tagTable = ms->tagTable; +} + +void shim_collect(ZSTD_CCtx* c, ZSTD_Sequence* seqs, size_t cap) { + c->seqCollector.collectSequences = 1; + c->seqCollector.seqStart = seqs; + c->seqCollector.seqIndex = 0; + c->seqCollector.maxSequences = cap; +} +size_t shim_collected(const ZSTD_CCtx* c) { return c->seqCollector.seqIndex; } diff --git a/src/zstd/wip/work/lazy/harness/src/c.rs b/src/zstd/wip/work/lazy/harness/src/c.rs new file mode 100644 index 000000000000..d5a82e1f6b78 --- /dev/null +++ b/src/zstd/wip/work/lazy/harness/src/c.rs @@ -0,0 +1,104 @@ +//! The reference C library. +#![allow(non_camel_case_types, dead_code)] +use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; + +#[repr(C)] +pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } +#[repr(C)] +pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } +pub enum ZSTD_CCtx {} +pub enum ZSTD_DCtx {} +pub enum ZSTD_DDict {} + +unsafe extern "C" { + pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; + pub fn ZSTD_compressBound(size: usize) -> usize; + pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_isError(code: usize) -> c_uint; + pub fn ZSTD_getErrorCode(code: usize) -> c_uint; + pub fn ZSTD_getErrorName(code: usize) -> *const c_char; + pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; + pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; + pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; + pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; + pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; + pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; + pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; + pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; + pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; + pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; + pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; + pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; + pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; + pub fn ZSTD_defaultCLevel() -> c_int; + pub fn ZSTD_minCLevel() -> c_int; + pub fn ZSTD_maxCLevel() -> c_int; +} + +/// `Ok(value)` or `Err(error code)`. +pub fn res(code: usize) -> Result { + unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } +} + +pub struct CCtx(pub *mut ZSTD_CCtx); +impl CCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } + pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + /// Returns (result, out pos, in pos). + pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); + (r, o.pos, i.pos) + } +} +impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } + +pub struct DCtx(pub *mut ZSTD_DCtx); +impl DCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } + pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } + pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); + (r, o.pos, i.pos) + } +} +impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } + +pub fn compress(src: &[u8], level: i32) -> Vec { + let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; + let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); + dst.truncate(n); + dst +} +pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) +} diff --git a/src/zstd/wip/work/lazy/harness/src/data.rs b/src/zstd/wip/work/lazy/harness/src/data.rs new file mode 100644 index 000000000000..c48a645e1b5d --- /dev/null +++ b/src/zstd/wip/work/lazy/harness/src/data.rs @@ -0,0 +1,65 @@ +//! Deterministic inputs. +pub struct Rng(pub u64); +impl Rng { + pub fn next(&mut self) -> u64 { + // splitmix64 + self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + } + pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } + pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } + pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } + pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } + /// Skewed towards small values. + pub fn size(&mut self, max: usize) -> usize { + let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); + (self.next() as usize & ((1usize << bits) - 1)).min(max) + } +} + +static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; + +/// One buffer of about `len` bytes, of a kind picked by `rng`. +pub fn data(rng: &mut Rng, len: usize) -> Vec { + let mut v = Vec::with_capacity(len + 64); + match rng.below(9) { + 0 => v.resize(len, rng.next() as u8), + 1 => while v.len() < len { v.push(rng.next() as u8) }, + 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, + 3 => { // few symbols, skewed + let n = rng.range(2, 40); + while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } + } + 4 => { // LZ-like: copies of earlier parts, at all distances + while v.len() < len { + if v.len() > 8 && rng.chance(3, 4) { + let dist = 1 + rng.size(v.len() - 1); + let n = 3 + rng.size(300); + for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } + } else { + for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } + } + } + } + 5 => { // periodic with noise + let period = rng.range(1, 300); + let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); + while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } + } + 6 => { // sections of different kinds + while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } + } + 7 => { // numbers as text + let mut x = rng.next() % 1000; + while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } + } + _ => { // long runs + while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } + } + } + v.truncate(len); + v +} diff --git a/src/zstd/wip/work/lazy/harness/src/main.rs b/src/zstd/wip/work/lazy/harness/src/main.rs new file mode 100644 index 000000000000..41d45b62720d --- /dev/null +++ b/src/zstd/wip/work/lazy/harness/src/main.rs @@ -0,0 +1,27 @@ +mod c; +mod data; +mod seqs; + +fn main() { + let args: Vec = std::env::args().collect(); + let cmd = args.get(1).map(String::as_str).unwrap_or(""); + let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); + let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); + match cmd { + // seqs + "seqs" => { + let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); + // 0: no dictionary, 1: a prefix or none, 2: an attached CDict or none + let modes: &[seqs::Mode] = match args.get(5).map(String::as_str) { + Some("1") => &[seqs::Mode::NoDict, seqs::Mode::Prefix], + Some("2") => &[seqs::Mode::CDict, seqs::Mode::CDict, seqs::Mode::CDict, seqs::Mode::NoDict], + Some("3") => &[seqs::Mode::NoDict, seqs::Mode::NoDict, seqs::Mode::Prefix, seqs::Mode::CDict], + _ => &[seqs::Mode::NoDict], + }; + // 3: through the buffer of ZSTD_compressStream2, which wraps around + seqs::run(seed, iters, &strategies, modes, args.get(5).is_some_and(|s| s == "3")); + } + "bench" => seqs::bench(&args[2], match args.get(3).map(String::as_str) { Some("1") => seqs::Mode::Prefix, Some("2") => seqs::Mode::CDict, _ => seqs::Mode::NoDict }), + _ => eprintln!("unknown command"), + } +} diff --git a/src/zstd/wip/work/lazy/harness/src/seqs.rs b/src/zstd/wip/work/lazy/harness/src/seqs.rs new file mode 100644 index 000000000000..fb7d35f839e4 --- /dev/null +++ b/src/zstd/wip/work/lazy/harness/src/seqs.rs @@ -0,0 +1,193 @@ +//! A frame through the Rust block compressors and through the C library (`ZSTD_generateSequences`). +use crate::c; +use crate::data::{Rng, data}; +use bun_zstd::testing::{Dict, State}; +use std::ffi::{c_uint, c_void}; + +#[repr(C)] +#[derive(Clone, Copy, Default, Debug)] +struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } +#[repr(C)] +#[derive(Clone, Copy, Debug)] +struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } +#[repr(C)] +#[derive(Clone, Copy)] +struct CustomMem { a: *const c_void, b: *const c_void, c: *const c_void } +#[repr(C)] +struct ShimState { + cp: [c_uint; 7], use_row: c_uint, has_dms: c_uint, + next_to_update: c_uint, loaded_dict_end: c_uint, low_limit: c_uint, dict_limit: c_uint, row_hash_log: c_uint, hash_salt_entropy: c_uint, lazy_skipping: c_uint, + hash_salt: u64, hash_cache: [c_uint; 8], rep: [c_uint; 3], + hash_table: *const u32, chain_table: *const u32, tag_table: *const u8, +} +enum CDict {} +enum CCtxParams {} +unsafe extern "C" { + fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; + fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; + fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; + fn ZSTD_createCCtxParams() -> *mut CCtxParams; + fn ZSTD_freeCCtxParams(p: *mut CCtxParams) -> usize; + fn ZSTD_CCtxParams_setParameter(p: *mut CCtxParams, param: c_uint, v: i32) -> usize; + fn ZSTD_createCDict_advanced2(d: *const c_void, n: usize, load: c_uint, ty: c_uint, p: *const CCtxParams, mem: CustomMem) -> *mut CDict; + fn ZSTD_freeCDict(d: *mut CDict) -> usize; + fn ZSTD_CCtx_refCDict(c: *mut c::ZSTD_CCtx, d: *const CDict) -> usize; + fn ZSTD_getCParamsFromCDict(d: *const CDict) -> CParams; + fn shim_state(c: *const c::ZSTD_CCtx, s: *mut ShimState); + fn shim_collect(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize); + fn shim_collected(c: *const c::ZSTD_CCtx) -> usize; +} + +fn arr(p: CParams) -> [u32; 7] { [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy] } + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum Mode { NoDict, Prefix, CDict } + +struct COut { seqs: Vec<(u32, u32, u32, u32)>, cp: [u32; 7], use_row: bool, cdict_cp: [u32; 7], state: State } + +/// `None` if the C library refuses (a last block that is too small). +fn c_frame(cp: [u32; 7], use_row: bool, mode: Mode, dict: &[u8], src: &[u8], streaming: bool) -> Option { + let mut cctx = c::CCtx::new(); + let mut cdict = std::ptr::null_mut(); + let mut cdict_cp = [0; 7]; + cctx.set(1010, 2).unwrap(); // no block splitter + if mode == Mode::CDict { + unsafe { + let p = ZSTD_createCCtxParams(); + for (i, v) in cp.iter().enumerate() { c::res(ZSTD_CCtxParams_setParameter(p, 101 + i as u32, *v as i32)).unwrap(); } + c::res(ZSTD_CCtxParams_setParameter(p, 1011, if use_row { 1 } else { 2 })).unwrap(); + let null = std::ptr::null(); + cdict = ZSTD_createCDict_advanced2(dict.as_ptr().cast(), dict.len(), 1, 1, p, CustomMem { a: null, b: null, c: null }); + assert!(!cdict.is_null()); + ZSTD_freeCCtxParams(p); + c::res(ZSTD_CCtx_refCDict(cctx.0, cdict)).unwrap(); + cdict_cp = arr(ZSTD_getCParamsFromCDict(cdict)); + } + cctx.set(1001, 1).unwrap(); // force attach + cctx.set(101, cp[0] as i32).unwrap(); + } else { + for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } + cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder + if mode == Mode::Prefix { cctx.ref_prefix(dict).unwrap(); } + } + let mut out = vec![Sequence::default(); src.len() / 3 + src.len() / 1024 + 8]; + let n = if streaming { + unsafe { shim_collect(cctx.0, out.as_mut_ptr(), out.len()) }; + let mut dst = vec![0u8; src.len() + src.len() / 64 + 1024]; + let (mut dpos, mut spos) = (0, 0); + let mut r = Ok(0); + while spos < src.len() && r.is_ok() { + (r, dpos, spos) = cctx.stream2(&mut dst, dpos, src, spos, 0); + } + if r.is_ok() { (r, _, _) = cctx.stream2(&mut dst, dpos, src, spos, 2); assert!(r != Ok(1) || true); } + r.map(|left| { assert_eq!(left, 0); unsafe { shim_collected(cctx.0) } }) + } else { + c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }) + }; + let r = n.ok().map(|n| { + let mut s: ShimState = unsafe { std::mem::zeroed() }; + unsafe { shim_state(cctx.0, &mut s) }; + let row_used = s.use_row != 0 && (3..=5).contains(&s.cp[6]); + let table = |p: *const u32, n: usize| if n == 0 { Vec::new() } else { unsafe { std::slice::from_raw_parts(p, n) }.to_vec() }; + let state = State { + has_dms: s.has_dms != 0, next_to_update: s.next_to_update, loaded_dict_end: s.loaded_dict_end, low_limit: s.low_limit, dict_limit: s.dict_limit, + hash_salt_entropy: s.hash_salt_entropy, lazy_skipping: s.lazy_skipping != 0, hash_cache: s.hash_cache, rep: s.rep, + hash_table: table(s.hash_table, 1 << s.cp[2]), + chain_table: table(s.chain_table, if s.cp[6] != 1 && !row_used { 1 << s.cp[1] } else { 0 }), + tag_table: if row_used { unsafe { std::slice::from_raw_parts(s.tag_table, 1 << s.cp[2]) }.to_vec() } else { Vec::new() }, + }; + COut { + seqs: out[..n].iter().map(|s| (s.lit_length, s.match_length, s.rep, if s.rep == 0 { s.offset } else { 0 })).collect(), + cp: s.cp, use_row: s.use_row != 0, cdict_cp, state, + } + }); + if !cdict.is_null() { unsafe { ZSTD_freeCDict(cdict) }; } + r +} + +fn first_diff(a: &[T], b: &[T]) -> Option { + if a.len() != b.len() { return Some(a.len().min(b.len())); } + a.iter().zip(b).position(|(a, b)| a != b) +} + +/// `strategies`: which of 1..=9 to test. `modes`: which kinds of dictionary. +pub fn run(seed: u64, iters: u64, strategies: &[u32], modes: &[Mode], stream: bool) { + let mut skipped = 0u64; + let mut dms_blocks = 0u64; + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if i % 2000 == 0 { eprintln!("seqs iter {i} (seed {s})"); } + let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071, 400_000]); + let len = rng.range(7, max); + let mut src = data(&mut rng, len); + let mode = *rng.pick(modes); + let dict = if mode != Mode::NoDict { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; + let mode = if dict.is_empty() { Mode::NoDict } else { mode }; + if !dict.is_empty() && rng.chance(2, 3) { + for _ in 0..rng.range(1, 6) { + let n = rng.size(dict.len().min(src.len())); + let at = if rng.chance(1, 3) { dict.len() - n } else { rng.below(dict.len() - n + 1) }; + let to = if rng.chance(1, 4) { 0 } else { rng.below(src.len() - n + 1) }; + src[to..to + n].copy_from_slice(&dict[at..at + n]); + } + } + let strategy = *rng.pick(strategies); + let p = if rng.chance(1, 3) { + // the parameters of a level, with the strategy forced + let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; + p.strategy = strategy; + p + } else { + CParams { + window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, + search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, + target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, + } + }; + let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; + let use_row = rng.chance(1, 2); + let streaming = stream && rng.chance(3, 4); + let Some(c) = c_frame(arr(p), use_row, mode, &dict, &src, streaming) else { skipped += 1; continue }; + let d = match mode { Mode::NoDict => Dict::None, Mode::Prefix => Dict::Prefix(&dict), Mode::CDict => Dict::CDict(&dict, c.cdict_cp) }; + let what = format!("seed {s}: {mode:?} streaming {streaming} cp {:?} cdict cp {:?} row {} dict {} src {}", c.cp, c.cdict_cp, c.use_row, dict.len(), src.len()); + let (got, state) = bun_zstd::testing::frame(c.cp, c.use_row, d, &src, streaming).unwrap_or_else(|e| panic!("{what}: error {e:?}")); + if mode == Mode::CDict && dict.len() >= 8 { dms_blocks += 1; } + if let Some(at) = first_diff(&got, &c.seqs) { + panic!("{what}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", got.get(at), c.seqs.get(at), got.len(), c.seqs.len()); + } + if state != c.state { + let (a, b) = (&state, &c.state); + panic!("{what}: state differs: hash table {:?} chain table {:?} tag table {:?}\n got {:?}\n exp {:?}", + first_diff(&a.hash_table, &b.hash_table), first_diff(&a.chain_table, &b.chain_table), first_diff(&a.tag_table, &b.tag_table), + (a.has_dms, a.next_to_update, a.loaded_dict_end, a.low_limit, a.dict_limit, a.hash_salt_entropy, a.lazy_skipping, a.hash_cache, a.rep), + (b.has_dms, b.next_to_update, b.loaded_dict_end, b.low_limit, b.dict_limit, b.hash_salt_entropy, b.lazy_skipping, b.hash_cache, b.rep)); + } + } + println!("seqs ok: {iters} iterations from seed {seed} ({skipped} skipped, {dms_blocks} with an attached CDict), strategies {strategies:?}, modes {modes:?}, streaming {stream}"); +} + +/// Speed of the Rust block compressors against the C ones, on a file. +pub fn bench(path: &str, mode: Mode) { + let all = std::fs::read(path).unwrap(); + let (dict, src) = if mode == Mode::NoDict { (&[][..], &all[..]) } else { all.split_at(100_000.min(all.len() / 2)) }; + for (level, strategy, use_row) in [(5, 3, false), (5, 3, true), (7, 4, false), (7, 4, true), (10, 5, false), (10, 5, true), (12, 5, true), (13, 6, false), (15, 6, false)] { + let mut p = unsafe { ZSTD_getCParams(level, if mode == Mode::CDict { 100_000 } else { src.len() as u64 }, dict.len()) }; + p.strategy = strategy; + let mut best = [f64::MAX; 2]; + let mut nseq = [0usize; 2]; + for _ in 0..3 { + let t = std::time::Instant::now(); + let c = c_frame(arr(p), use_row, mode, dict, src, false).unwrap(); + best[0] = best[0].min(t.elapsed().as_secs_f64()); + nseq[0] = c.seqs.len(); + let d = match mode { Mode::NoDict => Dict::None, Mode::Prefix => Dict::Prefix(dict), Mode::CDict => Dict::CDict(dict, c.cdict_cp) }; + let t = std::time::Instant::now(); + let (got, _) = bun_zstd::testing::frame(c.cp, c.use_row, d, src, false).unwrap(); + best[1] = best[1].min(t.elapsed().as_secs_f64()); + nseq[1] = got.len(); + } + let mb = src.len() as f64 / 1e6; + println!("{mode:?} level {level} strategy {strategy} row {use_row}: C {:.1} MB/s, Rust {:.1} MB/s ({:.0}%) seqs {} {}", mb / best[0], mb / best[1], 100.0 * best[0] / best[1], nseq[0], nseq[1]); + } +} diff --git a/src/zstd/wip/work/lazy/lazy.exp-filter.rs b/src/zstd/wip/work/lazy/lazy.exp-filter.rs new file mode 100644 index 000000000000..916221b41ed5 --- /dev/null +++ b/src/zstd/wip/work/lazy/lazy.exp-filter.rs @@ -0,0 +1,1360 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use super::internal::{ + BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, + SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, + hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, + offset_to_offbase, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +const K_LAZY_SKIPPING_STEP: usize = 8; + +// `ZSTD_dictMode_e`, as an argument of a const generic. +const NO_DICT: u8 = DictMode::NoDict as u8; +const EXT_DICT: u8 = DictMode::ExtDict as u8; +const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; + +/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. +#[inline(always)] +const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { + dms.end.wrapping_sub(dms.base) as u32 +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. +#[inline(always)] +fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { + let hash_log = ms.c_params.hash_log; + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let bt = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); + *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. +fn insert_dubt1( + ms: &mut MatchState, + src: &BlockSrc<'_>, + curr: u32, + input_end: usize, + mut nb_compares: u32, + bt_low: u32, +) -> Result<()> { + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let (ip_buf, ip, iend) = if curr >= dict_limit { + (src.buf, src.pos_of(curr), input_end) + } else { + (src.dict_buf, src.dict_pos_of(curr), dict_end) + }; + let window_valid = ms.window.low_limit; + let max_distance = 1u32 << ms.c_params.window_log; + let window_low = if curr.wrapping_sub(window_valid) > max_distance { + curr - max_distance + } else { + window_valid + }; + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + // The next sorted candidate. The other slot has the previous unsorted one. + let mut match_index = bt.at(smaller_ptr)?; + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let in_prefix = + DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; + // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = if in_prefix { + (src.buf, src.pos_of(match_index)) + } else { + (src.dict_buf, src.dict_pos_of(match_index)) + }; + + if in_prefix || curr < dict_limit { + // Both are in the same segment. + match_length += count( + ip_buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + ip_buf, + ip + match_length, + match_buf, + match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if match_index as usize + match_length >= dict_limit as usize { + (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); + } + } + + if ip + match_length == iend { + // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0) +} + +/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. +fn dubt_find_better_dict_match( + ms: &MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mut best_length: usize, + mut nb_compares: u32, + mls: u32, +) -> Result { + let dms = src.dms.ok_or(BUG)?; + let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(h)?; + + let prefix_start = src.pos_of(ms.window.dict_limit); + let curr = src.index_of(ip); + let dict_high_limit = dms_end_index(&dms); + let dict_low_limit = dms.ms.window.low_limit; + let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); + + let dict_bt = dms.ms.chain_table.as_slice(); + let bt_log = dms.ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { + dict_low_limit + } else { + dict_high_limit - bt_mask + }; + + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + + while nb_compares != 0 && dict_match_index > dict_low_limit { + let next_ptr = 2 * (dict_match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let match_index = dict_match_index.wrapping_add(dict_index_delta); + match_length += count_2segments( + src.buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + iend, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(offset_ptr.wrapping_add(1)) as i32 + { + best_length = match_length; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // No way to know which is smaller. + break; + } + } + + if dict_match_index <= bt_low { + // Beyond the tree. + break; + } + let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { + let match_pos = src.pos_of(match_index); + src.buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < src.buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dict_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dict_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + Ok(best_length) +} + +/// `ZSTD_DUBT_findBestMatch` +#[inline(always)] +fn dubt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + off_base_ptr: &mut u32, + mls: u32, +) -> Result { + let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; + let mut match_index = ms.hash_table.at(h)?; + + let curr = src.index_of(ip); + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = curr.saturating_sub(bt_mask); + let unsort_limit = bt_low.max(window_low); + let mut nb_compares = 1u32 << ms.c_params.search_log; + let mut nb_candidates = nb_compares; + let mut previous_candidate = 0u32; + + // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. + let bt = ms.chain_table.as_mut_slice(); + let mut next_candidate = 2 * (match_index & bt_mask) as usize; + while match_index > unsort_limit + && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK + && nb_candidates > 1 + { + bt.put(next_candidate + 1, previous_candidate)?; + previous_candidate = match_index; + match_index = bt.at(next_candidate)?; + next_candidate = 2 * (match_index & bt_mask) as usize; + nb_candidates -= 1; + } + + // If the last candidate is still unsorted, cut the list there. + if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { + *bt.array_at_mut(next_candidate)? = [0, 0]; + } + + // Sort the candidates, the oldest first. + match_index = previous_candidate; + while match_index != 0 { + let next_candidate_idx = ms + .chain_table + .at(2 * (match_index & bt_mask) as usize + 1)?; + insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; + match_index = next_candidate_idx; + nb_candidates += 1; + } + + // Find the longest match. + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + let mut match_end_idx = curr + 8 + 1; + let mut best_length = 0usize; + + match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); + + if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + src.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + let dict_match_pos = src.dict_pos_of(match_index); + match_length += count_2segments( + src.buf, + ip + match_length, + src.dict_buf, + dict_match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + (match_buf, match_pos) = (src.dict_buf, dict_match_pos); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index + match_length as u32; + } + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(*off_base_ptr) as i32 + { + best_length = match_length; + *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // Equal: no way to know which is smaller. + if DICT_MODE == DICT_MATCH_STATE { + // Nothing in the dictionary can be longer. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0)?; + + if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { + best_length = dubt_find_better_dict_match( + ms, + src, + ip, + iend, + off_base_ptr, + best_length, + nb_compares, + mls, + )?; + } + + // Skip repetitive patterns. + ms.next_to_update = match_end_idx - 8; + Ok(best_length) +} + +/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. +/// `off_base_ptr` is written to if there is one. +#[inline(never)] +fn bt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + off_base_ptr: &mut u32, +) -> Result { + if ip < src.pos_of(ms.next_to_update) { + // In an area that is skipped. + return Ok(0); + } + update_dubt(ms, src.prefix(), ip, MLS)?; + dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) +} + +// ── hash chains ─────────────────────────────────────────────────────────── + +/// `ZSTD_insertAndFindFirstIndex_internal` +#[inline(always)] +fn insert_and_find_first_index_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + lazy_skipping: bool, +) -> Result { + let hash_log = ms.c_params.hash_log; + let chain_mask = (1u32 << ms.c_params.chain_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let chain_table = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); + chain_table.put((idx & chain_mask) as usize, next_in_chain)?; + // While skipping, not every position is inserted. + if lazy_skipping { + break; + } + } + + ms.next_to_update = target; + hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) +} + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) +} + +/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn hc_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let chain_size = 1u32 << ms.c_params.chain_log; + let chain_mask = chain_size - 1; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let min_chain = curr.saturating_sub(chain_size); + let mut nb_attempts = 1u32 << ms.c_params.search_log; + let mut ml = 4 - 1; + + let mut match_index = + insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; + let chain_table = ms.chain_table.as_slice(); + + while match_index >= low_limit && nb_attempts > 0 { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + + if match_index <= min_chain { + break; + } + match_index = chain_table.at((match_index & chain_mask) as usize)?; + nb_attempts -= 1; + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_chain_table = dms.ms.chain_table.as_slice(); + let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; + let dms_chain_mask = dms_chain_size - 1; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_size = dms_end_index(&dms); + let dms_index_delta = dict_limit.wrapping_sub(dms_size); + let dms_min_chain = dms_size.saturating_sub(dms_chain_size); + + let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; + match_index = dms.ms.hash_table.at(h)?; + + while match_index >= dms_lowest_index && nb_attempts > 0 { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + + if match_index <= dms_min_chain { + break; + } + match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; + nb_attempts -= 1; + } + } + + Ok(ml) +} + +// ── row-based match finder ──────────────────────────────────────────────── +// +// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits +// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag +// table is the "head": which entry was written last. + +pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; +const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; +const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; + +/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. +#[inline(always)] +fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { + let mut next = u32::from(*head).wrapping_sub(1) & row_mask; + if next == 0 { + next = row_mask; + } + *head = next as u8; + next +} + +/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. +#[inline(always)] +fn row_fill_hash_cache( + ms: &mut MatchState, + prefix: Prefix<'_>, + mls: u32, + idx: u32, + i_limit: usize, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); + let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; + + for idx in idx..lim { + let hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + ms.hash_salt, + )? as u32; + let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; + ms.hash_cache.put(cached, hash)?; + } + Ok(()) +} + +/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes +/// further takes its place. +#[inline(always)] +fn row_next_cached_hash( + cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], + prefix: Prefix<'_>, + idx: u32, + hash_log: u32, + mls: u32, + hash_salt: u64, +) -> Result { + let new_hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32; + let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; + Ok(core::mem::replace(cached, new_hash)) +} + +/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. +#[inline(always)] +fn row_update_internal_impl( + ms: &mut MatchState, + prefix: Prefix<'_>, + update_start_idx: u32, + update_end_idx: u32, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let hash_salt = ms.hash_salt; + let hash_table = ms.hash_table.as_mut_slice(); + let tag_table = ms.tag_table.as_mut_slice(); + + for idx in update_start_idx..update_end_idx { + let hash = if use_cache { + row_next_cached_hash(&mut ms.hash_cache, prefix, idx, hash_log, mls, hash_salt)? + } else { + hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32 + }; + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; + tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; + hash_table.put(rel_row + pos, idx)?; + } + Ok(()) +} + +/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is +/// when compressing, not all of them if they are many. +#[inline(always)] +fn row_update_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + const K_SKIP_THRESHOLD: u32 = 384; + const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; + const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; + + let mut idx = ms.next_to_update; + let target = prefix.index_of(ip); + + if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { + let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; + row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; + idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; + row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; + } + row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the +/// row-based match finder. +pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { + let row_log = ms.c_params.search_log.clamp(4, 6); + let row_mask = (1u32 << row_log) - 1; + let mls = ms.c_params.min_match.min(6); + row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) +} + +/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. +#[inline(always)] +fn row_get_match_mask( + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + head: u32, +) -> u64 { + let mut matches = 0u64; + let (chunks, _) = tag_row.as_chunks::<16>(); + for (n, chunk) in chunks.iter().enumerate() { + let mut chunk_matches = 0u16; + for (i, &entry) in chunk.iter().enumerate() { + chunk_matches |= u16::from(entry == tag) << i; + } + matches |= u64::from(chunk_matches) << (16 * n); + } + match ROW_ENTRIES { + 16 => u64::from((matches as u16).rotate_right(head)), + 32 => u64::from((matches as u32).rotate_right(head)), + _ => matches.rotate_right(head), + } +} + +/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, +/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than +/// `low_limit`. Returns how many there are. +#[inline(always)] +fn row_collect_matches( + row: &[u32; ROW_ENTRIES], + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + low_limit: u32, + nb_attempts: &mut u32, + match_buffer: &mut [u32; ROW_ENTRIES], + read_match: impl Fn(u32) -> Result, + ip_bytes: u32, +) -> Result { + let row_mask = ROW_ENTRIES as u32 - 1; + let head = u32::from(tag_row.at(0)?) & row_mask; + let mut num_matches = 0usize; + let mut matches = row_get_match_mask(tag_row, tag, head); + + while matches > 0 && *nb_attempts > 0 { + let match_pos = (head + matches.trailing_zeros()) & row_mask; + matches &= matches - 1; + if match_pos == 0 { + // The head, which happens to be the same as the tag. + continue; + } + let match_index = row.at(match_pos as usize)?; + if match_index < low_limit { + break; + } + match_buffer.put(num_matches, match_index)?; + num_matches += usize::from(read_match(match_index)? == ip_bytes); + *nb_attempts -= 1; + } + Ok(num_matches) +} + +/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn row_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let hash_log = ms.row_hash_log; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let row_log = ROW_ENTRIES.trailing_zeros(); + let row_mask = ROW_ENTRIES as u32 - 1; + // There are no more searches than entries in a row. + let capped_search_log = ms.c_params.search_log.min(row_log); + let hash_salt = ms.hash_salt; + let mut nb_attempts = 1u32 << capped_search_log; + let mut ml = 4 - 1; + let mut match_buffer = [0u32; ROW_ENTRIES]; + let ip_bytes = read_le32(buf, ip)?; + + // Update the tables up to, and not including, `ip`. + let hash = if ms.lazy_skipping { + // While skipping, not every position is inserted. + ms.next_to_update = curr; + hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 + } else { + let prefix = src.prefix(); + row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; + row_next_cached_hash(&mut ms.hash_cache, prefix, curr, hash_log, MLS, hash_salt)? + }; + ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); + + { + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; + let row = ms.hash_table.array_at_mut::(rel_row)?; + let tag_row = ms.tag_table.array_at_mut::(rel_row)?; + let num_matches = row_collect_matches( + row, + tag_row, + tag, + low_limit, + &mut nb_attempts, + &mut match_buffer, + |match_index| { + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + read_le32(buf, src.pos_of(match_index)) + } else { + read_le32(src.dict_buf, src.dict_pos_of(match_index)) + } + }, + ip_bytes, + )?; + + // Insert `ip`. + let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; + tag_row.put(pos, tag)?; + row.put(pos, ms.next_to_update)?; + ms.next_to_update += 1; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + } + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); + + let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + let dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; + let dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let num_matches = row_collect_matches( + dms.ms.hash_table.array_at::(dms_rel_row)?, + dms.ms.tag_table.array_at::(dms_rel_row)?, + (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, + dms_lowest_index, + &mut nb_attempts, + &mut match_buffer, + |match_index| read_le32(dms.buf, dms.pos_of(match_index)), + ip_bytes, + )?; + + for &match_index in match_buffer.sub(..num_matches)? { + let current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + dms.pos_of(match_index) + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + } + } + + Ok(ml) +} + +// ── the parsers ─────────────────────────────────────────────────────────── + +// `searchMethod_e`, as an argument of a const generic. +const SEARCH_HASH_CHAIN: u8 = 0; +const SEARCH_BINARY_TREE: u8 = 1; +const SEARCH_ROW_HASH: u8 = 2; + +/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. +#[inline(always)] +fn search_max( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mls: u32, + row_log: u32, +) -> Result { + match SEARCH_METHOD { + SEARCH_HASH_CHAIN => match mls { + 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + SEARCH_BINARY_TREE => match mls { + 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + _ => match (mls, row_log) { + (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + } +} + +/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match +/// state is searched. +#[derive(Clone, Copy)] +struct DictSegment<'a> { + buf: &'a [u8], + /// Its sum with an index of the window is a position in `buf`. It wraps around. + base: usize, + /// Where the segment starts in `buf`. + start: usize, + /// Where it ends. + end: usize, +} + +/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. +#[inline(always)] +fn rep_length( + ms: &MatchState, + src: &BlockSrc<'_>, + dict: &DictSegment<'_>, + ip: usize, + offset: u32, +) -> Result { + let buf = src.buf; + let iend = src.end; + if DICT_MODE == NO_DICT { + let rep_match = ip.wrapping_sub(offset as usize); + if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { + return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); + } + return Ok(0); + } + + let dict_limit = ms.window.dict_limit; + let curr = src.index_of(ip); + let rep_index = curr.wrapping_sub(offset); + if !index_overlap_check(dict_limit, rep_index) { + return Ok(0); + } + if DICT_MODE == EXT_DICT { + // The offset may lead to what is no longer in the window. + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + if offset > curr.wrapping_sub(window_low) { + return Ok(0); + } + } + let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { + let rep_match = dict.base.wrapping_add(rep_index as usize); + (dict.buf, rep_match, dict.end) + } else { + (buf, src.pos_of(rep_index), iend) + }; + if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { + return Ok(0); + } + let prefix_start = src.pos_of(dict_limit); + Ok(count_2segments( + buf, + ip + 4, + rep_buf, + rep_match + 4, + iend, + rep_end, + prefix_start, + )? + 4) +} + +/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions +/// further a better match is looked for, once there is one. +#[inline(never)] +fn compress_block_lazy_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let prefix_lowest_index = ms.window.dict_limit; + let prefix_lowest = src.pos_of(prefix_lowest_index); + let mls = ms.c_params.min_match.clamp(4, 6); + let row_log = ms.c_params.search_log.clamp(4, 6); + + let [rep_0, rep_1, _] = rep; + let mut offset_1 = *rep_0; + let mut offset_2 = *rep_1; + let mut offset_saved_1 = 0u32; + let mut offset_saved_2 = 0u32; + + ms.lazy_skipping = false; + + let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { + 8 + ZSTD_ROW_HASH_CACHE_SIZE + } else { + 8 + }; + let Some(ilimit) = iend.checked_sub(margin) else { + // There is no position to search, nor to hash. + return Ok(iend - istart); + }; + + let dict = match DICT_MODE { + NO_DICT => DictSegment { + buf: &[], + base: 0, + start: 0, + end: 0, + }, + EXT_DICT => DictSegment { + buf: src.dict_buf, + base: src.dict_base, + start: src.dict_pos_of(ms.window.low_limit), + end: src.dict_pos_of(prefix_lowest_index), + }, + _ => { + let dms = src.dms.ok_or(BUG)?; + let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); + DictSegment { + buf: dms.buf, + base: dms.base.wrapping_sub(dict_index_delta as usize), + start: dms.pos_of(dms.ms.window.dict_limit), + end: dms.end, + } + } + }; + + if DICT_MODE == EXT_DICT { + ip += usize::from(ip == prefix_lowest); + } else { + let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); + ip += usize::from(dict_and_prefix_length == 0); + } + if DICT_MODE == NO_DICT { + let curr = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); + let max_rep = curr - window_low; + if offset_2 > max_rep { + offset_saved_2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved_1 = offset_1; + offset_1 = 0; + } + } + + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + + 'parse: while ip < ilimit { + let mut match_length; + let mut off_base = REPCODE1_TO_OFFBASE; + let mut start = ip + 1; + + 'found: { + // Check the repeat offset. + match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; + if DEPTH == 0 && match_length != 0 { + break 'found; + } + + // First search. + { + let mut off_base_found = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut off_base_found, + mls, + row_log, + )?; + if ml2 > match_length { + match_length = ml2; + start = ip; + off_base = off_base_found; + } + } + + if match_length < 4 { + // Jump faster over incompressible sections. + let step = (ip - anchor) >> K_SEARCH_STRENGTH; + ip += step + 1; + // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of + // no use out of the tables. As in the C code, it does not start at the same step in all modes. + ms.lazy_skipping = if DICT_MODE == EXT_DICT { + step > K_LAZY_SKIPPING_STEP + } else { + step + 1 > K_LAZY_SKIPPING_STEP + }; + continue 'parse; + } + + // Look for something better at the next positions. + if DEPTH >= 1 { + while ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 3) as i32; + let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + + // Nothing better: try one position further. + if DEPTH == 2 && ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 4) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + } + break; + } + } + + // Catch up: the match may start earlier. + if offbase_is_offset(off_base) { + let offset = offbase_to_offset(off_base); + let match_index = src.index_of(start).wrapping_sub(offset); + let (match_buf, mut match_pos, m_start) = + if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { + let match_pos = dict.base.wrapping_add(match_index as usize); + (dict.buf, match_pos, dict.start) + } else { + (buf, src.pos_of(match_index), prefix_lowest) + }; + while start > anchor + && match_pos > m_start + && buf.at(start - 1)? == match_buf.at(match_pos - 1)? + { + start -= 1; + match_pos -= 1; + match_length += 1; + } + offset_2 = offset_1; + offset_1 = offset; + } + } + + // Store the sequence. + seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; + ip = start + match_length; + anchor = ip; + + if ms.lazy_skipping { + // There was a match: every position is inserted again. + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + ms.lazy_skipping = false; + } + + // Check the second repeat offset right away. + while ip <= ilimit { + match_length = rep_length::(ms, src, &dict, ip, offset_2)?; + if match_length == 0 { + break; + } + (offset_1, offset_2) = (offset_2, offset_1); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; + ip += match_length; + anchor = ip; + } + } + + // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. + if offset_saved_1 != 0 && offset_1 != 0 { + offset_saved_2 = offset_saved_1; + } + + // Save the repeat offsets for the next block. + *rep_0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved_1 + }; + *rep_1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved_2 + }; + + // The size of the last literals. + Ok(iend - anchor) +} + +/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. +#[inline(always)] +fn compress_block_lazy_any_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + match dict_mode { + DictMode::NoDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::ExtDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + compress_block_lazy_generic::( + ms, seq_store, rep, src, + ) + } + } +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row: bool, + dict_mode: DictMode, +) -> Result { + match (strategy, use_row) { + (Strategy::Greedy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Greedy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( + ms, seq_store, rep, src, dict_mode, + ), + _ => Err(BUG), + } +} diff --git a/src/zstd/wip/work/lazy/lazy.exp-prefetch.rs b/src/zstd/wip/work/lazy/lazy.exp-prefetch.rs new file mode 100644 index 000000000000..267b0e7c9577 --- /dev/null +++ b/src/zstd/wip/work/lazy/lazy.exp-prefetch.rs @@ -0,0 +1,1384 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use super::internal::{ + BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, + SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, + hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, + offset_to_offbase, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +const K_LAZY_SKIPPING_STEP: usize = 8; + +// `ZSTD_dictMode_e`, as an argument of a const generic. +const NO_DICT: u8 = DictMode::NoDict as u8; +const EXT_DICT: u8 = DictMode::ExtDict as u8; +const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; + +/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. +#[inline(always)] +const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { + dms.end.wrapping_sub(dms.base) as u32 +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. +#[inline(always)] +fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { + let hash_log = ms.c_params.hash_log; + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let bt = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); + *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. +fn insert_dubt1( + ms: &mut MatchState, + src: &BlockSrc<'_>, + curr: u32, + input_end: usize, + mut nb_compares: u32, + bt_low: u32, +) -> Result<()> { + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let (ip_buf, ip, iend) = if curr >= dict_limit { + (src.buf, src.pos_of(curr), input_end) + } else { + (src.dict_buf, src.dict_pos_of(curr), dict_end) + }; + let window_valid = ms.window.low_limit; + let max_distance = 1u32 << ms.c_params.window_log; + let window_low = if curr.wrapping_sub(window_valid) > max_distance { + curr - max_distance + } else { + window_valid + }; + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + // The next sorted candidate. The other slot has the previous unsorted one. + let mut match_index = bt.at(smaller_ptr)?; + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let in_prefix = + DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; + // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = if in_prefix { + (src.buf, src.pos_of(match_index)) + } else { + (src.dict_buf, src.dict_pos_of(match_index)) + }; + + if in_prefix || curr < dict_limit { + // Both are in the same segment. + match_length += count( + ip_buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + ip_buf, + ip + match_length, + match_buf, + match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if match_index as usize + match_length >= dict_limit as usize { + (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); + } + } + + if ip + match_length == iend { + // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0) +} + +/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. +fn dubt_find_better_dict_match( + ms: &MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mut best_length: usize, + mut nb_compares: u32, + mls: u32, +) -> Result { + let dms = src.dms.ok_or(BUG)?; + let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(h)?; + + let prefix_start = src.pos_of(ms.window.dict_limit); + let curr = src.index_of(ip); + let dict_high_limit = dms_end_index(&dms); + let dict_low_limit = dms.ms.window.low_limit; + let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); + + let dict_bt = dms.ms.chain_table.as_slice(); + let bt_log = dms.ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { + dict_low_limit + } else { + dict_high_limit - bt_mask + }; + + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + + while nb_compares != 0 && dict_match_index > dict_low_limit { + let next_ptr = 2 * (dict_match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let match_index = dict_match_index.wrapping_add(dict_index_delta); + match_length += count_2segments( + src.buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + iend, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(offset_ptr.wrapping_add(1)) as i32 + { + best_length = match_length; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // No way to know which is smaller. + break; + } + } + + if dict_match_index <= bt_low { + // Beyond the tree. + break; + } + let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { + let match_pos = src.pos_of(match_index); + src.buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < src.buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dict_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dict_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + Ok(best_length) +} + +/// `ZSTD_DUBT_findBestMatch` +#[inline(always)] +fn dubt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + off_base_ptr: &mut u32, + mls: u32, +) -> Result { + let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; + let mut match_index = ms.hash_table.at(h)?; + + let curr = src.index_of(ip); + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = curr.saturating_sub(bt_mask); + let unsort_limit = bt_low.max(window_low); + let mut nb_compares = 1u32 << ms.c_params.search_log; + let mut nb_candidates = nb_compares; + let mut previous_candidate = 0u32; + + // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. + let bt = ms.chain_table.as_mut_slice(); + let mut next_candidate = 2 * (match_index & bt_mask) as usize; + while match_index > unsort_limit + && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK + && nb_candidates > 1 + { + bt.put(next_candidate + 1, previous_candidate)?; + previous_candidate = match_index; + match_index = bt.at(next_candidate)?; + next_candidate = 2 * (match_index & bt_mask) as usize; + nb_candidates -= 1; + } + + // If the last candidate is still unsorted, cut the list there. + if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { + *bt.array_at_mut(next_candidate)? = [0, 0]; + } + + // Sort the candidates, the oldest first. + match_index = previous_candidate; + while match_index != 0 { + let next_candidate_idx = ms + .chain_table + .at(2 * (match_index & bt_mask) as usize + 1)?; + insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; + match_index = next_candidate_idx; + nb_candidates += 1; + } + + // Find the longest match. + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + let mut match_end_idx = curr + 8 + 1; + let mut best_length = 0usize; + + match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); + + if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + src.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + let dict_match_pos = src.dict_pos_of(match_index); + match_length += count_2segments( + src.buf, + ip + match_length, + src.dict_buf, + dict_match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + (match_buf, match_pos) = (src.dict_buf, dict_match_pos); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index + match_length as u32; + } + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(*off_base_ptr) as i32 + { + best_length = match_length; + *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // Equal: no way to know which is smaller. + if DICT_MODE == DICT_MATCH_STATE { + // Nothing in the dictionary can be longer. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0)?; + + if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { + best_length = dubt_find_better_dict_match( + ms, + src, + ip, + iend, + off_base_ptr, + best_length, + nb_compares, + mls, + )?; + } + + // Skip repetitive patterns. + ms.next_to_update = match_end_idx - 8; + Ok(best_length) +} + +/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. +/// `off_base_ptr` is written to if there is one. +#[inline(never)] +fn bt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + off_base_ptr: &mut u32, +) -> Result { + if ip < src.pos_of(ms.next_to_update) { + // In an area that is skipped. + return Ok(0); + } + update_dubt(ms, src.prefix(), ip, MLS)?; + dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) +} + +// ── hash chains ─────────────────────────────────────────────────────────── + +/// `ZSTD_insertAndFindFirstIndex_internal` +#[inline(always)] +fn insert_and_find_first_index_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + lazy_skipping: bool, +) -> Result { + let hash_log = ms.c_params.hash_log; + let chain_mask = (1u32 << ms.c_params.chain_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let chain_table = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); + chain_table.put((idx & chain_mask) as usize, next_in_chain)?; + // While skipping, not every position is inserted. + if lazy_skipping { + break; + } + } + + ms.next_to_update = target; + hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) +} + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) +} + +/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn hc_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let chain_size = 1u32 << ms.c_params.chain_log; + let chain_mask = chain_size - 1; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let min_chain = curr.saturating_sub(chain_size); + let mut nb_attempts = 1u32 << ms.c_params.search_log; + let mut ml = 4 - 1; + + let mut match_index = + insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; + let chain_table = ms.chain_table.as_slice(); + + while match_index >= low_limit && nb_attempts > 0 { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + + if match_index <= min_chain { + break; + } + match_index = chain_table.at((match_index & chain_mask) as usize)?; + nb_attempts -= 1; + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_chain_table = dms.ms.chain_table.as_slice(); + let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; + let dms_chain_mask = dms_chain_size - 1; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_size = dms_end_index(&dms); + let dms_index_delta = dict_limit.wrapping_sub(dms_size); + let dms_min_chain = dms_size.saturating_sub(dms_chain_size); + + let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; + match_index = dms.ms.hash_table.at(h)?; + + while match_index >= dms_lowest_index && nb_attempts > 0 { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + + if match_index <= dms_min_chain { + break; + } + match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; + nb_attempts -= 1; + } + } + + Ok(ml) +} + +// ── row-based match finder ──────────────────────────────────────────────── +// +// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits +// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag +// table is the "head": which entry was written last. + +pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; +const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; +const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; + +/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. +#[inline(always)] +fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { + let mut next = u32::from(*head).wrapping_sub(1) & row_mask; + if next == 0 { + next = row_mask; + } + *head = next as u8; + next +} + +#[inline(always)] +fn prefetch_l1(slice: &[T], index: usize) { + core::hint::prefetch_read(slice.as_ptr().wrapping_add(index), core::hint::Locality::L1); +} + +#[inline(always)] +fn row_prefetch(hash_table: &[u32], tag_table: &[u8], rel_row: usize, row_log: u32) { + prefetch_l1(hash_table, rel_row); + if row_log >= 5 { + prefetch_l1(hash_table, rel_row + 16); + } + prefetch_l1(tag_table, rel_row); + if row_log == 6 { + prefetch_l1(tag_table, rel_row + 32); + } +} + +/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. +#[inline(always)] +fn row_fill_hash_cache( + ms: &mut MatchState, + prefix: Prefix<'_>, + mls: u32, + idx: u32, + i_limit: usize, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); + let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; + + for idx in idx..lim { + let hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + ms.hash_salt, + )? as u32; + let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; + ms.hash_cache.put(cached, hash)?; + } + Ok(()) +} + +/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes +/// further takes its place. +#[inline(always)] +fn row_next_cached_hash( + cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], + hash_table: &[u32], + tag_table: &[u8], + row_log: u32, + prefix: Prefix<'_>, + idx: u32, + hash_log: u32, + mls: u32, + hash_salt: u64, +) -> Result { + let new_hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32; + row_prefetch(hash_table, tag_table, ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize, row_log); + let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; + Ok(core::mem::replace(cached, new_hash)) +} + +/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. +#[inline(always)] +fn row_update_internal_impl( + ms: &mut MatchState, + prefix: Prefix<'_>, + update_start_idx: u32, + update_end_idx: u32, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let hash_salt = ms.hash_salt; + let hash_table = ms.hash_table.as_mut_slice(); + let tag_table = ms.tag_table.as_mut_slice(); + + for idx in update_start_idx..update_end_idx { + let hash = if use_cache { + row_next_cached_hash(&mut ms.hash_cache, hash_table, tag_table, row_log, prefix, idx, hash_log, mls, hash_salt)? + } else { + hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32 + }; + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; + tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; + hash_table.put(rel_row + pos, idx)?; + } + Ok(()) +} + +/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is +/// when compressing, not all of them if they are many. +#[inline(always)] +fn row_update_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + const K_SKIP_THRESHOLD: u32 = 384; + const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; + const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; + + let mut idx = ms.next_to_update; + let target = prefix.index_of(ip); + + if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { + let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; + row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; + idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; + row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; + } + row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the +/// row-based match finder. +pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { + let row_log = ms.c_params.search_log.clamp(4, 6); + let row_mask = (1u32 << row_log) - 1; + let mls = ms.c_params.min_match.min(6); + row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) +} + +/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. +#[inline(always)] +fn row_get_match_mask( + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + head: u32, +) -> u64 { + let mut matches = 0u64; + let (chunks, _) = tag_row.as_chunks::<16>(); + for (n, chunk) in chunks.iter().enumerate() { + let mut chunk_matches = 0u16; + for (i, &entry) in chunk.iter().enumerate() { + chunk_matches |= u16::from(entry == tag) << i; + } + matches |= u64::from(chunk_matches) << (16 * n); + } + match ROW_ENTRIES { + 16 => u64::from((matches as u16).rotate_right(head)), + 32 => u64::from((matches as u32).rotate_right(head)), + _ => matches.rotate_right(head), + } +} + +/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, +/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than +/// `low_limit`. Returns how many there are. +#[inline(always)] +fn row_collect_matches( + row: &[u32; ROW_ENTRIES], + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + low_limit: u32, + nb_attempts: &mut u32, + match_buffer: &mut [u32; ROW_ENTRIES], + prefetch: impl Fn(u32), +) -> Result { + let row_mask = ROW_ENTRIES as u32 - 1; + let head = u32::from(tag_row.at(0)?) & row_mask; + let mut num_matches = 0usize; + let mut matches = row_get_match_mask(tag_row, tag, head); + + while matches > 0 && *nb_attempts > 0 { + let match_pos = (head + matches.trailing_zeros()) & row_mask; + matches &= matches - 1; + if match_pos == 0 { + // The head, which happens to be the same as the tag. + continue; + } + let match_index = row.at(match_pos as usize)?; + if match_index < low_limit { + break; + } + prefetch(match_index); + match_buffer.put(num_matches, match_index)?; + num_matches += 1; + *nb_attempts -= 1; + } + Ok(num_matches) +} + +/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn row_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let hash_log = ms.row_hash_log; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let row_log = ROW_ENTRIES.trailing_zeros(); + let row_mask = ROW_ENTRIES as u32 - 1; + // There are no more searches than entries in a row. + let capped_search_log = ms.c_params.search_log.min(row_log); + let hash_salt = ms.hash_salt; + let mut nb_attempts = 1u32 << capped_search_log; + let mut ml = 4 - 1; + let mut match_buffer = [0u32; ROW_ENTRIES]; + + // Update the tables up to, and not including, `ip`. + let hash = if ms.lazy_skipping { + // While skipping, not every position is inserted. + ms.next_to_update = curr; + hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 + } else { + let prefix = src.prefix(); + row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; + row_next_cached_hash(&mut ms.hash_cache, &ms.hash_table, &ms.tag_table, row_log, prefix, curr, hash_log, MLS, hash_salt)? + }; + ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); + + { + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; + let row = ms.hash_table.array_at_mut::(rel_row)?; + let tag_row = ms.tag_table.array_at_mut::(rel_row)?; + let num_matches = row_collect_matches( + row, + tag_row, + tag, + low_limit, + &mut nb_attempts, + &mut match_buffer, + |match_index| { + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + prefetch_l1(buf, src.pos_of(match_index)); + } else { + prefetch_l1(src.dict_buf, src.dict_pos_of(match_index)); + } + }, + )?; + + // Insert `ip`. + let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; + tag_row.put(pos, tag)?; + row.put(pos, ms.next_to_update)?; + ms.next_to_update += 1; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + } + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); + + let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + let dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; + let dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let num_matches = row_collect_matches( + dms.ms.hash_table.array_at::(dms_rel_row)?, + dms.ms.tag_table.array_at::(dms_rel_row)?, + (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, + dms_lowest_index, + &mut nb_attempts, + &mut match_buffer, + |match_index| prefetch_l1(dms.buf, dms.pos_of(match_index)), + )?; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + } + } + + Ok(ml) +} + +// ── the parsers ─────────────────────────────────────────────────────────── + +// `searchMethod_e`, as an argument of a const generic. +const SEARCH_HASH_CHAIN: u8 = 0; +const SEARCH_BINARY_TREE: u8 = 1; +const SEARCH_ROW_HASH: u8 = 2; + +/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. +#[inline(always)] +fn search_max( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mls: u32, + row_log: u32, +) -> Result { + match SEARCH_METHOD { + SEARCH_HASH_CHAIN => match mls { + 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + SEARCH_BINARY_TREE => match mls { + 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + _ => match (mls, row_log) { + (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + } +} + +/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match +/// state is searched. +#[derive(Clone, Copy)] +struct DictSegment<'a> { + buf: &'a [u8], + /// Its sum with an index of the window is a position in `buf`. It wraps around. + base: usize, + /// Where the segment starts in `buf`. + start: usize, + /// Where it ends. + end: usize, +} + +/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. +#[inline(always)] +fn rep_length( + ms: &MatchState, + src: &BlockSrc<'_>, + dict: &DictSegment<'_>, + ip: usize, + offset: u32, +) -> Result { + let buf = src.buf; + let iend = src.end; + if DICT_MODE == NO_DICT { + let rep_match = ip.wrapping_sub(offset as usize); + if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { + return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); + } + return Ok(0); + } + + let dict_limit = ms.window.dict_limit; + let curr = src.index_of(ip); + let rep_index = curr.wrapping_sub(offset); + if !index_overlap_check(dict_limit, rep_index) { + return Ok(0); + } + if DICT_MODE == EXT_DICT { + // The offset may lead to what is no longer in the window. + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + if offset > curr.wrapping_sub(window_low) { + return Ok(0); + } + } + let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { + let rep_match = dict.base.wrapping_add(rep_index as usize); + (dict.buf, rep_match, dict.end) + } else { + (buf, src.pos_of(rep_index), iend) + }; + if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { + return Ok(0); + } + let prefix_start = src.pos_of(dict_limit); + Ok(count_2segments( + buf, + ip + 4, + rep_buf, + rep_match + 4, + iend, + rep_end, + prefix_start, + )? + 4) +} + +/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions +/// further a better match is looked for, once there is one. +#[inline(never)] +fn compress_block_lazy_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let prefix_lowest_index = ms.window.dict_limit; + let prefix_lowest = src.pos_of(prefix_lowest_index); + let mls = ms.c_params.min_match.clamp(4, 6); + let row_log = ms.c_params.search_log.clamp(4, 6); + + let [rep_0, rep_1, _] = rep; + let mut offset_1 = *rep_0; + let mut offset_2 = *rep_1; + let mut offset_saved_1 = 0u32; + let mut offset_saved_2 = 0u32; + + ms.lazy_skipping = false; + + let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { + 8 + ZSTD_ROW_HASH_CACHE_SIZE + } else { + 8 + }; + let Some(ilimit) = iend.checked_sub(margin) else { + // There is no position to search, nor to hash. + return Ok(iend - istart); + }; + + let dict = match DICT_MODE { + NO_DICT => DictSegment { + buf: &[], + base: 0, + start: 0, + end: 0, + }, + EXT_DICT => DictSegment { + buf: src.dict_buf, + base: src.dict_base, + start: src.dict_pos_of(ms.window.low_limit), + end: src.dict_pos_of(prefix_lowest_index), + }, + _ => { + let dms = src.dms.ok_or(BUG)?; + let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); + DictSegment { + buf: dms.buf, + base: dms.base.wrapping_sub(dict_index_delta as usize), + start: dms.pos_of(dms.ms.window.dict_limit), + end: dms.end, + } + } + }; + + if DICT_MODE == EXT_DICT { + ip += usize::from(ip == prefix_lowest); + } else { + let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); + ip += usize::from(dict_and_prefix_length == 0); + } + if DICT_MODE == NO_DICT { + let curr = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); + let max_rep = curr - window_low; + if offset_2 > max_rep { + offset_saved_2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved_1 = offset_1; + offset_1 = 0; + } + } + + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + + 'parse: while ip < ilimit { + let mut match_length; + let mut off_base = REPCODE1_TO_OFFBASE; + let mut start = ip + 1; + + 'found: { + // Check the repeat offset. + match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; + if DEPTH == 0 && match_length != 0 { + break 'found; + } + + // First search. + { + let mut off_base_found = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut off_base_found, + mls, + row_log, + )?; + if ml2 > match_length { + match_length = ml2; + start = ip; + off_base = off_base_found; + } + } + + if match_length < 4 { + // Jump faster over incompressible sections. + let step = (ip - anchor) >> K_SEARCH_STRENGTH; + ip += step + 1; + // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of + // no use out of the tables. As in the C code, it does not start at the same step in all modes. + ms.lazy_skipping = if DICT_MODE == EXT_DICT { + step > K_LAZY_SKIPPING_STEP + } else { + step + 1 > K_LAZY_SKIPPING_STEP + }; + continue 'parse; + } + + // Look for something better at the next positions. + if DEPTH >= 1 { + while ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 3) as i32; + let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + + // Nothing better: try one position further. + if DEPTH == 2 && ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 4) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + } + break; + } + } + + // Catch up: the match may start earlier. + if offbase_is_offset(off_base) { + let offset = offbase_to_offset(off_base); + let match_index = src.index_of(start).wrapping_sub(offset); + let (match_buf, mut match_pos, m_start) = + if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { + let match_pos = dict.base.wrapping_add(match_index as usize); + (dict.buf, match_pos, dict.start) + } else { + (buf, src.pos_of(match_index), prefix_lowest) + }; + while start > anchor + && match_pos > m_start + && buf.at(start - 1)? == match_buf.at(match_pos - 1)? + { + start -= 1; + match_pos -= 1; + match_length += 1; + } + offset_2 = offset_1; + offset_1 = offset; + } + } + + // Store the sequence. + seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; + ip = start + match_length; + anchor = ip; + + if ms.lazy_skipping { + // There was a match: every position is inserted again. + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + ms.lazy_skipping = false; + } + + // Check the second repeat offset right away. + while ip <= ilimit { + match_length = rep_length::(ms, src, &dict, ip, offset_2)?; + if match_length == 0 { + break; + } + (offset_1, offset_2) = (offset_2, offset_1); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; + ip += match_length; + anchor = ip; + } + } + + // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. + if offset_saved_1 != 0 && offset_1 != 0 { + offset_saved_2 = offset_saved_1; + } + + // Save the repeat offsets for the next block. + *rep_0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved_1 + }; + *rep_1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved_2 + }; + + // The size of the last literals. + Ok(iend - anchor) +} + +/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. +#[inline(always)] +fn compress_block_lazy_any_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + match dict_mode { + DictMode::NoDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::ExtDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + compress_block_lazy_generic::( + ms, seq_store, rep, src, + ) + } + } +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row: bool, + dict_mode: DictMode, +) -> Result { + match (strategy, use_row) { + (Strategy::Greedy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Greedy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( + ms, seq_store, rep, src, dict_mode, + ), + _ => Err(BUG), + } +} diff --git a/src/zstd/wip/work/lazy/lazy.v1.rs b/src/zstd/wip/work/lazy/lazy.v1.rs new file mode 100644 index 000000000000..94dbe308ce79 --- /dev/null +++ b/src/zstd/wip/work/lazy/lazy.v1.rs @@ -0,0 +1,1353 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use super::internal::{ + BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, + SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, + hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, + offset_to_offbase, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +const K_LAZY_SKIPPING_STEP: usize = 8; + +// `ZSTD_dictMode_e`, as an argument of a const generic. +const NO_DICT: u8 = DictMode::NoDict as u8; +const EXT_DICT: u8 = DictMode::ExtDict as u8; +const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; + +/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. +#[inline(always)] +const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { + dms.end.wrapping_sub(dms.base) as u32 +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. +#[inline(always)] +fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { + let hash_log = ms.c_params.hash_log; + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let bt = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); + *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. +fn insert_dubt1( + ms: &mut MatchState, + src: &BlockSrc<'_>, + curr: u32, + input_end: usize, + mut nb_compares: u32, + bt_low: u32, +) -> Result<()> { + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let (ip_buf, ip, iend) = if curr >= dict_limit { + (src.buf, src.pos_of(curr), input_end) + } else { + (src.dict_buf, src.dict_pos_of(curr), dict_end) + }; + let window_valid = ms.window.low_limit; + let max_distance = 1u32 << ms.c_params.window_log; + let window_low = if curr.wrapping_sub(window_valid) > max_distance { + curr - max_distance + } else { + window_valid + }; + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + // The next sorted candidate. The other slot has the previous unsorted one. + let mut match_index = bt.at(smaller_ptr)?; + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let in_prefix = + DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; + // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = if in_prefix { + (src.buf, src.pos_of(match_index)) + } else { + (src.dict_buf, src.dict_pos_of(match_index)) + }; + + if in_prefix || curr < dict_limit { + // Both are in the same segment. + match_length += count( + ip_buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + ip_buf, + ip + match_length, + match_buf, + match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if match_index as usize + match_length >= dict_limit as usize { + (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); + } + } + + if ip + match_length == iend { + // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0) +} + +/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. +fn dubt_find_better_dict_match( + ms: &MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mut best_length: usize, + mut nb_compares: u32, + mls: u32, +) -> Result { + let dms = src.dms.ok_or(BUG)?; + let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(h)?; + + let prefix_start = src.pos_of(ms.window.dict_limit); + let curr = src.index_of(ip); + let dict_high_limit = dms_end_index(&dms); + let dict_low_limit = dms.ms.window.low_limit; + let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); + + let dict_bt = dms.ms.chain_table.as_slice(); + let bt_log = dms.ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { + dict_low_limit + } else { + dict_high_limit - bt_mask + }; + + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + + while nb_compares != 0 && dict_match_index > dict_low_limit { + let next_ptr = 2 * (dict_match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let match_index = dict_match_index.wrapping_add(dict_index_delta); + match_length += count_2segments( + src.buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + iend, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(offset_ptr.wrapping_add(1)) as i32 + { + best_length = match_length; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // No way to know which is smaller. + break; + } + } + + if dict_match_index <= bt_low { + // Beyond the tree. + break; + } + let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { + let match_pos = src.pos_of(match_index); + src.buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < src.buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dict_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dict_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + Ok(best_length) +} + +/// `ZSTD_DUBT_findBestMatch` +#[inline(always)] +fn dubt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + off_base_ptr: &mut u32, + mls: u32, +) -> Result { + let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; + let mut match_index = ms.hash_table.at(h)?; + + let curr = src.index_of(ip); + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = curr.saturating_sub(bt_mask); + let unsort_limit = bt_low.max(window_low); + let mut nb_compares = 1u32 << ms.c_params.search_log; + let mut nb_candidates = nb_compares; + let mut previous_candidate = 0u32; + + // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. + let bt = ms.chain_table.as_mut_slice(); + let mut next_candidate = 2 * (match_index & bt_mask) as usize; + while match_index > unsort_limit + && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK + && nb_candidates > 1 + { + bt.put(next_candidate + 1, previous_candidate)?; + previous_candidate = match_index; + match_index = bt.at(next_candidate)?; + next_candidate = 2 * (match_index & bt_mask) as usize; + nb_candidates -= 1; + } + + // If the last candidate is still unsorted, cut the list there. + if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { + *bt.array_at_mut(next_candidate)? = [0, 0]; + } + + // Sort the candidates, the oldest first. + match_index = previous_candidate; + while match_index != 0 { + let next_candidate_idx = ms + .chain_table + .at(2 * (match_index & bt_mask) as usize + 1)?; + insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; + match_index = next_candidate_idx; + nb_candidates += 1; + } + + // Find the longest match. + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + let mut match_end_idx = curr + 8 + 1; + let mut best_length = 0usize; + + match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); + + if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + src.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + let dict_match_pos = src.dict_pos_of(match_index); + match_length += count_2segments( + src.buf, + ip + match_length, + src.dict_buf, + dict_match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + (match_buf, match_pos) = (src.dict_buf, dict_match_pos); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index + match_length as u32; + } + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(*off_base_ptr) as i32 + { + best_length = match_length; + *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // Equal: no way to know which is smaller. + if DICT_MODE == DICT_MATCH_STATE { + // Nothing in the dictionary can be longer. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0)?; + + if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { + best_length = dubt_find_better_dict_match( + ms, + src, + ip, + iend, + off_base_ptr, + best_length, + nb_compares, + mls, + )?; + } + + // Skip repetitive patterns. + ms.next_to_update = match_end_idx - 8; + Ok(best_length) +} + +/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. +/// `off_base_ptr` is written to if there is one. +#[inline(never)] +fn bt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + off_base_ptr: &mut u32, +) -> Result { + if ip < src.pos_of(ms.next_to_update) { + // In an area that is skipped. + return Ok(0); + } + update_dubt(ms, src.prefix(), ip, MLS)?; + dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) +} + +// ── hash chains ─────────────────────────────────────────────────────────── + +/// `ZSTD_insertAndFindFirstIndex_internal` +#[inline(always)] +fn insert_and_find_first_index_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + lazy_skipping: bool, +) -> Result { + let hash_log = ms.c_params.hash_log; + let chain_mask = (1u32 << ms.c_params.chain_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let chain_table = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); + chain_table.put((idx & chain_mask) as usize, next_in_chain)?; + // While skipping, not every position is inserted. + if lazy_skipping { + break; + } + } + + ms.next_to_update = target; + hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) +} + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) +} + +/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn hc_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let chain_size = 1u32 << ms.c_params.chain_log; + let chain_mask = chain_size - 1; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let min_chain = curr.saturating_sub(chain_size); + let mut nb_attempts = 1u32 << ms.c_params.search_log; + let mut ml = 4 - 1; + + let mut match_index = + insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; + let chain_table = ms.chain_table.as_slice(); + + while match_index >= low_limit && nb_attempts > 0 { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + + if match_index <= min_chain { + break; + } + match_index = chain_table.at((match_index & chain_mask) as usize)?; + nb_attempts -= 1; + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_chain_table = dms.ms.chain_table.as_slice(); + let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; + let dms_chain_mask = dms_chain_size - 1; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_size = dms_end_index(&dms); + let dms_index_delta = dict_limit.wrapping_sub(dms_size); + let dms_min_chain = dms_size.saturating_sub(dms_chain_size); + + let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; + match_index = dms.ms.hash_table.at(h)?; + + while match_index >= dms_lowest_index && nb_attempts > 0 { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + + if match_index <= dms_min_chain { + break; + } + match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; + nb_attempts -= 1; + } + } + + Ok(ml) +} + +// ── row-based match finder ──────────────────────────────────────────────── +// +// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits +// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag +// table is the "head": which entry was written last. + +pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; +const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; +const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; + +/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. +#[inline(always)] +fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { + let mut next = u32::from(*head).wrapping_sub(1) & row_mask; + if next == 0 { + next = row_mask; + } + *head = next as u8; + next +} + +/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. +#[inline(always)] +fn row_fill_hash_cache( + ms: &mut MatchState, + prefix: Prefix<'_>, + mls: u32, + idx: u32, + i_limit: usize, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); + let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; + + for idx in idx..lim { + let hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + ms.hash_salt, + )? as u32; + let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; + ms.hash_cache.put(cached, hash)?; + } + Ok(()) +} + +/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes +/// further takes its place. +#[inline(always)] +fn row_next_cached_hash( + cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], + prefix: Prefix<'_>, + idx: u32, + hash_log: u32, + mls: u32, + hash_salt: u64, +) -> Result { + let new_hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32; + let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; + Ok(core::mem::replace(cached, new_hash)) +} + +/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. +#[inline(always)] +fn row_update_internal_impl( + ms: &mut MatchState, + prefix: Prefix<'_>, + update_start_idx: u32, + update_end_idx: u32, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let hash_salt = ms.hash_salt; + let hash_table = ms.hash_table.as_mut_slice(); + let tag_table = ms.tag_table.as_mut_slice(); + + for idx in update_start_idx..update_end_idx { + let hash = if use_cache { + row_next_cached_hash(&mut ms.hash_cache, prefix, idx, hash_log, mls, hash_salt)? + } else { + hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32 + }; + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; + tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; + hash_table.put(rel_row + pos, idx)?; + } + Ok(()) +} + +/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is +/// when compressing, not all of them if they are many. +#[inline(always)] +fn row_update_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + const K_SKIP_THRESHOLD: u32 = 384; + const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; + const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; + + let mut idx = ms.next_to_update; + let target = prefix.index_of(ip); + + if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { + let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; + row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; + idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; + row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; + } + row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the +/// row-based match finder. +pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { + let row_log = ms.c_params.search_log.clamp(4, 6); + let row_mask = (1u32 << row_log) - 1; + let mls = ms.c_params.min_match.min(6); + row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) +} + +/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. +#[inline(always)] +fn row_get_match_mask( + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + head: u32, +) -> u64 { + let mut matches = 0u64; + let (chunks, _) = tag_row.as_chunks::<16>(); + for (n, chunk) in chunks.iter().enumerate() { + let mut chunk_matches = 0u16; + for (i, &entry) in chunk.iter().enumerate() { + chunk_matches |= u16::from(entry == tag) << i; + } + matches |= u64::from(chunk_matches) << (16 * n); + } + match ROW_ENTRIES { + 16 => u64::from((matches as u16).rotate_right(head)), + 32 => u64::from((matches as u32).rotate_right(head)), + _ => matches.rotate_right(head), + } +} + +/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, +/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than +/// `low_limit`. Returns how many there are. +#[inline(always)] +fn row_collect_matches( + row: &[u32; ROW_ENTRIES], + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + low_limit: u32, + nb_attempts: &mut u32, + match_buffer: &mut [u32; ROW_ENTRIES], +) -> Result { + let row_mask = ROW_ENTRIES as u32 - 1; + let head = u32::from(tag_row.at(0)?) & row_mask; + let mut num_matches = 0usize; + let mut matches = row_get_match_mask(tag_row, tag, head); + + while matches > 0 && *nb_attempts > 0 { + let match_pos = (head + matches.trailing_zeros()) & row_mask; + matches &= matches - 1; + if match_pos == 0 { + // The head, which happens to be the same as the tag. + continue; + } + let match_index = row.at(match_pos as usize)?; + if match_index < low_limit { + break; + } + match_buffer.put(num_matches, match_index)?; + num_matches += 1; + *nb_attempts -= 1; + } + Ok(num_matches) +} + +/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn row_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let hash_log = ms.row_hash_log; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let row_log = ROW_ENTRIES.trailing_zeros(); + let row_mask = ROW_ENTRIES as u32 - 1; + // There are no more searches than entries in a row. + let capped_search_log = ms.c_params.search_log.min(row_log); + let hash_salt = ms.hash_salt; + let mut nb_attempts = 1u32 << capped_search_log; + let mut ml = 4 - 1; + let mut match_buffer = [0u32; ROW_ENTRIES]; + + // Update the tables up to, and not including, `ip`. + let hash = if ms.lazy_skipping { + // While skipping, not every position is inserted. + ms.next_to_update = curr; + hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 + } else { + let prefix = src.prefix(); + row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; + row_next_cached_hash(&mut ms.hash_cache, prefix, curr, hash_log, MLS, hash_salt)? + }; + ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); + + { + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; + let row = ms.hash_table.array_at_mut::(rel_row)?; + let tag_row = ms.tag_table.array_at_mut::(rel_row)?; + let num_matches = row_collect_matches( + row, + tag_row, + tag, + low_limit, + &mut nb_attempts, + &mut match_buffer, + )?; + + // Insert `ip`. + let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; + tag_row.put(pos, tag)?; + row.put(pos, ms.next_to_update)?; + ms.next_to_update += 1; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + } + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); + + let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + let dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; + let dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let num_matches = row_collect_matches( + dms.ms.hash_table.array_at::(dms_rel_row)?, + dms.ms.tag_table.array_at::(dms_rel_row)?, + (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, + dms_lowest_index, + &mut nb_attempts, + &mut match_buffer, + )?; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + } + } + + Ok(ml) +} + +// ── the parsers ─────────────────────────────────────────────────────────── + +// `searchMethod_e`, as an argument of a const generic. +const SEARCH_HASH_CHAIN: u8 = 0; +const SEARCH_BINARY_TREE: u8 = 1; +const SEARCH_ROW_HASH: u8 = 2; + +/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. +#[inline(always)] +fn search_max( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mls: u32, + row_log: u32, +) -> Result { + match SEARCH_METHOD { + SEARCH_HASH_CHAIN => match mls { + 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + SEARCH_BINARY_TREE => match mls { + 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + _ => match (mls, row_log) { + (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + } +} + +/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match +/// state is searched. +#[derive(Clone, Copy)] +struct DictSegment<'a> { + buf: &'a [u8], + /// Its sum with an index of the window is a position in `buf`. It wraps around. + base: usize, + /// Where the segment starts in `buf`. + start: usize, + /// Where it ends. + end: usize, +} + +/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. +#[inline(always)] +fn rep_length( + ms: &MatchState, + src: &BlockSrc<'_>, + dict: &DictSegment<'_>, + ip: usize, + offset: u32, +) -> Result { + let buf = src.buf; + let iend = src.end; + if DICT_MODE == NO_DICT { + let rep_match = ip.wrapping_sub(offset as usize); + if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { + return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); + } + return Ok(0); + } + + let dict_limit = ms.window.dict_limit; + let curr = src.index_of(ip); + let rep_index = curr.wrapping_sub(offset); + if !index_overlap_check(dict_limit, rep_index) { + return Ok(0); + } + if DICT_MODE == EXT_DICT { + // The offset may lead to what is no longer in the window. + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + if offset > curr.wrapping_sub(window_low) { + return Ok(0); + } + } + let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { + let rep_match = dict.base.wrapping_add(rep_index as usize); + (dict.buf, rep_match, dict.end) + } else { + (buf, src.pos_of(rep_index), iend) + }; + if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { + return Ok(0); + } + let prefix_start = src.pos_of(dict_limit); + Ok(count_2segments( + buf, + ip + 4, + rep_buf, + rep_match + 4, + iend, + rep_end, + prefix_start, + )? + 4) +} + +/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions +/// further a better match is looked for, once there is one. +#[inline(never)] +fn compress_block_lazy_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let prefix_lowest_index = ms.window.dict_limit; + let prefix_lowest = src.pos_of(prefix_lowest_index); + let mls = ms.c_params.min_match.clamp(4, 6); + let row_log = ms.c_params.search_log.clamp(4, 6); + + let [rep_0, rep_1, _] = rep; + let mut offset_1 = *rep_0; + let mut offset_2 = *rep_1; + let mut offset_saved_1 = 0u32; + let mut offset_saved_2 = 0u32; + + ms.lazy_skipping = false; + + let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { + 8 + ZSTD_ROW_HASH_CACHE_SIZE + } else { + 8 + }; + let Some(ilimit) = iend.checked_sub(margin) else { + // There is no position to search, nor to hash. + return Ok(iend - istart); + }; + + let dict = match DICT_MODE { + NO_DICT => DictSegment { + buf: &[], + base: 0, + start: 0, + end: 0, + }, + EXT_DICT => DictSegment { + buf: src.dict_buf, + base: src.dict_base, + start: src.dict_pos_of(ms.window.low_limit), + end: src.dict_pos_of(prefix_lowest_index), + }, + _ => { + let dms = src.dms.ok_or(BUG)?; + let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); + DictSegment { + buf: dms.buf, + base: dms.base.wrapping_sub(dict_index_delta as usize), + start: dms.pos_of(dms.ms.window.dict_limit), + end: dms.end, + } + } + }; + + if DICT_MODE == EXT_DICT { + ip += usize::from(ip == prefix_lowest); + } else { + let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); + ip += usize::from(dict_and_prefix_length == 0); + } + if DICT_MODE == NO_DICT { + let curr = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); + let max_rep = curr - window_low; + if offset_2 > max_rep { + offset_saved_2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved_1 = offset_1; + offset_1 = 0; + } + } + + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + + 'parse: while ip < ilimit { + let mut match_length; + let mut off_base = REPCODE1_TO_OFFBASE; + let mut start = ip + 1; + + 'found: { + // Check the repeat offset. + match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; + if DEPTH == 0 && match_length != 0 { + break 'found; + } + + // First search. + { + let mut off_base_found = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut off_base_found, + mls, + row_log, + )?; + if ml2 > match_length { + match_length = ml2; + start = ip; + off_base = off_base_found; + } + } + + if match_length < 4 { + // Jump faster over incompressible sections. + let step = (ip - anchor) >> K_SEARCH_STRENGTH; + ip += step + 1; + // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of + // no use out of the tables. As in the C code, it does not start at the same step in all modes. + ms.lazy_skipping = if DICT_MODE == EXT_DICT { + step > K_LAZY_SKIPPING_STEP + } else { + step + 1 > K_LAZY_SKIPPING_STEP + }; + continue 'parse; + } + + // Look for something better at the next positions. + if DEPTH >= 1 { + while ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 3) as i32; + let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + + // Nothing better: try one position further. + if DEPTH == 2 && ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 4) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + } + break; + } + } + + // Catch up: the match may start earlier. + if offbase_is_offset(off_base) { + let offset = offbase_to_offset(off_base); + let match_index = src.index_of(start).wrapping_sub(offset); + let (match_buf, mut match_pos, m_start) = + if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { + let match_pos = dict.base.wrapping_add(match_index as usize); + (dict.buf, match_pos, dict.start) + } else { + (buf, src.pos_of(match_index), prefix_lowest) + }; + while start > anchor + && match_pos > m_start + && buf.at(start - 1)? == match_buf.at(match_pos - 1)? + { + start -= 1; + match_pos -= 1; + match_length += 1; + } + offset_2 = offset_1; + offset_1 = offset; + } + } + + // Store the sequence. + seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; + ip = start + match_length; + anchor = ip; + + if ms.lazy_skipping { + // There was a match: every position is inserted again. + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + ms.lazy_skipping = false; + } + + // Check the second repeat offset right away. + while ip <= ilimit { + match_length = rep_length::(ms, src, &dict, ip, offset_2)?; + if match_length == 0 { + break; + } + (offset_1, offset_2) = (offset_2, offset_1); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; + ip += match_length; + anchor = ip; + } + } + + // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. + if offset_saved_1 != 0 && offset_1 != 0 { + offset_saved_2 = offset_saved_1; + } + + // Save the repeat offsets for the next block. + *rep_0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved_1 + }; + *rep_1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved_2 + }; + + // The size of the last literals. + Ok(iend - anchor) +} + +/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. +#[inline(always)] +fn compress_block_lazy_any_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + match dict_mode { + DictMode::NoDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::ExtDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + compress_block_lazy_generic::( + ms, seq_store, rep, src, + ) + } + } +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row: bool, + dict_mode: DictMode, +) -> Result { + match (strategy, use_row) { + (Strategy::Greedy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Greedy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( + ms, seq_store, rep, src, dict_mode, + ), + _ => Err(BUG), + } +} diff --git a/src/zstd/wip/work/lazy/lazy.v2.rs b/src/zstd/wip/work/lazy/lazy.v2.rs new file mode 100644 index 000000000000..c19d790c6159 --- /dev/null +++ b/src/zstd/wip/work/lazy/lazy.v2.rs @@ -0,0 +1,1419 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use core::hint::{Locality, prefetch_read}; + +use super::internal::{ + BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, + SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, + hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, + offset_to_offbase, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +const K_LAZY_SKIPPING_STEP: usize = 8; + +// `ZSTD_dictMode_e`, as an argument of a const generic. +const NO_DICT: u8 = DictMode::NoDict as u8; +const EXT_DICT: u8 = DictMode::ExtDict as u8; +const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; + +/// `PREFETCH_L1`: a hint that `slice[index]` is about to be read. `index` may be anything. +#[inline(always)] +fn prefetch_l1(slice: &[T], index: usize) { + prefetch_read(slice.as_ptr().wrapping_add(index), Locality::L1); +} + +/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. +#[inline(always)] +const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { + dms.end.wrapping_sub(dms.base) as u32 +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. +#[inline(always)] +fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { + let hash_log = ms.c_params.hash_log; + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let bt = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); + *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. +fn insert_dubt1( + ms: &mut MatchState, + src: &BlockSrc<'_>, + curr: u32, + input_end: usize, + mut nb_compares: u32, + bt_low: u32, +) -> Result<()> { + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let (ip_buf, ip, iend) = if curr >= dict_limit { + (src.buf, src.pos_of(curr), input_end) + } else { + (src.dict_buf, src.dict_pos_of(curr), dict_end) + }; + let window_valid = ms.window.low_limit; + let max_distance = 1u32 << ms.c_params.window_log; + let window_low = if curr.wrapping_sub(window_valid) > max_distance { + curr - max_distance + } else { + window_valid + }; + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + // The next sorted candidate. The other slot has the previous unsorted one. + let mut match_index = bt.at(smaller_ptr)?; + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let in_prefix = + DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; + // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = if in_prefix { + (src.buf, src.pos_of(match_index)) + } else { + (src.dict_buf, src.dict_pos_of(match_index)) + }; + + if in_prefix || curr < dict_limit { + // Both are in the same segment. + match_length += count( + ip_buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + ip_buf, + ip + match_length, + match_buf, + match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if match_index as usize + match_length >= dict_limit as usize { + (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); + } + } + + if ip + match_length == iend { + // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0) +} + +/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. +fn dubt_find_better_dict_match( + ms: &MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mut best_length: usize, + mut nb_compares: u32, + mls: u32, +) -> Result { + let dms = src.dms.ok_or(BUG)?; + let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(h)?; + + let prefix_start = src.pos_of(ms.window.dict_limit); + let curr = src.index_of(ip); + let dict_high_limit = dms_end_index(&dms); + let dict_low_limit = dms.ms.window.low_limit; + let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); + + let dict_bt = dms.ms.chain_table.as_slice(); + let bt_log = dms.ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { + dict_low_limit + } else { + dict_high_limit - bt_mask + }; + + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + + while nb_compares != 0 && dict_match_index > dict_low_limit { + let next_ptr = 2 * (dict_match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let match_index = dict_match_index.wrapping_add(dict_index_delta); + match_length += count_2segments( + src.buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + iend, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(offset_ptr.wrapping_add(1)) as i32 + { + best_length = match_length; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // No way to know which is smaller. + break; + } + } + + if dict_match_index <= bt_low { + // Beyond the tree. + break; + } + let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { + let match_pos = src.pos_of(match_index); + src.buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < src.buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dict_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dict_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + Ok(best_length) +} + +/// `ZSTD_DUBT_findBestMatch` +#[inline(always)] +fn dubt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + off_base_ptr: &mut u32, + mls: u32, +) -> Result { + let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; + let mut match_index = ms.hash_table.at(h)?; + + let curr = src.index_of(ip); + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = curr.saturating_sub(bt_mask); + let unsort_limit = bt_low.max(window_low); + let mut nb_compares = 1u32 << ms.c_params.search_log; + let mut nb_candidates = nb_compares; + let mut previous_candidate = 0u32; + + // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. + let bt = ms.chain_table.as_mut_slice(); + let mut next_candidate = 2 * (match_index & bt_mask) as usize; + while match_index > unsort_limit + && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK + && nb_candidates > 1 + { + bt.put(next_candidate + 1, previous_candidate)?; + previous_candidate = match_index; + match_index = bt.at(next_candidate)?; + next_candidate = 2 * (match_index & bt_mask) as usize; + nb_candidates -= 1; + } + + // If the last candidate is still unsorted, cut the list there. + if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { + *bt.array_at_mut(next_candidate)? = [0, 0]; + } + + // Sort the candidates, the oldest first. + match_index = previous_candidate; + while match_index != 0 { + let next_candidate_idx = ms + .chain_table + .at(2 * (match_index & bt_mask) as usize + 1)?; + insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; + match_index = next_candidate_idx; + nb_candidates += 1; + } + + // Find the longest match. + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + let mut match_end_idx = curr + 8 + 1; + let mut best_length = 0usize; + + match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); + + if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + src.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + let dict_match_pos = src.dict_pos_of(match_index); + match_length += count_2segments( + src.buf, + ip + match_length, + src.dict_buf, + dict_match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + (match_buf, match_pos) = (src.dict_buf, dict_match_pos); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index + match_length as u32; + } + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(*off_base_ptr) as i32 + { + best_length = match_length; + *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // Equal: no way to know which is smaller. + if DICT_MODE == DICT_MATCH_STATE { + // Nothing in the dictionary can be longer. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0)?; + + if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { + best_length = dubt_find_better_dict_match( + ms, + src, + ip, + iend, + off_base_ptr, + best_length, + nb_compares, + mls, + )?; + } + + // Skip repetitive patterns. + ms.next_to_update = match_end_idx - 8; + Ok(best_length) +} + +/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. +/// `off_base_ptr` is written to if there is one. +#[inline(never)] +fn bt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + off_base_ptr: &mut u32, +) -> Result { + if ip < src.pos_of(ms.next_to_update) { + // In an area that is skipped. + return Ok(0); + } + update_dubt(ms, src.prefix(), ip, MLS)?; + dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) +} + +// ── hash chains ─────────────────────────────────────────────────────────── + +/// `ZSTD_insertAndFindFirstIndex_internal` +#[inline(always)] +fn insert_and_find_first_index_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + lazy_skipping: bool, +) -> Result { + let hash_log = ms.c_params.hash_log; + let chain_mask = (1u32 << ms.c_params.chain_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let chain_table = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); + chain_table.put((idx & chain_mask) as usize, next_in_chain)?; + // While skipping, not every position is inserted. + if lazy_skipping { + break; + } + } + + ms.next_to_update = target; + hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) +} + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) +} + +/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn hc_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let chain_size = 1u32 << ms.c_params.chain_log; + let chain_mask = chain_size - 1; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let min_chain = curr.saturating_sub(chain_size); + let mut nb_attempts = 1u32 << ms.c_params.search_log; + let mut ml = 4 - 1; + + let mut match_index = + insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; + let chain_table = ms.chain_table.as_slice(); + + while match_index >= low_limit && nb_attempts > 0 { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + + if match_index <= min_chain { + break; + } + match_index = chain_table.at((match_index & chain_mask) as usize)?; + nb_attempts -= 1; + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_chain_table = dms.ms.chain_table.as_slice(); + let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; + let dms_chain_mask = dms_chain_size - 1; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_size = dms_end_index(&dms); + let dms_index_delta = dict_limit.wrapping_sub(dms_size); + let dms_min_chain = dms_size.saturating_sub(dms_chain_size); + + let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; + match_index = dms.ms.hash_table.at(h)?; + + while match_index >= dms_lowest_index && nb_attempts > 0 { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + + if match_index <= dms_min_chain { + break; + } + match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; + nb_attempts -= 1; + } + } + + Ok(ml) +} + +// ── row-based match finder ──────────────────────────────────────────────── +// +// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits +// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag +// table is the "head": which entry was written last. + +pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; +const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; +const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; + +/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. +#[inline(always)] +fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { + let mut next = u32::from(*head).wrapping_sub(1) & row_mask; + if next == 0 { + next = row_mask; + } + *head = next as u8; + next +} + +/// `ZSTD_row_prefetch`: of the row that starts at `rel_row`. +#[inline(always)] +fn row_prefetch(hash_table: &[u32], tag_table: &[u8], rel_row: usize, row_log: u32) { + prefetch_l1(hash_table, rel_row); + if row_log >= 5 { + prefetch_l1(hash_table, rel_row + 16); + } + prefetch_l1(tag_table, rel_row); + if row_log == 6 { + prefetch_l1(tag_table, rel_row + 32); + } +} + +/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. +#[inline(always)] +fn row_fill_hash_cache( + ms: &mut MatchState, + prefix: Prefix<'_>, + row_log: u32, + mls: u32, + idx: u32, + i_limit: usize, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); + let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; + + for idx in idx..lim { + let hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + ms.hash_salt, + )? as u32; + let row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + row_prefetch(&ms.hash_table, &ms.tag_table, row, row_log); + let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; + ms.hash_cache.put(cached, hash)?; + } + Ok(()) +} + +/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes +/// further takes its place. +#[inline(always)] +fn row_next_cached_hash( + cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], + hash_table: &[u32], + tag_table: &[u8], + prefix: Prefix<'_>, + idx: u32, + hash_log: u32, + row_log: u32, + mls: u32, + hash_salt: u64, +) -> Result { + let new_hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32; + let row = ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + row_prefetch(hash_table, tag_table, row, row_log); + let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; + Ok(core::mem::replace(cached, new_hash)) +} + +/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. +#[inline(always)] +fn row_update_internal_impl( + ms: &mut MatchState, + prefix: Prefix<'_>, + update_start_idx: u32, + update_end_idx: u32, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let hash_salt = ms.hash_salt; + let hash_table = ms.hash_table.as_mut_slice(); + let tag_table = ms.tag_table.as_mut_slice(); + + for idx in update_start_idx..update_end_idx { + let hash = if use_cache { + row_next_cached_hash( + &mut ms.hash_cache, + hash_table, + tag_table, + prefix, + idx, + hash_log, + row_log, + mls, + hash_salt, + )? + } else { + hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32 + }; + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; + tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; + hash_table.put(rel_row + pos, idx)?; + } + Ok(()) +} + +/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is +/// when compressing, not all of them if they are many. +#[inline(always)] +fn row_update_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + const K_SKIP_THRESHOLD: u32 = 384; + const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; + const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; + + let mut idx = ms.next_to_update; + let target = prefix.index_of(ip); + + if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { + let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; + row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; + idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; + row_fill_hash_cache(ms, prefix, row_log, mls, idx, ip + 1)?; + } + row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the +/// row-based match finder. +pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { + let row_log = ms.c_params.search_log.clamp(4, 6); + let row_mask = (1u32 << row_log) - 1; + let mls = ms.c_params.min_match.min(6); + row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) +} + +/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. +#[inline(always)] +fn row_get_match_mask( + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + head: u32, +) -> u64 { + let mut matches = 0u64; + let (chunks, _) = tag_row.as_chunks::<16>(); + for (n, chunk) in chunks.iter().enumerate() { + let mut chunk_matches = 0u16; + for (i, &entry) in chunk.iter().enumerate() { + chunk_matches |= u16::from(entry == tag) << i; + } + matches |= u64::from(chunk_matches) << (16 * n); + } + match ROW_ENTRIES { + 16 => u64::from((matches as u16).rotate_right(head)), + 32 => u64::from((matches as u32).rotate_right(head)), + _ => matches.rotate_right(head), + } +} + +/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, +/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than +/// `low_limit`. `prefetch` is called with each. Returns how many there are. +#[inline(always)] +fn row_collect_matches( + row: &[u32; ROW_ENTRIES], + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + low_limit: u32, + nb_attempts: &mut u32, + match_buffer: &mut [u32; ROW_ENTRIES], + prefetch: impl Fn(u32), +) -> Result { + let row_mask = ROW_ENTRIES as u32 - 1; + let head = u32::from(tag_row.at(0)?) & row_mask; + let mut num_matches = 0usize; + let mut matches = row_get_match_mask(tag_row, tag, head); + + while matches > 0 && *nb_attempts > 0 { + let match_pos = (head + matches.trailing_zeros()) & row_mask; + matches &= matches - 1; + if match_pos == 0 { + // The head, which happens to be the same as the tag. + continue; + } + let match_index = row.at(match_pos as usize)?; + if match_index < low_limit { + break; + } + prefetch(match_index); + match_buffer.put(num_matches, match_index)?; + num_matches += 1; + *nb_attempts -= 1; + } + Ok(num_matches) +} + +/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn row_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let hash_log = ms.row_hash_log; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let row_log = ROW_ENTRIES.trailing_zeros(); + let row_mask = ROW_ENTRIES as u32 - 1; + // There are no more searches than entries in a row. + let capped_search_log = ms.c_params.search_log.min(row_log); + let hash_salt = ms.hash_salt; + let mut nb_attempts = 1u32 << capped_search_log; + let mut ml = 4 - 1; + let mut match_buffer = [0u32; ROW_ENTRIES]; + + let mut dms_hash = 0u32; + let mut dms_rel_row = 0usize; + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; + dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + row_prefetch(&dms.ms.hash_table, &dms.ms.tag_table, dms_rel_row, row_log); + } + + // Update the tables up to, and not including, `ip`. + let hash = if ms.lazy_skipping { + // While skipping, not every position is inserted. + ms.next_to_update = curr; + hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 + } else { + let prefix = src.prefix(); + row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; + row_next_cached_hash( + &mut ms.hash_cache, + &ms.hash_table, + &ms.tag_table, + prefix, + curr, + hash_log, + row_log, + MLS, + hash_salt, + )? + }; + ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); + + { + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; + let row = ms.hash_table.array_at_mut::(rel_row)?; + let tag_row = ms.tag_table.array_at_mut::(rel_row)?; + let num_matches = row_collect_matches( + row, + tag_row, + tag, + low_limit, + &mut nb_attempts, + &mut match_buffer, + |match_index| { + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + prefetch_l1(buf, src.pos_of(match_index)); + } else { + prefetch_l1(src.dict_buf, src.dict_pos_of(match_index)); + } + }, + )?; + + // Insert `ip`. + let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; + tag_row.put(pos, tag)?; + row.put(pos, ms.next_to_update)?; + ms.next_to_update += 1; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + } + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); + + let num_matches = row_collect_matches( + dms.ms.hash_table.array_at::(dms_rel_row)?, + dms.ms.tag_table.array_at::(dms_rel_row)?, + (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, + dms_lowest_index, + &mut nb_attempts, + &mut match_buffer, + |match_index| prefetch_l1(dms.buf, dms.pos_of(match_index)), + )?; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + } + } + + Ok(ml) +} + +// ── the parsers ─────────────────────────────────────────────────────────── + +// `searchMethod_e`, as an argument of a const generic. +const SEARCH_HASH_CHAIN: u8 = 0; +const SEARCH_BINARY_TREE: u8 = 1; +const SEARCH_ROW_HASH: u8 = 2; + +/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. +#[inline(always)] +fn search_max( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mls: u32, + row_log: u32, +) -> Result { + match SEARCH_METHOD { + SEARCH_HASH_CHAIN => match mls { + 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + SEARCH_BINARY_TREE => match mls { + 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + _ => match (mls, row_log) { + (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + } +} + +/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match +/// state is searched. +#[derive(Clone, Copy)] +struct DictSegment<'a> { + buf: &'a [u8], + /// Its sum with an index of the window is a position in `buf`. It wraps around. + base: usize, + /// Where the segment starts in `buf`. + start: usize, + /// Where it ends. + end: usize, +} + +/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. +#[inline(always)] +fn rep_length( + ms: &MatchState, + src: &BlockSrc<'_>, + dict: &DictSegment<'_>, + ip: usize, + offset: u32, +) -> Result { + let buf = src.buf; + let iend = src.end; + if DICT_MODE == NO_DICT { + let rep_match = ip.wrapping_sub(offset as usize); + if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { + return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); + } + return Ok(0); + } + + let dict_limit = ms.window.dict_limit; + let curr = src.index_of(ip); + let rep_index = curr.wrapping_sub(offset); + if !index_overlap_check(dict_limit, rep_index) { + return Ok(0); + } + if DICT_MODE == EXT_DICT { + // The offset may lead to what is no longer in the window. + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + if offset > curr.wrapping_sub(window_low) { + return Ok(0); + } + } + let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { + let rep_match = dict.base.wrapping_add(rep_index as usize); + (dict.buf, rep_match, dict.end) + } else { + (buf, src.pos_of(rep_index), iend) + }; + if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { + return Ok(0); + } + let prefix_start = src.pos_of(dict_limit); + Ok(count_2segments( + buf, + ip + 4, + rep_buf, + rep_match + 4, + iend, + rep_end, + prefix_start, + )? + 4) +} + +/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions +/// further a better match is looked for, once there is one. +#[inline(never)] +fn compress_block_lazy_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let prefix_lowest_index = ms.window.dict_limit; + let prefix_lowest = src.pos_of(prefix_lowest_index); + let mls = ms.c_params.min_match.clamp(4, 6); + let row_log = ms.c_params.search_log.clamp(4, 6); + + let [rep_0, rep_1, _] = rep; + let mut offset_1 = *rep_0; + let mut offset_2 = *rep_1; + let mut offset_saved_1 = 0u32; + let mut offset_saved_2 = 0u32; + + ms.lazy_skipping = false; + + let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { + 8 + ZSTD_ROW_HASH_CACHE_SIZE + } else { + 8 + }; + let Some(ilimit) = iend.checked_sub(margin) else { + // There is no position to search, nor to hash. + return Ok(iend - istart); + }; + + let dict = match DICT_MODE { + NO_DICT => DictSegment { + buf: &[], + base: 0, + start: 0, + end: 0, + }, + EXT_DICT => DictSegment { + buf: src.dict_buf, + base: src.dict_base, + start: src.dict_pos_of(ms.window.low_limit), + end: src.dict_pos_of(prefix_lowest_index), + }, + _ => { + let dms = src.dms.ok_or(BUG)?; + let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); + DictSegment { + buf: dms.buf, + base: dms.base.wrapping_sub(dict_index_delta as usize), + start: dms.pos_of(dms.ms.window.dict_limit), + end: dms.end, + } + } + }; + + if DICT_MODE == EXT_DICT { + ip += usize::from(ip == prefix_lowest); + } else { + let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); + ip += usize::from(dict_and_prefix_length == 0); + } + if DICT_MODE == NO_DICT { + let curr = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); + let max_rep = curr - window_low; + if offset_2 > max_rep { + offset_saved_2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved_1 = offset_1; + offset_1 = 0; + } + } + + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), row_log, mls, ms.next_to_update, ilimit)?; + } + + 'parse: while ip < ilimit { + let mut match_length; + let mut off_base = REPCODE1_TO_OFFBASE; + let mut start = ip + 1; + + 'found: { + // Check the repeat offset. + match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; + if DEPTH == 0 && match_length != 0 { + break 'found; + } + + // First search. + { + let mut off_base_found = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut off_base_found, + mls, + row_log, + )?; + if ml2 > match_length { + match_length = ml2; + start = ip; + off_base = off_base_found; + } + } + + if match_length < 4 { + // Jump faster over incompressible sections. + let step = (ip - anchor) >> K_SEARCH_STRENGTH; + ip += step + 1; + // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of + // no use out of the tables. As in the C code, it does not start at the same step in all modes. + ms.lazy_skipping = if DICT_MODE == EXT_DICT { + step > K_LAZY_SKIPPING_STEP + } else { + step + 1 > K_LAZY_SKIPPING_STEP + }; + continue 'parse; + } + + // Look for something better at the next positions. + if DEPTH >= 1 { + while ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 3) as i32; + let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + + // Nothing better: try one position further. + if DEPTH == 2 && ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 4) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + } + break; + } + } + + // Catch up: the match may start earlier. + if offbase_is_offset(off_base) { + let offset = offbase_to_offset(off_base); + let match_index = src.index_of(start).wrapping_sub(offset); + let (match_buf, mut match_pos, m_start) = + if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { + let match_pos = dict.base.wrapping_add(match_index as usize); + (dict.buf, match_pos, dict.start) + } else { + (buf, src.pos_of(match_index), prefix_lowest) + }; + while start > anchor + && match_pos > m_start + && buf.at(start - 1)? == match_buf.at(match_pos - 1)? + { + start -= 1; + match_pos -= 1; + match_length += 1; + } + offset_2 = offset_1; + offset_1 = offset; + } + } + + // Store the sequence. + seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; + ip = start + match_length; + anchor = ip; + + if ms.lazy_skipping { + // There was a match: every position is inserted again. + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), row_log, mls, ms.next_to_update, ilimit)?; + } + ms.lazy_skipping = false; + } + + // Check the second repeat offset right away. + while ip <= ilimit { + match_length = rep_length::(ms, src, &dict, ip, offset_2)?; + if match_length == 0 { + break; + } + (offset_1, offset_2) = (offset_2, offset_1); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; + ip += match_length; + anchor = ip; + } + } + + // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. + if offset_saved_1 != 0 && offset_1 != 0 { + offset_saved_2 = offset_saved_1; + } + + // Save the repeat offsets for the next block. + *rep_0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved_1 + }; + *rep_1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved_2 + }; + + // The size of the last literals. + Ok(iend - anchor) +} + +/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. +#[inline(always)] +fn compress_block_lazy_any_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + match dict_mode { + DictMode::NoDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::ExtDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + compress_block_lazy_generic::( + ms, seq_store, rep, src, + ) + } + } +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row: bool, + dict_mode: DictMode, +) -> Result { + match (strategy, use_row) { + (Strategy::Greedy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Greedy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( + ms, seq_store, rep, src, dict_mode, + ), + _ => Err(BUG), + } +} diff --git a/src/zstd/wip/work/lazy/lib.orig.rs b/src/zstd/wip/work/lazy/lib.orig.rs new file mode 100644 index 000000000000..31c98c72e5e4 --- /dev/null +++ b/src/zstd/wip/work/lazy/lib.orig.rs @@ -0,0 +1,385 @@ +//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the +//! same bytes in, in both directions, and the same errors. +#![forbid(unsafe_code)] +// Frames come from anywhere and a panic aborts the process. +#![deny( + clippy::indexing_slicing, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unreachable +)] +#![warn(unused_must_use)] + +mod common; +mod compress; +mod decompress; +mod error; +mod util; + +pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; +pub use decompress::{ + DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, + find_frame_compressed_size, get_frame_content_size, +}; +pub use error::ErrorCode; + +/// `ZSTD_inBuffer` +pub struct InBuffer<'a> { + pub src: &'a [u8], + /// How much of `src` is read. + pub pos: usize, +} + +/// `ZSTD_outBuffer` +pub struct OutBuffer<'a> { + pub dst: &'a mut [u8], + /// How much of `dst` is written. + pub pos: usize, +} + +/// `ZSTD_ResetDirective` +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ResetDirective { + SessionOnly = 1, + Parameters = 2, + SessionAndParameters = 3, +} + +// ----------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] +pub enum ZstdError { + /// The output, or the decoder state the frame's window size dictates, could not be allocated. + OutOfMemory, + InvalidZstdData, + DecompressionFailed, + ZstdDecompressionError, + ShortRead, +} + +impl core::fmt::Display for ZstdError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(<&'static str>::from(self)) + } +} +impl core::error::Error for ZstdError {} + +impl ZstdError { + /// The error for a failed decompression call; `other` is the non-allocation failure. + fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { + if code == ErrorCode::MemoryAllocation { + ZstdError::OutOfMemory + } else { + other + } + } +} + +/// `len` zeros for a decoder to write to. +fn output_buffer(len: usize) -> Result, ZstdError> { + util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) +} + +/// Decompress data, automatically allocating the output buffer. +/// Handles both frames with known and unknown content sizes. +/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. +/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. +pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { + let size = find_decompressed_size(src); + + const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit + + if size == ZSTD_CONTENTSIZE_ERROR { + return Err(ZstdError::InvalidZstdData); + } + + // Use streaming decompression if: + // 1. Content size is unknown, OR + // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) + if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { + let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { + // A frame's output is rarely smaller than its input. + src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) + } else { + // The header size is untrusted: reserve no more than the fast path below would. + MAX_PREALLOCATE_SIZE + }; + let mut list: Vec = Vec::new(); + list.try_reserve_exact(initial_capacity) + .map_err(|_| ZstdError::OutOfMemory)?; + StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; + return Ok(list); + } + + // Fast path: size is known and within reasonable limits + let mut output = output_buffer(size as usize)?; + let written = decompress(&mut output, src) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is +/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. +pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { + const MAX_SIZE: u64 = 16 * 1024 * 1024; + let size = find_decompressed_size(src); + // Also if it is unknown or invalid, which are the two largest values. + if size > MAX_SIZE { + return Err(ZstdError::InvalidZstdData); + } + let mut output = output_buffer(size as usize)?; + let written = DDict::prefix(prefix) + .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + Uninitialized, + Inflating, + End, + Error, +} + +/// Minimum spare output capacity offered to the decoder per call. +const STREAMING_OUTPUT_STEP: usize = 4096; + +// ────────────────────────────────────────────────────────────────────────── +// StreamingDecoder +// ────────────────────────────────────────────────────────────────────────── + +/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) +/// per call, so callers can hold the decoder across multiple body chunks. +pub struct StreamingDecoder { + stream: DCtx, + state: State, + /// The decoder filled its last window and may hold more. `max_output` can end a call there. + output_full: bool, +} + +impl Default for StreamingDecoder { + fn default() -> Self { + Self::new() + } +} + +impl StreamingDecoder { + pub fn new() -> Self { + Self { + stream: DCtx::new(), + state: State::Uninitialized, + output_full: false, + } + } + + #[inline] + pub fn is_inflating(&self) -> bool { + matches!(self.state, State::Inflating) + } + + /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is + /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. + /// Returns `ShortRead` when more input is required and `is_done` is false. + pub fn decompress( + &mut self, + input: &[u8], + out: &mut Vec, + max_output: usize, + is_done: bool, + ) -> Result { + if matches!(self.state, State::End | State::Error) { + return Ok(input.len()); + } + + let mut in_buf = InBuffer { src: input, pos: 0 }; + while matches!(self.state, State::Uninitialized | State::Inflating) { + let next_in_len = input.len() - in_buf.pos; + + // Call the decoder again with no input until it leaves the window short. + if next_in_len == 0 && !self.output_full { + if is_done { + if self.state == State::Inflating { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + self.state = State::End; + } + return Ok(in_buf.pos); + } + + if out.len() >= max_output { + return Ok(in_buf.pos); + } + + if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { + self.state = State::Error; + return Err(ZstdError::OutOfMemory); + } + // The decoder writes to initialized memory only. + let written_before = out.len(); + let window = (out.capacity() - written_before).min(max_output - written_before); + out.resize(written_before + window, 0); + let mut out_buf = OutBuffer { + dst: out.get_mut(written_before..).unwrap_or_default(), + pos: 0, + }; + let read_before = in_buf.pos; + + let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); + let bytes_written = out_buf.pos; + out.truncate(written_before + bytes_written); + let remaining = match result { + Ok(remaining) => remaining, + Err(code) => { + self.state = State::Error; + return Err(ZstdError::for_decompression( + code, + ZstdError::ZstdDecompressionError, + )); + } + }; + + self.output_full = bytes_written == window; + + if remaining == 0 { + // Frame complete, and fully flushed. + self.state = State::Uninitialized; + self.output_full = false; + if in_buf.pos >= input.len() { + if is_done { + self.state = State::End; + } + return Ok(in_buf.pos); + } + // More input available — reinitialize for the next frame. + self.stream.init_stream(); + continue; + } + + self.state = State::Inflating; + + if in_buf.pos - read_before == next_in_len { + if self.output_full { + continue; + } + if is_done { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + return Err(ZstdError::ShortRead); + } + } + Ok(in_buf.pos) + } +} + +// ── compressed embedded assets ──────────────────────────────────────────── + +/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes +/// Bun never executes or parses itself: the shell completion scripts and the +/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, +/// error overlay/page). Anything that runs inside Bun stays uncompressed. +/// +/// The exception is text that only `bun lint`, `bun fmt` and `bun check` +/// read. It may be compressed because it isn't on the path of running a program. +/// +/// Release (`bun_codegen_embed`) builds include the `.zst` twin that +/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to +/// `/compressed/.zst`; other builds evaluate `$fallback`. +#[macro_export] +macro_rules! embed_compressed { + // A codegen output (`/$sub`); debug builds read it at runtime. + (codegen $sub:literal) => { + $crate::embed_compressed!( + ("codegen/", $sub), + ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() + ) + }; + // Same, with a trailing NUL included in the slice (for C-string consumers). + (codegen_nul $sub:literal) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/codegen/", + $sub, + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); + &__COPY.get_or_init(|| { + let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); + let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); + copy.extend_from_slice(text); + copy.push(0); + copy.into_boxed_slice() + })[..] + } + }; + __bytes + }}; + // A file under `src/`; debug builds read it from the source tree at runtime. + (src $sub:literal) => { + $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) + }; + ($name:literal, $fallback:expr) => { + $crate::embed_compressed!(($name), $fallback) + }; + (($($name:literal),+), $fallback:expr) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/", + $($name,)+ + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + $fallback + } + }; + __bytes + }}; +} + +/// Cold, shared body of [`embed_compressed!`]'s release arm. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { + decompress_alloc(compressed) + .expect("embedded asset: invalid zstd frame") + .into_boxed_slice() +} + +/// [`inflate_embedded`] plus a trailing NUL byte. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { + let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); + inflated.reserve_exact(1); + inflated.push(0); + inflated.into_boxed_slice() +} +pub mod testing; diff --git a/src/zstd/wip/work/lazy/zstd/compress/double_fast.rs b/src/zstd/wip/work/lazy/zstd/compress/double_fast.rs new file mode 100644 index 000000000000..65d401675d1b --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/compress/double_fast.rs @@ -0,0 +1,48 @@ +//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_double_hash_table( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _end: usize, + _dtlm: DictTableLoadMethod, + _tfp: TableFillPurpose, +) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast` +pub(crate) fn compress_block_double_fast( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState` +pub(crate) fn compress_block_double_fast_dict_match_state( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast_extDict` +pub(crate) fn compress_block_double_fast_ext_dict( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/lazy/zstd/compress/fast.rs b/src/zstd/wip/work/lazy/zstd/compress/fast.rs new file mode 100644 index 000000000000..1f4788af28b4 --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/compress/fast.rs @@ -0,0 +1,48 @@ +//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_hash_table( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _end: usize, + _dtlm: DictTableLoadMethod, + _tfp: TableFillPurpose, +) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast` +pub(crate) fn compress_block_fast( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast_dictMatchState` +pub(crate) fn compress_block_fast_dict_match_state( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast_extDict` +pub(crate) fn compress_block_fast_ext_dict( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/lazy/zstd/compress/fse.rs b/src/zstd/wip/work/lazy/zstd/compress/fse.rs new file mode 100644 index 000000000000..1b8f7cbf3173 --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/compress/fse.rs @@ -0,0 +1,70 @@ +//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. + +use crate::common::{MAX_SEQ, ML_FSE_LOG}; +use crate::util::SliceExt; + +/// `FSE_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum FseRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `FSE_symbolCompressionTransform` +#[derive(Clone, Copy, Default)] +pub(crate) struct FseSymbolTransform { + pub(crate) delta_find_state: i32, + pub(crate) delta_nb_bits: u32, +} + +/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. +#[derive(Clone)] +pub(crate) struct FseCTable { + pub(crate) table_log: u32, + pub(crate) max_symbol_value: u32, + pub(crate) state_table: [u16; 1 << ML_FSE_LOG], + pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], +} + +impl FseCTable { + pub(crate) const fn new() -> Self { + Self { + table_log: 0, + max_symbol_value: 0, + state_table: [0; 1 << ML_FSE_LOG], + symbol_tt: [FseSymbolTransform { + delta_find_state: 0, + delta_nb_bits: 0, + }; MAX_SEQ as usize + 1], + } + } + + /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log + /// if the table has no code for it. + #[inline(always)] + pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 + } + + /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. + /// The table has a code for it. + #[inline(always)] + pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + let min_nb_bits = transform.delta_nb_bits >> 16; + let threshold = (min_nb_bits + 1) << 16; + let table_size = 1u32 << self.table_log; + let delta_from_threshold = + threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); + // A linear interpolation, which is very approximate. + let normalized_delta_from_threshold = + (delta_from_threshold << accuracy_log) >> self.table_log; + let bit_multiplier = 1u32 << accuracy_log; + ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) + } +} diff --git a/src/zstd/wip/work/lazy/zstd/compress/hist.rs b/src/zstd/wip/work/lazy/zstd/compress/hist.rs new file mode 100644 index 000000000000..becf42516867 --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/compress/hist.rs @@ -0,0 +1,33 @@ +//! `hist.c`: how many times each byte value occurs. + +use crate::error::Result; +use crate::util::SliceExt; + +/// `HIST_add`: `count` has room for every value in `src`. +pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { + for &byte in src { + *count.at_mut(usize::from(byte))? += 1; + } + Ok(()) +} + +/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. +/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. +pub(crate) fn count_simple( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + let mut max_symbol = *max_symbol_value as usize; + count.sub_mut(..=max_symbol)?.fill(0); + if src.is_empty() { + *max_symbol_value = 0; + return Ok(0); + } + add(count, src)?; + while count.at(max_symbol)? == 0 { + max_symbol -= 1; + } + *max_symbol_value = max_symbol as u32; + Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) +} diff --git a/src/zstd/wip/work/lazy/zstd/compress/huf.rs b/src/zstd/wip/work/lazy/zstd/compress/huf.rs new file mode 100644 index 000000000000..78373a89ebf0 --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/compress/huf.rs @@ -0,0 +1,38 @@ +//! `huf_compress.c`: Huffman coding of literals. + +use crate::common::HUF_SYMBOLVALUE_MAX; +use crate::util::SliceExt; + +/// `HUF_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum HufRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the +/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in +/// its highest bits. +#[derive(Clone)] +pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); + +impl HufCTable { + pub(crate) const fn new() -> Self { + Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) + } + + /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. + #[inline] + pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { + let [header, elements @ ..] = &self.0; + let max_symbol_value = (header >> 8) as u8; + if symbol_value > u32::from(max_symbol_value) { + return 0; + } + (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 + } +} diff --git a/src/zstd/wip/work/lazy/zstd/compress/internal.rs b/src/zstd/wip/work/lazy/zstd/compress/internal.rs new file mode 100644 index 000000000000..51157ea6c1ce --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/compress/internal.rs @@ -0,0 +1,1212 @@ +//! `zstd_compress_internal.h`: what the parts of the compressor share. + +use super::fse::{FseCTable, FseRepeat}; +use super::huf::{HufCTable, HufRepeat}; +use crate::common::{ + BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, + ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; + +pub(crate) const K_SEARCH_STRENGTH: u32 = 8; +pub(crate) const HASH_READ_SIZE: usize = 8; +/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. +pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; + +pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; +pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; + +pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; +pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; + +/// `ZSTD_strategy` +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub(crate) enum Strategy { + Fast = 1, + DFast = 2, + Greedy = 3, + Lazy = 4, + Lazy2 = 5, + BtLazy2 = 6, + BtOpt = 7, + BtUltra = 8, + BtUltra2 = 9, +} + +impl Strategy { + pub(crate) const fn from_int(value: i32) -> Option { + Some(match value { + 1 => Self::Fast, + 2 => Self::DFast, + 3 => Self::Greedy, + 4 => Self::Lazy, + 5 => Self::Lazy2, + 6 => Self::BtLazy2, + 7 => Self::BtOpt, + 8 => Self::BtUltra, + 9 => Self::BtUltra2, + _ => return None, + }) + } +} + +/// `ZSTD_ParamSwitch_e` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum ParamSwitch { + Auto = 0, + Enable = 1, + Disable = 2, +} + +/// `ZSTD_compressionParameters` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct CompressionParameters { + pub(crate) window_log: u32, + pub(crate) chain_log: u32, + pub(crate) hash_log: u32, + pub(crate) search_log: u32, + pub(crate) min_match: u32, + pub(crate) target_length: u32, + /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. + pub(crate) strategy: Option, +} + +impl CompressionParameters { + /// The strategy of parameters that are ready to be applied. + #[inline(always)] + pub(crate) fn strat(&self) -> Strategy { + self.strategy.unwrap_or(Strategy::Fast) + } +} + +/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum DictMode { + NoDict, + ExtDict, + DictMatchState, +} + +/// `ZSTD_dictTableLoadMethod_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DictTableLoadMethod { + Fast, + Full, +} + +/// `ZSTD_tableFillPurpose_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TableFillPurpose { + ForCCtx, + ForCDict, +} + +// ── entropy tables ──────────────────────────────────────────────────────── + +/// `ZSTD_hufCTables_t` +#[derive(Clone)] +pub(crate) struct HufCTables { + pub(crate) c_table: HufCTable, + pub(crate) repeat_mode: HufRepeat, +} + +/// `ZSTD_fseCTables_t` +#[derive(Clone)] +pub(crate) struct FseCTables { + pub(crate) offcode_c_table: FseCTable, + pub(crate) matchlength_c_table: FseCTable, + pub(crate) litlength_c_table: FseCTable, + pub(crate) offcode_repeat_mode: FseRepeat, + pub(crate) matchlength_repeat_mode: FseRepeat, + pub(crate) litlength_repeat_mode: FseRepeat, +} + +/// `ZSTD_entropyCTables_t` +#[derive(Clone)] +pub(crate) struct EntropyCTables { + pub(crate) huf: HufCTables, + pub(crate) fse: FseCTables, +} + +/// `ZSTD_compressedBlockState_t` +#[derive(Clone)] +pub(crate) struct CompressedBlockState { + pub(crate) entropy: EntropyCTables, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +impl CompressedBlockState { + /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. + pub(crate) fn new() -> Self { + Self { + entropy: EntropyCTables { + huf: HufCTables { + c_table: HufCTable::new(), + repeat_mode: HufRepeat::None, + }, + fse: FseCTables { + offcode_c_table: FseCTable::new(), + matchlength_c_table: FseCTable::new(), + litlength_c_table: FseCTable::new(), + offcode_repeat_mode: FseRepeat::None, + matchlength_repeat_mode: FseRepeat::None, + litlength_repeat_mode: FseRepeat::None, + }, + }, + rep: REP_START_VALUE, + } + } + + /// `ZSTD_reset_compressedBlockState` + pub(crate) fn reset(&mut self) { + self.rep = REP_START_VALUE; + self.entropy.huf.repeat_mode = HufRepeat::None; + self.entropy.fse.offcode_repeat_mode = FseRepeat::None; + self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; + self.entropy.fse.litlength_repeat_mode = FseRepeat::None; + } +} + +/// `ZSTD_hufCTablesMetadata_t` +pub(crate) struct HufCTablesMetadata { + pub(crate) h_type: SymbolEncodingType, + pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], + pub(crate) huf_des_size: usize, +} + +/// `ZSTD_fseCTablesMetadata_t` +pub(crate) struct FseCTablesMetadata { + pub(crate) ll_type: SymbolEncodingType, + pub(crate) of_type: SymbolEncodingType, + pub(crate) ml_type: SymbolEncodingType, + pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], + pub(crate) fse_tables_size: usize, + /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. + pub(crate) last_count_size: usize, +} + +/// `ZSTD_entropyCTablesMetadata_t` +pub(crate) struct EntropyCTablesMetadata { + pub(crate) huf_metadata: HufCTablesMetadata, + pub(crate) fse_metadata: FseCTablesMetadata, +} + +impl EntropyCTablesMetadata { + pub(crate) const fn new() -> Self { + Self { + huf_metadata: HufCTablesMetadata { + h_type: SymbolEncodingType::Basic, + huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], + huf_des_size: 0, + }, + fse_metadata: FseCTablesMetadata { + ll_type: SymbolEncodingType::Basic, + of_type: SymbolEncodingType::Basic, + ml_type: SymbolEncodingType::Basic, + fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], + fse_tables_size: 0, + last_count_size: 0, + }, + } + } +} + +// ── sequences ───────────────────────────────────────────────────────────── + +/// `SeqDef` +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct SeqDef { + /// A repeat code (1 to 3), or an offset plus 3. + pub(crate) off_base: u32, + pub(crate) lit_length: u16, + /// The match length minus [`MINMATCH`]. + pub(crate) ml_base: u16, +} + +/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum LongLengthType { + None, + LiteralLength, + MatchLength, +} + +/// `SeqStore_t`: the sequences and the literals of a block. +pub(crate) struct SeqStore { + /// Never grows beyond the capacity it is given by [`Self::reserve`]. + pub(crate) sequences: Vec, + /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. + pub(crate) lit_buffer: Vec, + /// How much of `lit_buffer` is literals. + pub(crate) lit_len: usize, + pub(crate) ll_code: Vec, + pub(crate) ml_code: Vec, + pub(crate) of_code: Vec, + pub(crate) max_nb_seq: usize, + pub(crate) max_nb_lit: usize, + pub(crate) long_length_type: LongLengthType, + /// Which sequence has the long length. + pub(crate) long_length_pos: u32, +} + +/// `ZSTD_SequenceLength` +#[derive(Clone, Copy)] +pub(crate) struct SequenceLength { + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +impl SeqStore { + pub(crate) const fn new() -> Self { + Self { + sequences: Vec::new(), + lit_buffer: Vec::new(), + lit_len: 0, + ll_code: Vec::new(), + ml_code: Vec::new(), + of_code: Vec::new(), + max_nb_seq: 0, + max_nb_lit: 0, + long_length_type: LongLengthType::None, + long_length_pos: 0, + } + } + + /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. + pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { + fn grow(vec: &mut Vec, len: usize) -> Result<()> { + vec.clear(); + vec.try_reserve_exact(len) + .map_err(|_| ErrorCode::MemoryAllocation)?; + vec.resize(len, T::default()); + Ok(()) + } + grow(&mut self.sequences, max_nb_seq)?; + grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; + grow(&mut self.ll_code, max_nb_seq)?; + grow(&mut self.ml_code, max_nb_seq)?; + grow(&mut self.of_code, max_nb_seq)?; + self.max_nb_seq = max_nb_seq; + self.max_nb_lit = max_nb_lit; + self.reset(); + Ok(()) + } + + /// `ZSTD_resetSeqStore` + #[inline] + pub(crate) fn reset(&mut self) { + self.lit_len = 0; + self.sequences.clear(); + self.long_length_type = LongLengthType::None; + } + + #[inline] + pub(crate) fn literals(&self) -> &[u8] { + self.lit_buffer.get(..self.lit_len).unwrap_or_default() + } + + /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. + #[inline(always)] + pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { + debug_assert!(self.sequences.len() < self.max_nb_seq); + if lit_length > 0xFFFF { + self.long_length_type = LongLengthType::LiteralLength; + self.long_length_pos = self.sequences.len() as u32; + } + let ml_base = match_length.wrapping_sub(MINMATCH as usize); + if ml_base > 0xFFFF { + self.long_length_type = LongLengthType::MatchLength; + self.long_length_pos = self.sequences.len() as u32; + } + self.sequences.push(SeqDef { + off_base, + lit_length: lit_length as u16, + ml_base: ml_base as u16, + }); + } + + /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. + #[inline(always)] + pub(crate) fn store_seq( + &mut self, + lit_length: usize, + buf: &[u8], + literals: usize, + off_base: u32, + match_length: usize, + ) -> Result<()> { + match ( + buf.array_at::<16>(literals), + self.lit_buffer.array_at_mut::<16>(self.lit_len), + ) { + (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, + _ => self + .lit_buffer + .write_at(self.lit_len, buf.span(literals, lit_length)?)?, + } + self.lit_len += lit_length; + self.store_seq_only(lit_length, off_base, match_length); + Ok(()) + } + + /// Appends the literals that follow the last sequence of a block. + #[inline] + pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { + self.lit_buffer.write_at(self.lit_len, literals)?; + self.lit_len += literals.len(); + Ok(()) + } +} + +pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; +pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; +pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; + +/// `OFFSET_TO_OFFBASE`: `offset` is not 0. +#[inline(always)] +pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { + offset.wrapping_add(ZSTD_REP_NUM as u32) +} + +/// `OFFBASE_IS_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { + off_base > ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_IS_REPCODE` +#[inline(always)] +pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { + 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_TO_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { + off_base.wrapping_sub(ZSTD_REP_NUM as u32) +} + +/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. +#[inline(always)] +pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { + let [rep0, rep1, rep2] = *rep; + if offbase_is_offset(off_base) { + *rep = [offbase_to_offset(off_base), rep0, rep1]; + } else { + match off_base - 1 + u32::from(ll0) { + 0 => {} + 1 => *rep = [rep1, rep0, rep2], + 2 => *rep = [rep2, rep0, rep1], + _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], + } + } +} + +/// `ZSTD_newRep` +#[inline(always)] +pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { + let mut new_reps = *rep; + update_rep(&mut new_reps, off_base, ll0); + new_reps +} + +/// `ZSTD_LLcode` +#[inline(always)] +pub(crate) fn ll_code(lit_length: u32) -> u32 { + #[rustfmt::skip] + const LL_CODE: [u8; 64] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + ]; + const LL_DELTA_CODE: u32 = 19; + match LL_CODE.get(lit_length as usize) { + Some(&code) => u32::from(code), + None => highbit32(lit_length) + LL_DELTA_CODE, + } +} + +/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. +#[inline(always)] +pub(crate) fn ml_code(ml_base: u32) -> u32 { + #[rustfmt::skip] + const ML_CODE: [u8; 128] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ]; + const ML_DELTA_CODE: u32 = 36; + match ML_CODE.get(ml_base as usize) { + Some(&code) => u32::from(code), + None => highbit32(ml_base) + ML_DELTA_CODE, + } +} + +/// `ZSTD_noCompressBlock` +pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { + let c_block_header24 = + u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); + if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header24)?; + dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; + Ok(ZSTD_BLOCKHEADERSIZE + src.len()) +} + +/// `ZSTD_rleCompressBlock` +pub(crate) fn rle_compress_block( + dst: &mut [u8], + byte: u8, + src_size: usize, + last_block: bool, +) -> Result { + let c_block_header = + u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); + if dst.len() < 4 { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header)?; + dst.put(3, byte)?; + Ok(4) +} + +/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. +#[inline] +pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { + let minlog = if strat >= Strategy::BtUltra { + strat as u32 - 1 + } else { + 6 + }; + (src_size >> minlog) + 2 +} + +// ── matching ────────────────────────────────────────────────────────────── + +/// How many bytes `a` and `b` have in common at their start. +#[inline(always)] +pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { + let (a_words, _) = a.as_chunks::<8>(); + let (b_words, _) = b.as_chunks::<8>(); + let mut len = 0usize; + for (a_word, b_word) in a_words.iter().zip(b_words) { + let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); + if diff != 0 { + return len + (diff.trailing_zeros() >> 3) as usize; + } + len += 8; + } + let a_tail = a.get(len..).unwrap_or_default(); + let b_tail = b.get(len..).unwrap_or_default(); + len + a_tail + .iter() + .zip(b_tail) + .take_while(|(a_byte, b_byte)| a_byte == b_byte) + .count() +} + +/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. +#[inline(always)] +pub(crate) fn count( + buf: &[u8], + p_in: usize, + match_buf: &[u8], + p_match: usize, + p_in_limit: usize, +) -> Result { + Ok(common_prefix( + buf.sub(p_in..p_in_limit)?, + match_buf.sub(p_match..)?, + )) +} + +/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at +/// `i_start` in `buf`. +#[inline(always)] +pub(crate) fn count_2segments( + buf: &[u8], + ip: usize, + match_buf: &[u8], + match_pos: usize, + i_end: usize, + m_end: usize, + i_start: usize, +) -> Result { + let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); + if match_pos + match_length != m_end { + return Ok(match_length); + } + Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) +} + +const PRIME_3_BYTES: u32 = 506_832_829; +const PRIME_4_BYTES: u32 = 2_654_435_761; +const PRIME_5_BYTES: u64 = 889_523_592_379; +const PRIME_6_BYTES: u64 = 227_718_039_650_203; +const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; +pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; + +/// `ZSTD_hash3` +#[inline(always)] +const fn hash3(u: u32, h: u32, s: u32) -> u32 { + (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash4(u: u32, h: u32, s: u32) -> u32 { + ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash5(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash6(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash7(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +/// `ZSTD_hash8` +#[inline(always)] +pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { + (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} + +/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. +#[inline(always)] +pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { + Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) +} + +/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, +/// else 8. +#[inline(always)] +pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { + hash_ptr_salted(buf, pos, h_bits, mls, 0) +} + +/// `ZSTD_hashPtrSalted` +#[inline(always)] +pub(crate) fn hash_ptr_salted( + buf: &[u8], + pos: usize, + h_bits: u32, + mls: u32, + hash_salt: u64, +) -> Result { + Ok(match mls { + 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), + 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), + 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), + 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), + _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, + }) +} + +// ── the window ──────────────────────────────────────────────────────────── + +/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are +/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the +/// others, and an address is turned into a position in the buffer only to read. +pub(crate) type Addr = usize; + +/// The buffers that a window can have a segment in. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum BufId { + /// Where an empty window points. + None = 0, + /// The context's own buffer of input. + InBuff = 1, + /// What the caller gave to the call in progress. + User = 2, + /// The content of the dictionary, or the prefix. + Dict = 3, +} + +impl BufId { + /// The address of the first byte of the buffer. + #[inline(always)] + pub(crate) const fn start(self) -> Addr { + (self as usize + 1) << 44 + } +} + +/// The buffers themselves, for as long as a call lasts. +#[derive(Clone, Copy)] +pub(crate) struct Buffers<'a> { + pub(crate) in_buff: &'a [u8], + pub(crate) user: &'a [u8], + pub(crate) dict: &'a [u8], +} + +impl<'a> Buffers<'a> { + #[inline] + pub(crate) fn get(&self, id: BufId) -> &'a [u8] { + match id { + BufId::None => &[], + BufId::InBuff => self.in_buff, + BufId::User => self.user, + BufId::Dict => self.dict, + } + } +} + +/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. +pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; + +/// The index past which indices are brought down. +pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; +/// The most input that can be indexed at once. +pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; + +/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the +/// index is less than `dict_limit`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Window { + /// Where input has to continue to be contiguous. + pub(crate) next_src: Addr, + pub(crate) base: Addr, + pub(crate) dict_base: Addr, + /// The buffer of the segment that starts at `dict_limit`. + pub(crate) base_buf: BufId, + /// The buffer of the segment from `low_limit` to `dict_limit`. + pub(crate) dict_buf: BufId, + pub(crate) dict_limit: u32, + /// Below this, there is nothing valid. + pub(crate) low_limit: u32, + pub(crate) nb_overflow_corrections: u32, +} + +impl Window { + /// `ZSTD_window_init` + pub(crate) const fn new() -> Self { + let base = BufId::None.start(); + Self { + next_src: base + ZSTD_WINDOW_START_INDEX as usize, + base, + dict_base: base, + base_buf: BufId::None, + dict_buf: BufId::None, + dict_limit: ZSTD_WINDOW_START_INDEX, + low_limit: ZSTD_WINDOW_START_INDEX, + nb_overflow_corrections: 0, + } + } + + /// The index of `addr`, which is in the current segment or at its end. + #[inline(always)] + pub(crate) const fn index_of(&self, addr: Addr) -> u32 { + addr.wrapping_sub(self.base) as u32 + } + + /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. + pub(crate) fn clear(&mut self) { + let end = self.index_of(self.next_src); + self.low_limit = end; + self.dict_limit = end; + } + + /// `ZSTD_window_isEmpty` + pub(crate) fn is_empty(&self) -> bool { + self.dict_limit == ZSTD_WINDOW_START_INDEX + && self.low_limit == ZSTD_WINDOW_START_INDEX + && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize + } + + /// `ZSTD_window_hasExtDict` + #[inline] + pub(crate) fn has_ext_dict(&self) -> bool { + self.low_limit < self.dict_limit + } + + /// `ZSTD_window_needOverflowCorrection` + #[inline] + pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { + self.index_of(src_end) > ZSTD_CURRENT_MAX + } + + /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns + /// what to subtract from every index that is stored. + pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { + let cycle_size = 1u32 << cycle_log; + let cycle_mask = cycle_size - 1; + let curr = self.index_of(src); + let current_cycle = curr & cycle_mask; + // Indices must not fall below the start index. + let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { + cycle_size.max(ZSTD_WINDOW_START_INDEX) + } else { + 0 + }; + let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); + let correction = curr.wrapping_sub(new_current); + + self.base = self.base.wrapping_add(correction as usize); + self.dict_base = self.dict_base.wrapping_add(correction as usize); + self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.low_limit - correction + }; + self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.dict_limit - correction + }; + self.nb_overflow_corrections += 1; + correction + } + + /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the + /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. + pub(crate) fn enforce_max_dist( + &mut self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { + let new_low_limit = block_end_idx - max_dist; + if self.low_limit < new_low_limit { + self.low_limit = new_low_limit; + } + if self.dict_limit < self.low_limit { + self.dict_limit = self.low_limit; + } + return true; + } + false + } + + /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not + /// right before the current segment, and is to be forgotten. + pub(crate) fn dict_is_invalid( + &self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit + } + + /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not + /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. + pub(crate) fn update( + &mut self, + src: Addr, + src_buf: BufId, + src_size: usize, + force_non_contiguous: bool, + ) -> bool { + let mut contiguous = true; + if src_size == 0 { + return contiguous; + } + if src != self.next_src || force_non_contiguous { + let distance_from_base = self.next_src.wrapping_sub(self.base); + self.low_limit = self.dict_limit; + self.dict_limit = distance_from_base as u32; + self.dict_base = self.base; + self.dict_buf = self.base_buf; + self.base = src.wrapping_sub(distance_from_base); + self.base_buf = src_buf; + if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { + // Too small to be of use. + self.low_limit = self.dict_limit; + } + contiguous = false; + } + self.next_src = src + src_size; + // Where the input overlaps the older segment, it has overwritten it. + if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) + & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) + { + let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); + self.low_limit = if high_input_idx > self.dict_limit as usize { + self.dict_limit + } else { + high_input_idx as u32 + }; + } + contiguous + } +} + +/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes +/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. +#[inline(always)] +pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { + prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 +} + +pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; +pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; + +/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. +#[inline(always)] +pub(crate) fn write_tagged_index( + hash_table: &mut [u32], + hash_and_tag: usize, + index: u32, +) -> Result<()> { + let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; + hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) +} + +/// `ZSTD_comparePackedTags` +#[inline(always)] +pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { + (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) + == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) +} + +// ── the state of the match finders ──────────────────────────────────────── + +/// `ZSTD_match_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Match { + /// As [`SeqDef::off_base`]. + pub(crate) off: u32, + pub(crate) len: u32, +} + +/// `rawSeq`: a sequence found by long distance matching. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct RawSeq { + pub(crate) offset: u32, + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +/// `RawSeqStore_t` +#[derive(Clone, Default)] +pub(crate) struct RawSeqStore { + /// `capacity` of them, of which `size` are in use. + pub(crate) seq: Vec, + /// The sequence that is being read. + pub(crate) pos: usize, + /// How many of its bytes, literals first, are read. + pub(crate) pos_in_sequence: usize, + pub(crate) size: usize, + pub(crate) capacity: usize, +} + +/// `ZSTD_optimal_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Optimal { + /// The cost of getting to this position. + pub(crate) price: i32, + pub(crate) off: u32, + pub(crate) mlen: u32, + pub(crate) litlen: u32, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +/// `ZSTD_OptPrice_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum OptPrice { + Dynamic, + Predef, +} + +/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. +pub(crate) struct OptState { + pub(crate) lit_freq: Vec, + pub(crate) lit_length_freq: Vec, + pub(crate) match_length_freq: Vec, + pub(crate) off_code_freq: Vec, + pub(crate) match_table: Vec, + pub(crate) price_table: Vec, + pub(crate) lit_sum: u32, + pub(crate) lit_length_sum: u32, + pub(crate) match_length_sum: u32, + pub(crate) off_code_sum: u32, + pub(crate) lit_sum_base_price: u32, + pub(crate) lit_length_sum_base_price: u32, + pub(crate) match_length_sum_base_price: u32, + pub(crate) off_code_sum_base_price: u32, + pub(crate) price_type: OptPrice, + pub(crate) literal_compression_mode: ParamSwitch, +} + +impl OptState { + pub(crate) const fn new() -> Self { + Self { + lit_freq: Vec::new(), + lit_length_freq: Vec::new(), + match_length_freq: Vec::new(), + off_code_freq: Vec::new(), + match_table: Vec::new(), + price_table: Vec::new(), + lit_sum: 0, + lit_length_sum: 0, + match_length_sum: 0, + off_code_sum: 0, + lit_sum_base_price: 0, + lit_length_sum_base_price: 0, + match_length_sum_base_price: 0, + off_code_sum_base_price: 0, + price_type: OptPrice::Dynamic, + literal_compression_mode: ParamSwitch::Auto, + } + } +} + +pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; + +/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures +/// own, are in [`BlockSrc`]. +pub(crate) struct MatchState { + pub(crate) window: Window, + /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. + pub(crate) loaded_dict_end: u32, + /// The index from which the tables are still to be filled. + pub(crate) next_to_update: u32, + /// 0 if there is no table of 3-byte hashes. + pub(crate) hash_log3: u32, + pub(crate) row_hash_log: u32, + /// For the row-based match finder: a byte of hash for each entry of `hash_table`. + pub(crate) tag_table: Vec, + pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], + /// Makes what a previous frame left in `tag_table` meaningless. + pub(crate) hash_salt: u64, + pub(crate) hash_salt_entropy: u32, + pub(crate) hash_table: Vec, + pub(crate) hash_table3: Vec, + pub(crate) chain_table: Vec, + pub(crate) force_non_contiguous: bool, + pub(crate) opt: OptState, + /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. + pub(crate) has_dict_match_state: bool, + pub(crate) c_params: CompressionParameters, + pub(crate) lazy_skipping: bool, +} + +impl MatchState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + loaded_dict_end: 0, + next_to_update: 0, + hash_log3: 0, + row_hash_log: 0, + tag_table: Vec::new(), + hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + hash_salt: 0, + hash_salt_entropy: 0, + hash_table: Vec::new(), + hash_table3: Vec::new(), + chain_table: Vec::new(), + force_non_contiguous: false, + opt: OptState::new(), + has_dict_match_state: false, + c_params: CompressionParameters { + window_log: 0, + chain_log: 0, + hash_log: 0, + search_log: 0, + min_match: 0, + target_length: 0, + strategy: None, + }, + lazy_skipping: false, + } + } + + /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. + pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { + if self.window.has_ext_dict() { + return DictMode::ExtDict; + } + if dms.is_some() { + DictMode::DictMatchState + } else { + DictMode::NoDict + } + } + + /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. + #[inline(always)] + pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.low_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + // A dictionary is valid as a whole until it is invalidated as a whole. + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } + + /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. + #[inline(always)] + pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.dict_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } +} + +/// The match state of a dictionary, and where its indices lead. +#[derive(Clone, Copy)] +pub(crate) struct DictMatchState<'a> { + pub(crate) ms: &'a MatchState, + /// The buffer its window is in, which is one segment. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// `window.nextSrc`, as a position in `buf`. + pub(crate) end: usize, +} + +/// The current segment of a window, for the functions that index a dictionary. +#[derive(Clone, Copy)] +pub(crate) struct Prefix<'a> { + /// The buffer that the segment is in. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, +} + +impl Prefix<'_> { + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. +#[derive(Clone, Copy)] +pub(crate) struct BlockSrc<'a> { + /// The buffer that the current segment of the window is in, and so the block. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// The buffer that the older segment of the window is in. + pub(crate) dict_buf: &'a [u8], + /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. + pub(crate) dict_base: usize, + /// Where the block starts in `buf`. + pub(crate) start: usize, + /// Where it ends. + pub(crate) end: usize, + pub(crate) dms: Option>, + /// `opt.symbolCosts`: the tables of the previous block. + pub(crate) symbol_costs: &'a EntropyCTables, + /// `ldmSeqStore`: matches that the optimal parser is to consider. + pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, +} + +impl<'a> BlockSrc<'a> { + #[inline(always)] + pub(crate) const fn prefix(&self) -> Prefix<'a> { + Prefix { + buf: self.buf, + base: self.base, + } + } + + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`, which is in the current segment. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } + + /// The position in `dict_buf` of `index`, which is in the older segment. + #[inline(always)] + pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { + self.dict_base.wrapping_add(index as usize) + } +} + +impl DictMatchState<'_> { + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +// ── long distance matching ──────────────────────────────────────────────── + +/// `ldmEntry_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct LdmEntry { + pub(crate) offset: u32, + pub(crate) checksum: u32, +} + +/// `ldmParams_t` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct LdmParams { + pub(crate) enable_ldm: ParamSwitch, + pub(crate) hash_log: u32, + pub(crate) bucket_size_log: u32, + pub(crate) min_match_length: u32, + pub(crate) hash_rate_log: u32, + pub(crate) window_log: u32, +} + +/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. +pub(crate) struct LdmState { + pub(crate) window: Window, + pub(crate) hash_table: Vec, + pub(crate) loaded_dict_end: u32, + /// The next entry to replace in each bucket. + pub(crate) bucket_offsets: Vec, +} + +impl LdmState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + hash_table: Vec::new(), + loaded_dict_end: 0, + bucket_offsets: Vec::new(), + } + } +} diff --git a/src/zstd/wip/work/lazy/zstd/compress/lazy.rs b/src/zstd/wip/work/lazy/zstd/compress/lazy.rs new file mode 100644 index 000000000000..6ca0df1fe9fa --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/compress/lazy.rs @@ -0,0 +1,1420 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use core::hint::{Locality, prefetch_read}; + +use super::internal::{ + BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, + SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, + hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, + offset_to_offbase, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +const K_LAZY_SKIPPING_STEP: usize = 8; + +// `ZSTD_dictMode_e`, as an argument of a const generic. +const NO_DICT: u8 = DictMode::NoDict as u8; +const EXT_DICT: u8 = DictMode::ExtDict as u8; +const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; + +/// `PREFETCH_L1`: a hint that `slice[index]` is about to be read. `index` may be anything. +#[inline(always)] +fn prefetch_l1(slice: &[T], index: usize) { + prefetch_read(slice.as_ptr().wrapping_add(index), Locality::L1); +} + +/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. +#[inline(always)] +const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { + dms.end.wrapping_sub(dms.base) as u32 +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. +#[inline(always)] +fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { + let hash_log = ms.c_params.hash_log; + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let bt = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); + *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. +fn insert_dubt1( + ms: &mut MatchState, + src: &BlockSrc<'_>, + curr: u32, + input_end: usize, + mut nb_compares: u32, + bt_low: u32, +) -> Result<()> { + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let (ip_buf, ip, iend) = if curr >= dict_limit { + (src.buf, src.pos_of(curr), input_end) + } else { + (src.dict_buf, src.dict_pos_of(curr), dict_end) + }; + let window_valid = ms.window.low_limit; + let max_distance = 1u32 << ms.c_params.window_log; + let window_low = if curr.wrapping_sub(window_valid) > max_distance { + curr - max_distance + } else { + window_valid + }; + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + // The next sorted candidate. The other slot has the previous unsorted one. + let mut match_index = bt.at(smaller_ptr)?; + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let in_prefix = + DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; + // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = if in_prefix { + (src.buf, src.pos_of(match_index)) + } else { + (src.dict_buf, src.dict_pos_of(match_index)) + }; + + if in_prefix || curr < dict_limit { + // Both are in the same segment. + match_length += count( + ip_buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + ip_buf, + ip + match_length, + match_buf, + match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if match_index as usize + match_length >= dict_limit as usize { + (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); + } + } + + if ip + match_length == iend { + // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0) +} + +/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. +fn dubt_find_better_dict_match( + ms: &MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mut best_length: usize, + mut nb_compares: u32, + mls: u32, +) -> Result { + let dms = src.dms.ok_or(BUG)?; + let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(h)?; + + let prefix_start = src.pos_of(ms.window.dict_limit); + let curr = src.index_of(ip); + let dict_high_limit = dms_end_index(&dms); + let dict_low_limit = dms.ms.window.low_limit; + let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); + + let dict_bt = dms.ms.chain_table.as_slice(); + let bt_log = dms.ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { + dict_low_limit + } else { + dict_high_limit - bt_mask + }; + + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + + while nb_compares != 0 && dict_match_index > dict_low_limit { + let next_ptr = 2 * (dict_match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let match_index = dict_match_index.wrapping_add(dict_index_delta); + match_length += count_2segments( + src.buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + iend, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(offset_ptr.wrapping_add(1)) as i32 + { + best_length = match_length; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // No way to know which is smaller. + break; + } + } + + if dict_match_index <= bt_low { + // Beyond the tree. + break; + } + let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { + let match_pos = src.pos_of(match_index); + src.buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < src.buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dict_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dict_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + Ok(best_length) +} + +/// `ZSTD_DUBT_findBestMatch` +#[inline(always)] +fn dubt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + off_base_ptr: &mut u32, + mls: u32, +) -> Result { + let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; + let mut match_index = ms.hash_table.at(h)?; + + let curr = src.index_of(ip); + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = curr.saturating_sub(bt_mask); + let unsort_limit = bt_low.max(window_low); + let mut nb_compares = 1u32 << ms.c_params.search_log; + let mut nb_candidates = nb_compares; + let mut previous_candidate = 0u32; + + // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. + let bt = ms.chain_table.as_mut_slice(); + let mut next_candidate = 2 * (match_index & bt_mask) as usize; + while match_index > unsort_limit + && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK + && nb_candidates > 1 + { + bt.put(next_candidate + 1, previous_candidate)?; + previous_candidate = match_index; + match_index = bt.at(next_candidate)?; + next_candidate = 2 * (match_index & bt_mask) as usize; + nb_candidates -= 1; + } + + // If the last candidate is still unsorted, cut the list there. + if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { + *bt.array_at_mut(next_candidate)? = [0, 0]; + } + + // Sort the candidates, the oldest first. + match_index = previous_candidate; + while match_index != 0 { + let next_candidate_idx = ms + .chain_table + .at(2 * (match_index & bt_mask) as usize + 1)?; + insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; + match_index = next_candidate_idx; + nb_candidates += 1; + } + + // Find the longest match. + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + let mut match_end_idx = curr + 8 + 1; + let mut best_length = 0usize; + + match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); + + if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + src.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + let dict_match_pos = src.dict_pos_of(match_index); + match_length += count_2segments( + src.buf, + ip + match_length, + src.dict_buf, + dict_match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + (match_buf, match_pos) = (src.dict_buf, dict_match_pos); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index + match_length as u32; + } + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(*off_base_ptr) as i32 + { + best_length = match_length; + *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // Equal: no way to know which is smaller. + if DICT_MODE == DICT_MATCH_STATE { + // Nothing in the dictionary can be longer. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0)?; + + if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { + best_length = dubt_find_better_dict_match( + ms, + src, + ip, + iend, + off_base_ptr, + best_length, + nb_compares, + mls, + )?; + } + + // Skip repetitive patterns. + ms.next_to_update = match_end_idx - 8; + Ok(best_length) +} + +/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. +/// `off_base_ptr` is written to if there is one. +#[inline(never)] +fn bt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + off_base_ptr: &mut u32, +) -> Result { + if ip < src.pos_of(ms.next_to_update) { + // In an area that is skipped. + return Ok(0); + } + update_dubt(ms, src.prefix(), ip, MLS)?; + dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) +} + +// ── hash chains ─────────────────────────────────────────────────────────── + +/// `ZSTD_insertAndFindFirstIndex_internal` +#[inline(always)] +fn insert_and_find_first_index_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + lazy_skipping: bool, +) -> Result { + let hash_log = ms.c_params.hash_log; + let chain_mask = (1u32 << ms.c_params.chain_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let chain_table = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); + chain_table.put((idx & chain_mask) as usize, next_in_chain)?; + // While skipping, not every position is inserted. + if lazy_skipping { + break; + } + } + + ms.next_to_update = target; + hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) +} + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) +} + +/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn hc_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let chain_size = 1u32 << ms.c_params.chain_log; + let chain_mask = chain_size - 1; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let min_chain = curr.saturating_sub(chain_size); + let mut nb_attempts = 1u32 << ms.c_params.search_log; + let mut ml = 4 - 1; + + let mut match_index = + insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; + let chain_table = ms.chain_table.as_slice(); + + while match_index >= low_limit && nb_attempts > 0 { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + + if match_index <= min_chain { + break; + } + match_index = chain_table.at((match_index & chain_mask) as usize)?; + nb_attempts -= 1; + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_chain_table = dms.ms.chain_table.as_slice(); + let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; + let dms_chain_mask = dms_chain_size - 1; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_size = dms_end_index(&dms); + let dms_index_delta = dict_limit.wrapping_sub(dms_size); + let dms_min_chain = dms_size.saturating_sub(dms_chain_size); + + let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; + match_index = dms.ms.hash_table.at(h)?; + + while match_index >= dms_lowest_index && nb_attempts > 0 { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + + if match_index <= dms_min_chain { + break; + } + match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; + nb_attempts -= 1; + } + } + + Ok(ml) +} + +// ── row-based match finder ──────────────────────────────────────────────── +// +// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits +// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag +// table is the "head": which entry was written last. + +pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; +const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; +const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; + +/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. +#[inline(always)] +fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { + let mut next = u32::from(*head).wrapping_sub(1) & row_mask; + if next == 0 { + next = row_mask; + } + *head = next as u8; + next +} + +/// `ZSTD_row_prefetch`: of the row that starts at `rel_row`. +#[inline(always)] +fn row_prefetch(hash_table: &[u32], tag_table: &[u8], rel_row: usize, row_log: u32) { + prefetch_l1(hash_table, rel_row); + if row_log >= 5 { + prefetch_l1(hash_table, rel_row + 16); + } + prefetch_l1(tag_table, rel_row); + if row_log == 6 { + prefetch_l1(tag_table, rel_row + 32); + } +} + +/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. +#[inline(always)] +fn row_fill_hash_cache( + ms: &mut MatchState, + prefix: Prefix<'_>, + row_log: u32, + mls: u32, + idx: u32, + i_limit: usize, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); + let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; + + for idx in idx..lim { + let hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + ms.hash_salt, + )? as u32; + let row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + row_prefetch(&ms.hash_table, &ms.tag_table, row, row_log); + let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; + ms.hash_cache.put(cached, hash)?; + } + Ok(()) +} + +/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes +/// further takes its place. +#[inline(always)] +fn row_next_cached_hash( + cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], + hash_table: &[u32], + tag_table: &[u8], + prefix: Prefix<'_>, + idx: u32, + hash_log: u32, + row_log: u32, + mls: u32, + hash_salt: u64, +) -> Result { + let new_hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32; + let row = ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + row_prefetch(hash_table, tag_table, row, row_log); + let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; + Ok(core::mem::replace(cached, new_hash)) +} + +/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. +#[inline(always)] +fn row_update_internal_impl( + ms: &mut MatchState, + prefix: Prefix<'_>, + update_start_idx: u32, + update_end_idx: u32, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let hash_salt = ms.hash_salt; + let hash_table = ms.hash_table.as_mut_slice(); + let tag_table = ms.tag_table.as_mut_slice(); + + for idx in update_start_idx..update_end_idx { + let hash = if use_cache { + row_next_cached_hash( + &mut ms.hash_cache, + hash_table, + tag_table, + prefix, + idx, + hash_log, + row_log, + mls, + hash_salt, + )? + } else { + hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32 + }; + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; + tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; + hash_table.put(rel_row + pos, idx)?; + } + Ok(()) +} + +/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is +/// when compressing, not all of them if they are many. +#[inline(always)] +fn row_update_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + const K_SKIP_THRESHOLD: u32 = 384; + const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; + const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; + + let mut idx = ms.next_to_update; + let target = prefix.index_of(ip); + + if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { + let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; + row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; + idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; + row_fill_hash_cache(ms, prefix, row_log, mls, idx, ip + 1)?; + } + row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the +/// row-based match finder. +pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { + let row_log = ms.c_params.search_log.clamp(4, 6); + let row_mask = (1u32 << row_log) - 1; + let mls = ms.c_params.min_match.min(6); + row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) +} + +/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. +#[inline(always)] +fn row_get_match_mask( + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + head: u32, +) -> u64 { + let mut matches = 0u64; + let (chunks, _) = tag_row.as_chunks::<16>(); + for (n, chunk) in chunks.iter().enumerate() { + let mut chunk_matches = 0u16; + let word = u128::from_le_bytes(*chunk); + for i in 0..16 { + chunk_matches |= u16::from((word >> (8 * i)) as u8 == tag) << i; + } + matches |= u64::from(chunk_matches) << (16 * n); + } + match ROW_ENTRIES { + 16 => u64::from((matches as u16).rotate_right(head)), + 32 => u64::from((matches as u32).rotate_right(head)), + _ => matches.rotate_right(head), + } +} + +/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, +/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than +/// `low_limit`. `prefetch` is called with each. Returns how many there are. +#[inline(always)] +fn row_collect_matches( + row: &[u32; ROW_ENTRIES], + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + low_limit: u32, + nb_attempts: &mut u32, + match_buffer: &mut [u32; ROW_ENTRIES], + prefetch: impl Fn(u32), +) -> Result { + let row_mask = ROW_ENTRIES as u32 - 1; + let head = u32::from(tag_row.at(0)?) & row_mask; + let mut num_matches = 0usize; + let mut matches = row_get_match_mask(tag_row, tag, head); + + while matches > 0 && *nb_attempts > 0 { + let match_pos = (head + matches.trailing_zeros()) & row_mask; + matches &= matches - 1; + if match_pos == 0 { + // The head, which happens to be the same as the tag. + continue; + } + let match_index = row.at(match_pos as usize)?; + if match_index < low_limit { + break; + } + prefetch(match_index); + match_buffer.put(num_matches, match_index)?; + num_matches += 1; + *nb_attempts -= 1; + } + Ok(num_matches) +} + +/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn row_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let hash_log = ms.row_hash_log; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let row_log = ROW_ENTRIES.trailing_zeros(); + let row_mask = ROW_ENTRIES as u32 - 1; + // There are no more searches than entries in a row. + let capped_search_log = ms.c_params.search_log.min(row_log); + let hash_salt = ms.hash_salt; + let mut nb_attempts = 1u32 << capped_search_log; + let mut ml = 4 - 1; + let mut match_buffer = [0u32; ROW_ENTRIES]; + + let mut dms_hash = 0u32; + let mut dms_rel_row = 0usize; + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; + dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + row_prefetch(&dms.ms.hash_table, &dms.ms.tag_table, dms_rel_row, row_log); + } + + // Update the tables up to, and not including, `ip`. + let hash = if ms.lazy_skipping { + // While skipping, not every position is inserted. + ms.next_to_update = curr; + hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 + } else { + let prefix = src.prefix(); + row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; + row_next_cached_hash( + &mut ms.hash_cache, + &ms.hash_table, + &ms.tag_table, + prefix, + curr, + hash_log, + row_log, + MLS, + hash_salt, + )? + }; + ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); + + { + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; + let row = ms.hash_table.array_at_mut::(rel_row)?; + let tag_row = ms.tag_table.array_at_mut::(rel_row)?; + let num_matches = row_collect_matches( + row, + tag_row, + tag, + low_limit, + &mut nb_attempts, + &mut match_buffer, + |match_index| { + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + prefetch_l1(buf, src.pos_of(match_index)); + } else { + prefetch_l1(src.dict_buf, src.dict_pos_of(match_index)); + } + }, + )?; + + // Insert `ip`. + let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; + tag_row.put(pos, tag)?; + row.put(pos, ms.next_to_update)?; + ms.next_to_update += 1; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + } + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); + + let num_matches = row_collect_matches( + dms.ms.hash_table.array_at::(dms_rel_row)?, + dms.ms.tag_table.array_at::(dms_rel_row)?, + (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, + dms_lowest_index, + &mut nb_attempts, + &mut match_buffer, + |match_index| prefetch_l1(dms.buf, dms.pos_of(match_index)), + )?; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + } + } + + Ok(ml) +} + +// ── the parsers ─────────────────────────────────────────────────────────── + +// `searchMethod_e`, as an argument of a const generic. +const SEARCH_HASH_CHAIN: u8 = 0; +const SEARCH_BINARY_TREE: u8 = 1; +const SEARCH_ROW_HASH: u8 = 2; + +/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. +#[inline(always)] +fn search_max( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mls: u32, + row_log: u32, +) -> Result { + match SEARCH_METHOD { + SEARCH_HASH_CHAIN => match mls { + 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + SEARCH_BINARY_TREE => match mls { + 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + _ => match (mls, row_log) { + (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + } +} + +/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match +/// state is searched. +#[derive(Clone, Copy)] +struct DictSegment<'a> { + buf: &'a [u8], + /// Its sum with an index of the window is a position in `buf`. It wraps around. + base: usize, + /// Where the segment starts in `buf`. + start: usize, + /// Where it ends. + end: usize, +} + +/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. +#[inline(always)] +fn rep_length( + ms: &MatchState, + src: &BlockSrc<'_>, + dict: &DictSegment<'_>, + ip: usize, + offset: u32, +) -> Result { + let buf = src.buf; + let iend = src.end; + if DICT_MODE == NO_DICT { + let rep_match = ip.wrapping_sub(offset as usize); + if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { + return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); + } + return Ok(0); + } + + let dict_limit = ms.window.dict_limit; + let curr = src.index_of(ip); + let rep_index = curr.wrapping_sub(offset); + if !index_overlap_check(dict_limit, rep_index) { + return Ok(0); + } + if DICT_MODE == EXT_DICT { + // The offset may lead to what is no longer in the window. + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + if offset > curr.wrapping_sub(window_low) { + return Ok(0); + } + } + let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { + let rep_match = dict.base.wrapping_add(rep_index as usize); + (dict.buf, rep_match, dict.end) + } else { + (buf, src.pos_of(rep_index), iend) + }; + if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { + return Ok(0); + } + let prefix_start = src.pos_of(dict_limit); + Ok(count_2segments( + buf, + ip + 4, + rep_buf, + rep_match + 4, + iend, + rep_end, + prefix_start, + )? + 4) +} + +/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions +/// further a better match is looked for, once there is one. +#[inline(never)] +fn compress_block_lazy_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let prefix_lowest_index = ms.window.dict_limit; + let prefix_lowest = src.pos_of(prefix_lowest_index); + let mls = ms.c_params.min_match.clamp(4, 6); + let row_log = ms.c_params.search_log.clamp(4, 6); + + let [rep_0, rep_1, _] = rep; + let mut offset_1 = *rep_0; + let mut offset_2 = *rep_1; + let mut offset_saved_1 = 0u32; + let mut offset_saved_2 = 0u32; + + ms.lazy_skipping = false; + + let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { + 8 + ZSTD_ROW_HASH_CACHE_SIZE + } else { + 8 + }; + let Some(ilimit) = iend.checked_sub(margin) else { + // There is no position to search, nor to hash. + return Ok(iend - istart); + }; + + let dict = match DICT_MODE { + NO_DICT => DictSegment { + buf: &[], + base: 0, + start: 0, + end: 0, + }, + EXT_DICT => DictSegment { + buf: src.dict_buf, + base: src.dict_base, + start: src.dict_pos_of(ms.window.low_limit), + end: src.dict_pos_of(prefix_lowest_index), + }, + _ => { + let dms = src.dms.ok_or(BUG)?; + let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); + DictSegment { + buf: dms.buf, + base: dms.base.wrapping_sub(dict_index_delta as usize), + start: dms.pos_of(dms.ms.window.dict_limit), + end: dms.end, + } + } + }; + + if DICT_MODE == EXT_DICT { + ip += usize::from(ip == prefix_lowest); + } else { + let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); + ip += usize::from(dict_and_prefix_length == 0); + } + if DICT_MODE == NO_DICT { + let curr = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); + let max_rep = curr - window_low; + if offset_2 > max_rep { + offset_saved_2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved_1 = offset_1; + offset_1 = 0; + } + } + + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), row_log, mls, ms.next_to_update, ilimit)?; + } + + 'parse: while ip < ilimit { + let mut match_length; + let mut off_base = REPCODE1_TO_OFFBASE; + let mut start = ip + 1; + + 'found: { + // Check the repeat offset. + match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; + if DEPTH == 0 && match_length != 0 { + break 'found; + } + + // First search. + { + let mut off_base_found = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut off_base_found, + mls, + row_log, + )?; + if ml2 > match_length { + match_length = ml2; + start = ip; + off_base = off_base_found; + } + } + + if match_length < 4 { + // Jump faster over incompressible sections. + let step = (ip - anchor) >> K_SEARCH_STRENGTH; + ip += step + 1; + // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of + // no use out of the tables. As in the C code, it does not start at the same step in all modes. + ms.lazy_skipping = if DICT_MODE == EXT_DICT { + step > K_LAZY_SKIPPING_STEP + } else { + step + 1 > K_LAZY_SKIPPING_STEP + }; + continue 'parse; + } + + // Look for something better at the next positions. + if DEPTH >= 1 { + while ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 3) as i32; + let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + + // Nothing better: try one position further. + if DEPTH == 2 && ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 4) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + } + break; + } + } + + // Catch up: the match may start earlier. + if offbase_is_offset(off_base) { + let offset = offbase_to_offset(off_base); + let match_index = src.index_of(start).wrapping_sub(offset); + let (match_buf, mut match_pos, m_start) = + if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { + let match_pos = dict.base.wrapping_add(match_index as usize); + (dict.buf, match_pos, dict.start) + } else { + (buf, src.pos_of(match_index), prefix_lowest) + }; + while start > anchor + && match_pos > m_start + && buf.at(start - 1)? == match_buf.at(match_pos - 1)? + { + start -= 1; + match_pos -= 1; + match_length += 1; + } + offset_2 = offset_1; + offset_1 = offset; + } + } + + // Store the sequence. + seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; + ip = start + match_length; + anchor = ip; + + if ms.lazy_skipping { + // There was a match: every position is inserted again. + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), row_log, mls, ms.next_to_update, ilimit)?; + } + ms.lazy_skipping = false; + } + + // Check the second repeat offset right away. + while ip <= ilimit { + match_length = rep_length::(ms, src, &dict, ip, offset_2)?; + if match_length == 0 { + break; + } + (offset_1, offset_2) = (offset_2, offset_1); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; + ip += match_length; + anchor = ip; + } + } + + // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. + if offset_saved_1 != 0 && offset_1 != 0 { + offset_saved_2 = offset_saved_1; + } + + // Save the repeat offsets for the next block. + *rep_0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved_1 + }; + *rep_1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved_2 + }; + + // The size of the last literals. + Ok(iend - anchor) +} + +/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. +#[inline(always)] +fn compress_block_lazy_any_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + match dict_mode { + DictMode::NoDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::ExtDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + compress_block_lazy_generic::( + ms, seq_store, rep, src, + ) + } + } +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row: bool, + dict_mode: DictMode, +) -> Result { + match (strategy, use_row) { + (Strategy::Greedy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Greedy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( + ms, seq_store, rep, src, dict_mode, + ), + _ => Err(BUG), + } +} diff --git a/src/zstd/wip/work/lazy/zstd/compress/ldm.rs b/src/zstd/wip/work/lazy/zstd/compress/ldm.rs new file mode 100644 index 000000000000..2177fd945e10 --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/compress/ldm.rs @@ -0,0 +1 @@ +//! `zstd_ldm.c`: long distance matching. diff --git a/src/zstd/wip/work/lazy/zstd/compress/mod.rs b/src/zstd/wip/work/lazy/zstd/compress/mod.rs new file mode 100644 index 000000000000..13ddecdf82e4 --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/compress/mod.rs @@ -0,0 +1,68 @@ +//! `lib/compress` + +pub(crate) mod double_fast; +pub(crate) mod fast; +pub(crate) mod fse; +pub(crate) mod hist; +pub(crate) mod huf; +pub(crate) mod internal; +pub(crate) mod lazy; +pub(crate) mod ldm; +pub(crate) mod opt; +pub(crate) mod presplit; + +use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; + +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; + +/// `ZSTD_rowMatchFinderSupported` +pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { + strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 +} + +/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to +/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies +/// that support it look at. Returns how many bytes at the end of the block are in no sequence. +pub(crate) fn compress_block( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row_match_finder: bool, + dict_mode: DictMode, +) -> Result { + match strategy { + Strategy::Fast => match dict_mode { + DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), + DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), + DictMode::DictMatchState => { + fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::DFast => match dict_mode { + DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), + DictMode::ExtDict => { + double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { + lazy::compress_block_lazy( + ms, + seq_store, + rep, + src, + strategy, + use_row_match_finder && row_match_finder_supported(strategy), + dict_mode, + ) + } + Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { + opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) + } + } +} diff --git a/src/zstd/wip/work/lazy/zstd/compress/opt.rs b/src/zstd/wip/work/lazy/zstd/compress/opt.rs new file mode 100644 index 000000000000..584379ea28d3 --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/compress/opt.rs @@ -0,0 +1,96 @@ +//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. + +use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. +/// Matches are not compared past the position `iend`. +pub(crate) fn update_tree( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + iend: usize, +) -> Result<()> { + // Test scaffolding only: noDict `ZSTD_updateTree_internal` + `ZSTD_insertBt1`. + use super::internal::{count, hash_ptr}; + use crate::util::SliceExt; + let mls = ms.c_params.min_match; + let target = prefix.index_of(ip); + let mut idx = ms.next_to_update; + while idx < target { + let ip = prefix.pos_of(idx); + let h = hash_ptr(prefix.buf, ip, ms.c_params.hash_log, mls)?; + let bt_mask = (1u32 << (ms.c_params.chain_log - 1)) - 1; + let mut match_index = ms.hash_table.at(h)?; + let (mut cls, mut cll) = (0usize, 0usize); + let curr = idx; + let bt_low = curr.saturating_sub(bt_mask); + let mut smaller = 2 * (curr & bt_mask) as usize; + let mut larger = smaller + 1; + let window_low = ms.get_lowest_match_index(target, ms.c_params.window_log); + let mut match_end_idx = curr + 8 + 1; + let mut best_length = 8usize; + let mut nb_compares = 1u32 << ms.c_params.search_log; + ms.hash_table.put(h, curr)?; + let bt = ms.chain_table.as_mut_slice(); + while nb_compares != 0 && match_index >= window_low { + let next = 2 * (match_index & bt_mask) as usize; + let mut ml = cls.min(cll); + let m = prefix.pos_of(match_index); + ml += count(prefix.buf, ip + ml, prefix.buf, m + ml, iend)?; + if ml > best_length { + best_length = ml; + if ml > (match_end_idx - match_index) as usize { + match_end_idx = match_index + ml as u32; + } + } + if ip + ml == iend { + break; + } + if prefix.buf.at(m + ml)? < prefix.buf.at(ip + ml)? { + bt.put(smaller, match_index)?; + cls = ml; + if match_index <= bt_low { + smaller = larger; + break; + } + smaller = next + 1; + match_index = bt.at(next + 1)?; + } else { + bt.put(larger, match_index)?; + cll = ml; + if match_index <= bt_low { + larger = smaller; + break; + } + larger = next; + match_index = bt.at(next)?; + } + nb_compares -= 1; + } + bt.put(smaller, 0)?; + bt.put(larger, 0)?; + let positions = if best_length > 384 { + 192.min(best_length as u32 - 384) + } else { + 0 + }; + idx += positions.max(match_end_idx - (curr + 8)); + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` +/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. +pub(crate) fn compress_block_opt( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, + _strategy: Strategy, + _dict_mode: DictMode, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/lazy/zstd/compress/presplit.rs b/src/zstd/wip/work/lazy/zstd/compress/presplit.rs new file mode 100644 index 000000000000..8b778e2989f9 --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/compress/presplit.rs @@ -0,0 +1 @@ +//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. diff --git a/src/zstd/wip/work/lazy/zstd/lib.rs b/src/zstd/wip/work/lazy/zstd/lib.rs new file mode 100644 index 000000000000..3b2a2c6fd519 --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/lib.rs @@ -0,0 +1,386 @@ +//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the +//! same bytes in, in both directions, and the same errors. +#![forbid(unsafe_code)] +#![feature(hint_prefetch)] +// Frames come from anywhere and a panic aborts the process. +#![deny( + clippy::indexing_slicing, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unreachable +)] +#![warn(unused_must_use)] + +mod common; +mod compress; +mod decompress; +mod error; +mod util; + +pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; +pub use decompress::{ + DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, + find_frame_compressed_size, get_frame_content_size, +}; +pub use error::ErrorCode; + +/// `ZSTD_inBuffer` +pub struct InBuffer<'a> { + pub src: &'a [u8], + /// How much of `src` is read. + pub pos: usize, +} + +/// `ZSTD_outBuffer` +pub struct OutBuffer<'a> { + pub dst: &'a mut [u8], + /// How much of `dst` is written. + pub pos: usize, +} + +/// `ZSTD_ResetDirective` +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ResetDirective { + SessionOnly = 1, + Parameters = 2, + SessionAndParameters = 3, +} + +// ----------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] +pub enum ZstdError { + /// The output, or the decoder state the frame's window size dictates, could not be allocated. + OutOfMemory, + InvalidZstdData, + DecompressionFailed, + ZstdDecompressionError, + ShortRead, +} + +impl core::fmt::Display for ZstdError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(<&'static str>::from(self)) + } +} +impl core::error::Error for ZstdError {} + +impl ZstdError { + /// The error for a failed decompression call; `other` is the non-allocation failure. + fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { + if code == ErrorCode::MemoryAllocation { + ZstdError::OutOfMemory + } else { + other + } + } +} + +/// `len` zeros for a decoder to write to. +fn output_buffer(len: usize) -> Result, ZstdError> { + util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) +} + +/// Decompress data, automatically allocating the output buffer. +/// Handles both frames with known and unknown content sizes. +/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. +/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. +pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { + let size = find_decompressed_size(src); + + const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit + + if size == ZSTD_CONTENTSIZE_ERROR { + return Err(ZstdError::InvalidZstdData); + } + + // Use streaming decompression if: + // 1. Content size is unknown, OR + // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) + if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { + let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { + // A frame's output is rarely smaller than its input. + src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) + } else { + // The header size is untrusted: reserve no more than the fast path below would. + MAX_PREALLOCATE_SIZE + }; + let mut list: Vec = Vec::new(); + list.try_reserve_exact(initial_capacity) + .map_err(|_| ZstdError::OutOfMemory)?; + StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; + return Ok(list); + } + + // Fast path: size is known and within reasonable limits + let mut output = output_buffer(size as usize)?; + let written = decompress(&mut output, src) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is +/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. +pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { + const MAX_SIZE: u64 = 16 * 1024 * 1024; + let size = find_decompressed_size(src); + // Also if it is unknown or invalid, which are the two largest values. + if size > MAX_SIZE { + return Err(ZstdError::InvalidZstdData); + } + let mut output = output_buffer(size as usize)?; + let written = DDict::prefix(prefix) + .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + Uninitialized, + Inflating, + End, + Error, +} + +/// Minimum spare output capacity offered to the decoder per call. +const STREAMING_OUTPUT_STEP: usize = 4096; + +// ────────────────────────────────────────────────────────────────────────── +// StreamingDecoder +// ────────────────────────────────────────────────────────────────────────── + +/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) +/// per call, so callers can hold the decoder across multiple body chunks. +pub struct StreamingDecoder { + stream: DCtx, + state: State, + /// The decoder filled its last window and may hold more. `max_output` can end a call there. + output_full: bool, +} + +impl Default for StreamingDecoder { + fn default() -> Self { + Self::new() + } +} + +impl StreamingDecoder { + pub fn new() -> Self { + Self { + stream: DCtx::new(), + state: State::Uninitialized, + output_full: false, + } + } + + #[inline] + pub fn is_inflating(&self) -> bool { + matches!(self.state, State::Inflating) + } + + /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is + /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. + /// Returns `ShortRead` when more input is required and `is_done` is false. + pub fn decompress( + &mut self, + input: &[u8], + out: &mut Vec, + max_output: usize, + is_done: bool, + ) -> Result { + if matches!(self.state, State::End | State::Error) { + return Ok(input.len()); + } + + let mut in_buf = InBuffer { src: input, pos: 0 }; + while matches!(self.state, State::Uninitialized | State::Inflating) { + let next_in_len = input.len() - in_buf.pos; + + // Call the decoder again with no input until it leaves the window short. + if next_in_len == 0 && !self.output_full { + if is_done { + if self.state == State::Inflating { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + self.state = State::End; + } + return Ok(in_buf.pos); + } + + if out.len() >= max_output { + return Ok(in_buf.pos); + } + + if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { + self.state = State::Error; + return Err(ZstdError::OutOfMemory); + } + // The decoder writes to initialized memory only. + let written_before = out.len(); + let window = (out.capacity() - written_before).min(max_output - written_before); + out.resize(written_before + window, 0); + let mut out_buf = OutBuffer { + dst: out.get_mut(written_before..).unwrap_or_default(), + pos: 0, + }; + let read_before = in_buf.pos; + + let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); + let bytes_written = out_buf.pos; + out.truncate(written_before + bytes_written); + let remaining = match result { + Ok(remaining) => remaining, + Err(code) => { + self.state = State::Error; + return Err(ZstdError::for_decompression( + code, + ZstdError::ZstdDecompressionError, + )); + } + }; + + self.output_full = bytes_written == window; + + if remaining == 0 { + // Frame complete, and fully flushed. + self.state = State::Uninitialized; + self.output_full = false; + if in_buf.pos >= input.len() { + if is_done { + self.state = State::End; + } + return Ok(in_buf.pos); + } + // More input available — reinitialize for the next frame. + self.stream.init_stream(); + continue; + } + + self.state = State::Inflating; + + if in_buf.pos - read_before == next_in_len { + if self.output_full { + continue; + } + if is_done { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + return Err(ZstdError::ShortRead); + } + } + Ok(in_buf.pos) + } +} + +// ── compressed embedded assets ──────────────────────────────────────────── + +/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes +/// Bun never executes or parses itself: the shell completion scripts and the +/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, +/// error overlay/page). Anything that runs inside Bun stays uncompressed. +/// +/// The exception is text that only `bun lint`, `bun fmt` and `bun check` +/// read. It may be compressed because it isn't on the path of running a program. +/// +/// Release (`bun_codegen_embed`) builds include the `.zst` twin that +/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to +/// `/compressed/.zst`; other builds evaluate `$fallback`. +#[macro_export] +macro_rules! embed_compressed { + // A codegen output (`/$sub`); debug builds read it at runtime. + (codegen $sub:literal) => { + $crate::embed_compressed!( + ("codegen/", $sub), + ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() + ) + }; + // Same, with a trailing NUL included in the slice (for C-string consumers). + (codegen_nul $sub:literal) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/codegen/", + $sub, + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); + &__COPY.get_or_init(|| { + let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); + let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); + copy.extend_from_slice(text); + copy.push(0); + copy.into_boxed_slice() + })[..] + } + }; + __bytes + }}; + // A file under `src/`; debug builds read it from the source tree at runtime. + (src $sub:literal) => { + $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) + }; + ($name:literal, $fallback:expr) => { + $crate::embed_compressed!(($name), $fallback) + }; + (($($name:literal),+), $fallback:expr) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/", + $($name,)+ + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + $fallback + } + }; + __bytes + }}; +} + +/// Cold, shared body of [`embed_compressed!`]'s release arm. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { + decompress_alloc(compressed) + .expect("embedded asset: invalid zstd frame") + .into_boxed_slice() +} + +/// [`inflate_embedded`] plus a trailing NUL byte. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { + let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); + inflated.reserve_exact(1); + inflated.push(0); + inflated.into_boxed_slice() +} +pub mod testing; diff --git a/src/zstd/wip/work/lazy/zstd/testing.rs b/src/zstd/wip/work/lazy/zstd/testing.rs new file mode 100644 index 000000000000..2e64ee5e7a3e --- /dev/null +++ b/src/zstd/wip/work/lazy/zstd/testing.rs @@ -0,0 +1,372 @@ +//! Hooks for the differential tests. Only in scratch copies of the crate. +#![allow(clippy::all, dead_code, unreachable_pub)] + +use crate::ErrorCode; +use crate::compress::internal::*; +use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; + +fn bitmix(mut val: u64, len: u64) -> u64 { + val ^= val.rotate_right(49) ^ val.rotate_right(24); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^= (val >> 35).wrapping_add(len); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^ (val >> 28) +} + +#[derive(Clone, Copy)] +pub enum Dict<'a> { + None, + /// `ZSTD_CCtx_refPrefix`, raw content. + Prefix(&'a [u8]), + /// `ZSTD_CCtx_refCDict`, raw content, attached. With the parameters of the CDict. + CDict(&'a [u8], [u32; 7]), +} + +/// What is left in the match state after the frame. +#[derive(PartialEq, Eq, Debug, Default)] +pub struct State { + pub has_dms: bool, + pub next_to_update: u32, + pub loaded_dict_end: u32, + pub low_limit: u32, + pub dict_limit: u32, + pub hash_salt_entropy: u32, + pub lazy_skipping: bool, + pub hash_cache: [u32; 8], + pub rep: [u32; 3], + pub hash_table: Vec, + pub chain_table: Vec, + pub tag_table: Vec, +} + +fn params(cp: [u32; 7]) -> Result { + Ok(CompressionParameters { + window_log: cp[0], + chain_log: cp[1], + hash_log: cp[2], + search_log: cp[3], + min_match: cp[4], + target_length: cp[5], + strategy: Some(Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?), + }) +} + +/// `ZSTD_reset_matchState`, on a new one. +fn reset_match_state( + c_params: CompressionParameters, + use_row: bool, + for_cdict: bool, +) -> MatchState { + let strategy = c_params.strat(); + let row_used = row_match_finder_supported(strategy) && use_row; + let mut ms = MatchState::new(); + ms.hash_log3 = if !for_cdict && c_params.min_match == 3 { + c_params.window_log.min(17) + } else { + 0 + }; + ms.window.clear(); + ms.next_to_update = ms.window.dict_limit; + ms.hash_table = vec![0; 1 << c_params.hash_log]; + ms.chain_table = vec![ + 0; + if strategy != Strategy::Fast && !row_used { + 1 << c_params.chain_log + } else { + 0 + } + ]; + ms.hash_table3 = vec![ + 0; + if ms.hash_log3 != 0 { + 1 << ms.hash_log3 + } else { + 0 + } + ]; + if row_used { + ms.tag_table = vec![0; 1 << c_params.hash_log]; + ms.hash_salt = if for_cdict { + 0 + } else { + bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4) + }; + ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); + } + if !for_cdict && strategy >= Strategy::BtOpt { + ms.opt.lit_freq = vec![0; 256]; + ms.opt.lit_length_freq = vec![0; 36]; + ms.opt.match_length_freq = vec![0; 53]; + ms.opt.off_code_freq = vec![0; 32]; + ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; + ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; + } + ms.c_params = c_params; + ms +} + +/// `ZSTD_loadDictionaryContent` +fn load_dictionary_content( + ms: &mut MatchState, + use_row: bool, + dict: &[u8], + dtlm: DictTableLoadMethod, + tfp: TableFillPurpose, +) -> Result<(), ErrorCode> { + let c_params = ms.c_params; + let iend = dict.len(); + ms.window + .update(BufId::Dict.start(), BufId::Dict, dict.len(), false); + let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); + let ip = iend - dict.len().min(max_dict_size); + let prefix = Prefix { + buf: dict, + base: ms.window.base.wrapping_sub(BufId::Dict.start()), + }; + ms.next_to_update = prefix.index_of(ip); + ms.loaded_dict_end = prefix.index_of(iend); + if iend - ip > HASH_READ_SIZE { + match c_params.strat() { + Strategy::Fast => fast::fill_hash_table(ms, prefix, iend, dtlm, tfp)?, + Strategy::DFast => double_fast::fill_double_hash_table(ms, prefix, iend, dtlm, tfp)?, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { + if use_row { + ms.tag_table.fill(0); + lazy::row_update(ms, prefix, iend - HASH_READ_SIZE)?; + } else { + lazy::insert_and_find_first_index(ms, prefix, iend - HASH_READ_SIZE)?; + } + } + _ => opt::update_tree(ms, prefix, iend - HASH_READ_SIZE, iend)?, + } + ms.next_to_update = prefix.index_of(iend); + } + Ok(()) +} + +/// The sequences of all the blocks of a frame, as (literal length, match length, repeat code or 0, offset or 0). Each +/// block ends with the literals that are left as (length, 0, 0, 0). +/// +/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, as applied. +pub fn frame( + cp: [u32; 7], + use_row: bool, + dict: Dict<'_>, + src: &[u8], + streaming: bool, +) -> Result<(Vec<(u32, u32, u32, u32)>, State), ErrorCode> { + let c_params = params(cp)?; + let strategy = c_params.strat(); + let mut ms = reset_match_state(c_params, use_row, false); + + let block_size_max = (1usize << c_params.window_log).min(128 << 10); + let mut seq_store = SeqStore::new(); + seq_store.reserve( + block_size_max / if c_params.min_match == 3 { 3 } else { 4 }, + block_size_max, + )?; + + let dict_content = match dict { + Dict::None => &[][..], + Dict::Prefix(d) | Dict::CDict(d, _) => d, + }; + // As `ZSTD_compressStream2` does: blocks are copied to a buffer of the size of the window and a block. + let mut in_buff = vec![ + 0u8; + if streaming { + (1usize << c_params.window_log) + block_size_max + } else { + 0 + } + ]; + let mut in_buff_pos = 0usize; + + let mut cdict_ms = None; + match dict { + Dict::Prefix(d) if d.len() >= 8 => load_dictionary_content( + &mut ms, + use_row, + d, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + )?, + Dict::CDict(d, cdict_cp) if d.len() >= 8 => { + let mut c = reset_match_state(params(cdict_cp)?, use_row, true); + load_dictionary_content( + &mut c, + use_row, + d, + DictTableLoadMethod::Full, + TableFillPurpose::ForCDict, + )?; + // ZSTD_resetCCtx_byAttachingCDict + let cdict_end = c.window.index_of(c.window.next_src); + if ms.window.dict_limit < cdict_end { + ms.window.next_src = ms.window.base + cdict_end as usize; + ms.window.clear(); + } + ms.loaded_dict_end = ms.window.dict_limit; + cdict_ms = Some(c); + } + _ => {} + } + let mut dms = cdict_ms.as_ref().map(|c| DictMatchState { + ms: c, + buf: dict_content, + base: c.window.base.wrapping_sub(BufId::Dict.start()), + end: c.window.next_src - BufId::Dict.start(), + }); + + // ZSTD_compressContinue_internal + if !streaming + && !ms.window.update( + BufId::User.start(), + BufId::User, + src.len(), + ms.force_non_contiguous, + ) + { + ms.force_non_contiguous = false; + ms.next_to_update = ms.window.dict_limit; + } + // ZSTD_compress_frameChunk + let max_dist = 1u32 << c_params.window_log; + let prev = CompressedBlockState::new(); + let mut rep = prev.rep; + let mut out = Vec::new(); + let mut pos = 0; + while pos < src.len() { + let block_size = (src.len() - pos).min(block_size_max); + let (block_addr, block_start) = if streaming { + in_buff[in_buff_pos..in_buff_pos + block_size] + .copy_from_slice(&src[pos..pos + block_size]); + let addr = BufId::InBuff.start() + in_buff_pos; + if !ms + .window + .update(addr, BufId::InBuff, block_size, ms.force_non_contiguous) + { + ms.force_non_contiguous = false; + ms.next_to_update = ms.window.dict_limit; + } + (addr, in_buff_pos) + } else { + (BufId::User.start() + pos, pos) + }; + let bufs = Buffers { + in_buff: &in_buff, + user: src, + dict: dict_content, + }; + if ms + .window + .dict_is_invalid(block_addr + block_size, max_dist, ms.loaded_dict_end) + { + ms.loaded_dict_end = 0; + dms = None; + } + if ms + .window + .enforce_max_dist(block_addr, max_dist, ms.loaded_dict_end) + { + ms.loaded_dict_end = 0; + dms = None; + } + if ms.next_to_update < ms.window.low_limit { + ms.next_to_update = ms.window.low_limit; + } + // ZSTD_buildSeqStore + assert!(block_size >= 7); + seq_store.reset(); + let block = BlockSrc { + buf: bufs.get(ms.window.base_buf), + base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), + dict_buf: bufs.get(ms.window.dict_buf), + dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), + start: block_start, + end: block_start + block_size, + dms, + symbol_costs: &prev.entropy, + ldm_seq_store: None, + }; + let curr = block.index_of(block_start); + if curr > ms.next_to_update + 384 { + ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); + } + let dict_mode = ms.dict_mode(dms.as_ref()); + let mut reps = rep; + let last_ll = compress::compress_block( + &mut ms, + &mut seq_store, + &mut rep, + &block, + strategy, + use_row, + dict_mode, + )?; + + let mut total = last_ll; + for (i, seq) in seq_store.sequences.iter().enumerate() { + let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); + if seq_store.long_length_pos as usize == i { + match seq_store.long_length_type { + LongLengthType::LiteralLength => ll += 0x10000, + LongLengthType::MatchLength => ml += 0x10000, + LongLengthType::None => {} + } + } + update_rep(&mut reps, seq.off_base, ll == 0); + total += (ll + ml) as usize; + out.push(if seq.off_base <= 3 { + (ll, ml, seq.off_base, 0) + } else { + (ll, ml, 0, seq.off_base - 3) + }); + } + out.push((last_ll as u32, 0, 0, 0)); + assert_eq!(total, block_size, "the sequences cover the block"); + assert_eq!( + seq_store.lit_len + + last_ll + + seq_store + .sequences + .iter() + .enumerate() + .map(|(i, s)| s.ml_base as usize + + 3 + + if seq_store.long_length_pos as usize == i + && seq_store.long_length_type == LongLengthType::MatchLength + { + 0x10000 + } else { + 0 + }) + .sum::(), + block_size + ); + assert_eq!( + reps[..2], + rep[..2], + "the repeat offsets the block compressor leaves are those of its sequences" + ); + pos += block_size; + in_buff_pos += block_size; + if in_buff_pos + block_size_max > in_buff.len() { + in_buff_pos = 0; + } + } + let state = State { + has_dms: dms.is_some(), + next_to_update: ms.next_to_update, + loaded_dict_end: ms.loaded_dict_end, + low_limit: ms.window.low_limit, + dict_limit: ms.window.dict_limit, + hash_salt_entropy: ms.hash_salt_entropy, + lazy_skipping: ms.lazy_skipping, + hash_cache: ms.hash_cache, + rep, + hash_table: ms.hash_table, + chain_table: ms.chain_table, + tag_table: ms.tag_table, + }; + Ok((out, state)) +} diff --git a/src/zstd/wip/work/ldm/harness/Cargo.toml b/src/zstd/wip/work/ldm/harness/Cargo.toml new file mode 100644 index 000000000000..2c22a568dc5f --- /dev/null +++ b/src/zstd/wip/work/ldm/harness/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "zstd_harness" +version = "0.0.0" +edition = "2024" + +[dependencies] +bun_zstd = { path = "/root/zstd-ref/work/ldm/zstd" } + +[profile.release] +debug = "line-tables-only" +overflow-checks = true +debug-assertions = true +panic = "abort" + +[profile.bench] +inherits = "release" +overflow-checks = false +debug-assertions = false + +[workspace] diff --git a/src/zstd/wip/work/ldm/harness/build.rs b/src/zstd/wip/work/ldm/harness/build.rs new file mode 100644 index 000000000000..85ae4db83966 --- /dev/null +++ b/src/zstd/wip/work/ldm/harness/build.rs @@ -0,0 +1,19 @@ +use std::process::Command; + +fn main() { + let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); + println!("cargo:rerun-if-env-changed=ZSTD_REF"); + println!("cargo:rerun-if-changed=shim.c"); + let out = std::env::var("OUT_DIR").unwrap(); + let ok = Command::new("clang-23") + .args(["-c", "-O2", "-g", "-fPIC", "-I/root/zstd-ref/zstd/lib", "-I/root/zstd-ref/zstd/lib/common", "-DZSTD_MULTITHREAD", + "-DZSTD_LEGACY_SUPPORT=0", "-DXXH_NAMESPACE=ZSTD_", "shim.c", "-o"]) + .arg(format!("{out}/shim.o")).status().unwrap().success(); + assert!(ok); + let _ = std::fs::remove_file(format!("{out}/libshim.a")); + assert!(Command::new("llvm-ar-23").args(["rcs", &format!("{out}/libshim.a"), &format!("{out}/shim.o")]).status().unwrap().success()); + println!("cargo:rustc-link-search=native={out}"); + println!("cargo:rustc-link-lib=static=shim"); + println!("cargo:rustc-link-search=native=/root/zstd-ref/build/{variant}"); + println!("cargo:rustc-link-lib=static=zstd"); +} diff --git a/src/zstd/wip/work/ldm/harness/rust-toolchain.toml b/src/zstd/wip/work/ldm/harness/rust-toolchain.toml new file mode 100644 index 000000000000..c71778e8ccd3 --- /dev/null +++ b/src/zstd/wip/work/ldm/harness/rust-toolchain.toml @@ -0,0 +1,35 @@ +[toolchain] +channel = "nightly-2026-09-15" +# rust-src is needed for -Zbuild-std (Tier 3 targets like +# aarch64-unknown-freebsd have no prebuilt std). miri is for +# `bun run rust:miri`. targets ensures the +# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain +# (CI's images install this same channel with these components and +# targets, from scripts/build/ci-images/spec.ts; a source lint keeps +# the two files in step). Note: rustup ignores this whole file when +# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree +# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` +# self-heals that path. Keep this list as a fallback for direct +# `cargo build`/`cargo check`/rust-analyzer use. +# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd +# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). +# darwin/windows triples build natively on their own agents, so the host std +# comes with the toolchain install regardless; they're listed here so a dev +# running `bun run rust:check-all` or `cargo check --target` from any host +# still gets prebuilt std for every Tier 1/2 triple. +# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at +# what it emits. The build does not use them. +components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] +targets = [ + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-musl", + "aarch64-linux-android", + "x86_64-linux-android", + "x86_64-unknown-freebsd", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-pc-windows-msvc", + "x86_64-pc-windows-msvc", +] diff --git a/src/zstd/wip/work/ldm/harness/shim.c b/src/zstd/wip/work/ldm/harness/shim.c new file mode 100644 index 000000000000..97def6081ebb --- /dev/null +++ b/src/zstd/wip/work/ldm/harness/shim.c @@ -0,0 +1,180 @@ +/* What the tests need of the internals of the C library. */ +#include +#include +#include "compress/zstd_ldm.h" + +typedef struct { + ldmState_t st; + ldmParams_t p; + BYTE* bufs[3]; + RawSeqStore_t store; +} shim_ldm; + +shim_ldm* shim_ldm_new(const U32* p, const size_t* sizes, size_t capacity, U32 startIndex) +{ + shim_ldm* s = (shim_ldm*)calloc(1, sizeof(*s)); + int i; + s->p.enableLdm = (ZSTD_ParamSwitch_e)p[0]; + s->p.hashLog = p[1]; s->p.bucketSizeLog = p[2]; s->p.minMatchLength = p[3]; s->p.hashRateLog = p[4]; s->p.windowLog = p[5]; + ZSTD_window_init(&s->st.window); + s->st.hashTable = (ldmEntry_t*)calloc((size_t)1 << p[1], sizeof(ldmEntry_t)); + s->st.bucketOffsets = (BYTE*)calloc((size_t)1 << (p[1] - p[2]), 1); + if (startIndex) { + s->st.window.nextSrc = s->st.window.base + startIndex; + ZSTD_window_clear(&s->st.window); + } + for (i = 0; i < 3; i++) s->bufs[i] = (BYTE*)calloc(sizes[i] + 64, 1); + s->store.seq = (rawSeq*)calloc(capacity + 1, sizeof(rawSeq)); + s->store.capacity = capacity; + return s; +} + +void shim_ldm_free(shim_ldm* s) +{ + int i; + for (i = 0; i < 3; i++) free(s->bufs[i]); + free(s->st.hashTable); free(s->st.bucketOffsets); free(s->store.seq); free(s); +} + +void shim_ldm_update(shim_ldm* s, int buf, size_t pos, const void* data, size_t len) +{ + memcpy(s->bufs[buf] + pos, data, len); + ZSTD_window_update(&s->st.window, s->bufs[buf] + pos, len, 0); +} + +void shim_ldm_poke(shim_ldm* s, int buf, size_t pos, BYTE byte) { s->bufs[buf][pos] = byte; } + +void shim_ldm_load_dict(shim_ldm* s, int buf, size_t pos, const void* data, size_t len) +{ + const BYTE* ip = s->bufs[buf] + pos; + shim_ldm_update(s, buf, pos, data, len); + s->st.loadedDictEnd = (U32)(ip + len - s->st.window.base); + ZSTD_ldm_fillHashTable(&s->st, ip, ip + len, &s->p); +} + +size_t shim_ldm_generate(shim_ldm* s, int buf, size_t pos, size_t len, int reset) +{ + if (reset) { s->store.size = 0; s->store.pos = 0; } + return ZSTD_ldm_generateSequences(&s->st, &s->store, &s->p, s->bufs[buf] + pos, len); +} + +const rawSeq* shim_ldm_seqs(const shim_ldm* s, size_t* nb) { *nb = s->store.size; return s->store.seq; } + +void shim_ldm_window(const shim_ldm* s, U32* out) +{ + out[0] = s->st.window.dictLimit; out[1] = s->st.window.lowLimit; out[2] = s->st.loadedDictEnd; + out[3] = (U32)(s->st.window.nextSrc - s->st.window.base); out[4] = s->st.window.nbOverflowCorrections; +} + +const ldmEntry_t* shim_ldm_table(const shim_ldm* s) { return s->st.hashTable; } +const BYTE* shim_ldm_bucket_offsets(const shim_ldm* s) { return s->st.bucketOffsets; } + +/* ── ZSTD_ldm_blockCompress, on the match state of a real context ── */ + +typedef struct { + ZSTD_CCtx* cctx; + BYTE* dict; + BYTE* src; + RawSeqStore_t store; + U32 rep[ZSTD_REP_NUM]; +} shim_blk; + +/* cp: windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy. Rewritten with what is applied. */ +shim_blk* shim_blk_new(U32* cp, int useRow, const void* dict, size_t dictSize, const void* src, size_t srcSize, + const rawSeq* seqs, size_t nbSeqs) +{ + shim_blk* s = (shim_blk*)calloc(1, sizeof(*s)); + ZSTD_CCtx* c = ZSTD_createCCtx(); + char tmp[64]; + s->cctx = c; + s->dict = (BYTE*)malloc(dictSize + 64); memcpy(s->dict, dict, dictSize); + s->src = (BYTE*)malloc(srcSize + 64); memcpy(s->src, src, srcSize); + s->store.seq = (rawSeq*)malloc((nbSeqs + 1) * sizeof(rawSeq)); memcpy(s->store.seq, seqs, nbSeqs * sizeof(rawSeq)); + s->store.size = s->store.capacity = nbSeqs; + ZSTD_CCtx_setParameter(c, ZSTD_c_windowLog, (int)cp[0]); + ZSTD_CCtx_setParameter(c, ZSTD_c_chainLog, (int)cp[1]); + ZSTD_CCtx_setParameter(c, ZSTD_c_hashLog, (int)cp[2]); + ZSTD_CCtx_setParameter(c, ZSTD_c_searchLog, (int)cp[3]); + ZSTD_CCtx_setParameter(c, ZSTD_c_minMatch, (int)cp[4]); + ZSTD_CCtx_setParameter(c, ZSTD_c_targetLength, (int)cp[5]); + ZSTD_CCtx_setParameter(c, ZSTD_c_strategy, (int)cp[6]); + ZSTD_CCtx_setParameter(c, ZSTD_c_useRowMatchFinder, useRow ? ZSTD_ps_enable : ZSTD_ps_disable); + ZSTD_CCtx_setPledgedSrcSize(c, srcSize); + if (dictSize) ZSTD_CCtx_refPrefix(c, s->dict, dictSize); + { ZSTD_inBuffer in = { s->src, 0, 0 }; + ZSTD_outBuffer out = { tmp, sizeof(tmp), 0 }; + if (ZSTD_isError(ZSTD_compressStream2(c, &out, &in, ZSTD_e_continue))) abort(); + } + { ZSTD_compressionParameters const a = c->appliedParams.cParams; + cp[0] = a.windowLog; cp[1] = a.chainLog; cp[2] = a.hashLog; cp[3] = a.searchLog; cp[4] = a.minMatch; + cp[5] = a.targetLength; cp[6] = (U32)a.strategy; + } + memcpy(s->rep, c->blockState.prevCBlock->rep, sizeof(s->rep)); + { ZSTD_MatchState_t* const ms = &c->blockState.matchState; + if (!ZSTD_window_update(&ms->window, s->src, srcSize, ms->forceNonContiguous)) { + ms->forceNonContiguous = 0; + ms->nextToUpdate = ms->window.dictLimit; + } + } + return s; +} + +void shim_blk_free(shim_blk* s) +{ + ZSTD_freeCCtx(s->cctx); free(s->dict); free(s->src); free(s->store.seq); free(s); +} + +size_t shim_blk_max_block(const shim_blk* s) { return s->cctx->blockSizeMax; } + +/* What ZSTD_compress_frameChunk and ZSTD_buildSeqStore do for a block, with the sequences as external ones. */ +size_t shim_blk_run(shim_blk* s, size_t pos, size_t blockSize) +{ + ZSTD_CCtx* const zc = s->cctx; + ZSTD_MatchState_t* const ms = &zc->blockState.matchState; + const BYTE* const ip = s->src + pos; + U32 const maxDist = (U32)1 << zc->appliedParams.cParams.windowLog; + size_t lastLL; + ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; + + ZSTD_resetSeqStore(&zc->seqStore); + if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { + if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { + ZSTD_ldm_skipRawSeqStoreBytes(&s->store, blockSize); + } else { + ZSTD_ldm_skipSequences(&s->store, blockSize, zc->appliedParams.cParams.minMatch); + } + return 0; + } + ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; + ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; + { U32 const curr = (U32)(ip - ms->window.base); + if (curr > ms->nextToUpdate + 384) + ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); + } + lastLL = ZSTD_ldm_blockCompress(&s->store, ms, &zc->seqStore, s->rep, zc->appliedParams.useRowMatchFinder, ip, blockSize); + if (ZSTD_isError(lastLL)) return lastLL; + memcpy(zc->seqStore.lit, ip + blockSize - lastLL, lastLL); + zc->seqStore.lit += lastLL; + return lastLL; +} + +const SeqDef* shim_blk_seqs(const shim_blk* s, size_t* nb) +{ + *nb = (size_t)(s->cctx->seqStore.sequences - s->cctx->seqStore.sequencesStart); + return s->cctx->seqStore.sequencesStart; +} +const BYTE* shim_blk_literals(const shim_blk* s, size_t* nb) +{ + *nb = (size_t)(s->cctx->seqStore.lit - s->cctx->seqStore.litStart); + return s->cctx->seqStore.litStart; +} +/* longLengthType, longLengthPos, rep[3], nextToUpdate */ +void shim_blk_state(const shim_blk* s, U32* out) +{ + out[0] = (U32)s->cctx->seqStore.longLengthType; out[1] = s->cctx->seqStore.longLengthPos; + out[2] = s->rep[0]; out[3] = s->rep[1]; out[4] = s->rep[2]; + out[5] = s->cctx->blockState.matchState.nextToUpdate; +} +const RawSeqStore_t* shim_blk_store(const shim_blk* s) { return &s->store; } diff --git a/src/zstd/wip/work/ldm/harness/src/c.rs b/src/zstd/wip/work/ldm/harness/src/c.rs new file mode 100644 index 000000000000..d5a82e1f6b78 --- /dev/null +++ b/src/zstd/wip/work/ldm/harness/src/c.rs @@ -0,0 +1,104 @@ +//! The reference C library. +#![allow(non_camel_case_types, dead_code)] +use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; + +#[repr(C)] +pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } +#[repr(C)] +pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } +pub enum ZSTD_CCtx {} +pub enum ZSTD_DCtx {} +pub enum ZSTD_DDict {} + +unsafe extern "C" { + pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; + pub fn ZSTD_compressBound(size: usize) -> usize; + pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_isError(code: usize) -> c_uint; + pub fn ZSTD_getErrorCode(code: usize) -> c_uint; + pub fn ZSTD_getErrorName(code: usize) -> *const c_char; + pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; + pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; + pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; + pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; + pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; + pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; + pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; + pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; + pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; + pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; + pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; + pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; + pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; + pub fn ZSTD_defaultCLevel() -> c_int; + pub fn ZSTD_minCLevel() -> c_int; + pub fn ZSTD_maxCLevel() -> c_int; +} + +/// `Ok(value)` or `Err(error code)`. +pub fn res(code: usize) -> Result { + unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } +} + +pub struct CCtx(pub *mut ZSTD_CCtx); +impl CCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } + pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + /// Returns (result, out pos, in pos). + pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); + (r, o.pos, i.pos) + } +} +impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } + +pub struct DCtx(pub *mut ZSTD_DCtx); +impl DCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } + pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } + pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); + (r, o.pos, i.pos) + } +} +impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } + +pub fn compress(src: &[u8], level: i32) -> Vec { + let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; + let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); + dst.truncate(n); + dst +} +pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) +} diff --git a/src/zstd/wip/work/ldm/harness/src/data.rs b/src/zstd/wip/work/ldm/harness/src/data.rs new file mode 100644 index 000000000000..c48a645e1b5d --- /dev/null +++ b/src/zstd/wip/work/ldm/harness/src/data.rs @@ -0,0 +1,65 @@ +//! Deterministic inputs. +pub struct Rng(pub u64); +impl Rng { + pub fn next(&mut self) -> u64 { + // splitmix64 + self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + } + pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } + pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } + pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } + pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } + /// Skewed towards small values. + pub fn size(&mut self, max: usize) -> usize { + let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); + (self.next() as usize & ((1usize << bits) - 1)).min(max) + } +} + +static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; + +/// One buffer of about `len` bytes, of a kind picked by `rng`. +pub fn data(rng: &mut Rng, len: usize) -> Vec { + let mut v = Vec::with_capacity(len + 64); + match rng.below(9) { + 0 => v.resize(len, rng.next() as u8), + 1 => while v.len() < len { v.push(rng.next() as u8) }, + 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, + 3 => { // few symbols, skewed + let n = rng.range(2, 40); + while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } + } + 4 => { // LZ-like: copies of earlier parts, at all distances + while v.len() < len { + if v.len() > 8 && rng.chance(3, 4) { + let dist = 1 + rng.size(v.len() - 1); + let n = 3 + rng.size(300); + for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } + } else { + for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } + } + } + } + 5 => { // periodic with noise + let period = rng.range(1, 300); + let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); + while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } + } + 6 => { // sections of different kinds + while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } + } + 7 => { // numbers as text + let mut x = rng.next() % 1000; + while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } + } + _ => { // long runs + while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } + } + } + v.truncate(len); + v +} diff --git a/src/zstd/wip/work/ldm/harness/src/ldm.rs b/src/zstd/wip/work/ldm/harness/src/ldm.rs new file mode 100644 index 000000000000..03581042d108 --- /dev/null +++ b/src/zstd/wip/work/ldm/harness/src/ldm.rs @@ -0,0 +1,452 @@ +//! Long distance matching and the block pre-splitter, against the C library. +use crate::c; +use crate::data::{Rng, data}; +use bun_zstd::testing as t; +use std::ffi::{c_int, c_void}; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct LdmParams { enable: u32, hash_log: u32, bucket_size_log: u32, min_match_length: u32, hash_rate_log: u32, window_log: u32 } +#[repr(C)] +#[derive(Clone, Copy, Debug)] +struct CParams { window_log: u32, chain_log: u32, hash_log: u32, search_log: u32, min_match: u32, target_length: u32, strategy: u32 } +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct RawSeq { offset: u32, lit_length: u32, match_length: u32 } +#[repr(C)] +struct RawSeqStore { seq: *mut RawSeq, pos: usize, pos_in_sequence: usize, size: usize, capacity: usize } +#[repr(C)] +#[derive(Clone, Copy)] +struct LdmEntry { offset: u32, checksum: u32 } +enum Shim {} + +unsafe extern "C" { + fn ZSTD_splitBlock(block: *const c_void, size: usize, level: c_int, wksp: *mut c_void, wksp_size: usize) -> usize; + fn ZSTD_ldm_adjustParameters(p: *mut LdmParams, cp: *const CParams); + fn ZSTD_ldm_getTableSize(p: LdmParams) -> usize; + fn ZSTD_ldm_getMaxNbSeq(p: LdmParams, max_chunk_size: usize) -> usize; + fn ZSTD_ldm_skipSequences(s: *mut RawSeqStore, src_size: usize, min_match: u32); + fn ZSTD_ldm_skipRawSeqStoreBytes(s: *mut RawSeqStore, nb_bytes: usize); + fn shim_ldm_new(p: *const u32, sizes: *const usize, capacity: usize, start_index: u32) -> *mut Shim; + fn shim_ldm_free(s: *mut Shim); + fn shim_ldm_update(s: *mut Shim, buf: c_int, pos: usize, data: *const c_void, len: usize); + fn shim_ldm_poke(s: *mut Shim, buf: c_int, pos: usize, byte: u8); + fn shim_ldm_load_dict(s: *mut Shim, buf: c_int, pos: usize, data: *const c_void, len: usize); + fn shim_ldm_generate(s: *mut Shim, buf: c_int, pos: usize, len: usize, reset: c_int) -> usize; + fn shim_ldm_seqs(s: *const Shim, nb: *mut usize) -> *const RawSeq; + fn shim_ldm_window(s: *const Shim, out: *mut u32); + fn shim_ldm_table(s: *const Shim) -> *const LdmEntry; + fn shim_ldm_bucket_offsets(s: *const Shim) -> *const u8; +} + +// ── the pre-splitter ────────────────────────────────────────────────────── + +pub fn presplit(seed: u64, iters: u64) { + const SIZE: usize = 128 << 10; + let mut results = std::collections::BTreeMap::<(i32, usize), u64>::new(); + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + let mut block = Vec::with_capacity(SIZE + 64); + match rng.below(4) { + 0 => block = data(&mut rng, SIZE), + 1 => { // sections, cut anywhere + while block.len() < SIZE { let n = 1 + rng.below(SIZE); block.extend(data(&mut rng, n)); } + } + 2 => { // sections, cut at chunk boundaries + while block.len() < SIZE { let n = (8 << 10) * rng.range(1, 16); let mut d = data(&mut rng, n); d.resize(n, 0); block.extend(d); } + } + _ => { // a slow drift of the distribution + let (a, b) = (rng.range(1, 255), rng.range(1, 255)); + let base = rng.next() as u8; + for k in 0..SIZE { let n = (a * (SIZE - k) + b * k) / SIZE; block.push(base.wrapping_add(rng.below(n.max(1)) as u8)); } + } + } + block.resize(SIZE, 0); + for level in 0..=4 { + let mut wksp = vec![0xA5A5_A5A5_A5A5_A5A5u64; 8208 / 8]; + let expected = unsafe { ZSTD_splitBlock(block.as_ptr().cast(), SIZE, level, wksp.as_mut_ptr().cast(), 8208) }; + let got = t::split_block(&block, level).unwrap_or_else(|e| panic!("seed {s} level {level}: {e:?}")); + assert_eq!(got, expected, "seed {s} level {level}"); + *results.entry((level, got >> 10)).or_default() += 1; + } + } + println!("presplit ok: {iters} blocks from seed {seed}, at 5 levels. (level, KB) -> count: {results:?}"); +} + +// ── parameters ──────────────────────────────────────────────────────────── + +fn arr(p: LdmParams) -> [u32; 6] { [p.enable, p.hash_log, p.bucket_size_log, p.min_match_length, p.hash_rate_log, p.window_log] } + +pub fn params(seed: u64, iters: u64) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + let zero = |rng: &mut Rng, lo: usize, hi: usize| if rng.chance(1, 3) { 0 } else { rng.range(lo, hi) as u32 }; + let p = LdmParams { + enable: rng.below(3) as u32, hash_log: zero(&mut rng, 6, 30), bucket_size_log: zero(&mut rng, 1, 8), + min_match_length: zero(&mut rng, 4, 4096), hash_rate_log: zero(&mut rng, 0, 25), window_log: rng.below(32) as u32, + }; + let cp = CParams { + window_log: rng.range(10, 31) as u32, chain_log: rng.range(6, 30) as u32, hash_log: rng.range(6, 30) as u32, search_log: rng.range(1, 30) as u32, + min_match: rng.range(3, 7) as u32, target_length: rng.below(131073) as u32, strategy: rng.range(1, 9) as u32, + }; + let mut expected = p; + unsafe { ZSTD_ldm_adjustParameters(&mut expected, &cp) }; + let got = t::ldm_adjust_parameters(arr(p), [cp.window_log, cp.chain_log, cp.hash_log, cp.search_log, cp.min_match, cp.target_length, cp.strategy]); + assert_eq!(got, arr(expected), "seed {s}: {p:?} {cp:?}"); + for q in [expected, LdmParams { min_match_length: p.min_match_length.max(1), ..p }] { + assert_eq!(t::ldm_get_table_size(arr(q)), unsafe { ZSTD_ldm_getTableSize(q) }, "seed {s}: {q:?}"); + let n = rng.next() as usize >> rng.range(1, 63); + assert_eq!(t::ldm_get_max_nb_seq(arr(q), n), unsafe { ZSTD_ldm_getMaxNbSeq(q, n) }, "seed {s}: {q:?} {n}"); + } + } + println!("ldm params ok: {iters} iterations from seed {seed}"); +} + +// ── moving in a store of sequences ──────────────────────────────────────── + +pub fn skip(seed: u64, iters: u64) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + let capacity = rng.below(12); + let size = rng.below(capacity + 1); + let max = *rng.pick(&[3usize, 20, 300, 70_000, 5_000_000]); + let seq: Vec = (0..capacity).map(|_| (1 + rng.size(1 << 27) as u32, rng.size(max) as u32, rng.size(max) as u32)).collect(); + let mut store = t::RawStore { seq, pos: rng.below(size + 1), pos_in_sequence: 0, size }; + for step in 0..rng.range(1, 12) { + let mut c_seq: Vec = store.seq.iter().map(|q| RawSeq { offset: q.0, lit_length: q.1, match_length: q.2 }).collect(); + let mut c_store = RawSeqStore { seq: c_seq.as_mut_ptr(), pos: store.pos, pos_in_sequence: store.pos_in_sequence, size, capacity }; + let n = rng.size(max * 3); + let before = store.clone(); + let which = rng.chance(1, 2); + let min_match = rng.range(3, 7) as u32; + if which { + unsafe { ZSTD_ldm_skipSequences(&mut c_store, n, min_match) }; + t::ldm_skip_sequences(&mut store, n, min_match).unwrap(); + } else { + unsafe { ZSTD_ldm_skipRawSeqStoreBytes(&mut c_store, n) }; + t::ldm_skip_raw_seq_store_bytes(&mut store, n).unwrap(); + } + let expected = t::RawStore { + pos: c_store.pos, pos_in_sequence: c_store.pos_in_sequence, size: c_store.size, + seq: c_seq.iter().map(|q| (q.offset, q.lit_length, q.match_length)).collect(), + }; + assert_eq!(store, expected, "seed {s} step {step}: skip{} {n} min_match {min_match} from {before:?}", if which { "Sequences" } else { "RawSeqStoreBytes" }); + } + } + println!("ldm skip ok: {iters} iterations from seed {seed}"); +} + +// ── finding the sequences ───────────────────────────────────────────────── + +/// Content with repeats of all lengths at all distances. +pub fn long_repeats(rng: &mut Rng, len: usize) -> Vec { + let mut v: Vec = Vec::with_capacity(len + 64); + let max_copy = *rng.pick(&[200usize, 5000, 100_000, 2_000_000]); + let max_fresh = *rng.pick(&[100usize, 5000, 100_000, 1_000_000]); + let noise = rng.chance(1, 2); + let copies = rng.range(1, 7); + while v.len() < len { + if v.len() > 64 && rng.chance(copies, 8) { + let n = (1 + rng.size(max_copy)).min(v.len()); + let from = if rng.chance(1, 2) { rng.below(v.len() - n + 1) } else { v.len() - n - rng.size(v.len() - n) }; + v.extend_from_within(from..from + n); + if rng.chance(1, 3) { + for _ in 0..rng.range(1, 4) { let at = v.len() - 1 - rng.below(n); v[at] ^= 1 + (rng.next() as u8 & 0x7f); } + } + } else { + let n = 1 + rng.size(max_fresh); + if noise { for _ in 0..n.div_ceil(8) { v.extend_from_slice(&rng.next().to_le_bytes()); } } else { v.extend(data(rng, n)); } + } + } + v.truncate(len); + v +} + +#[derive(Default, Debug)] +pub struct Stats { scenarios: u64, bytes: u64, calls: u64, seqs: u64, calls_ext_dict: u64, seqs_ext_dict: u64, corrections: u64, errors: u64, dicts: u64, full: u64 } + +struct Pair { c: *mut Shim, r: t::Ldm, ctx: String } +impl Drop for Pair { fn drop(&mut self) { unsafe { shim_ldm_free(self.c) } } } +impl Pair { + fn c_window(&self) -> [u32; 5] { let mut w = [0u32; 5]; unsafe { shim_ldm_window(self.c, w.as_mut_ptr()) }; w } + fn check_window(&self, at: &str) { assert_eq!(self.r.window(), self.c_window(), "{}: window after {at}", self.ctx); } + fn update(&mut self, buf: usize, pos: usize, d: &[u8]) { + unsafe { shim_ldm_update(self.c, buf as c_int, pos, d.as_ptr().cast(), d.len()) }; + self.r.update(buf, pos, d); + self.check_window("update"); + } + fn poke(&mut self, buf: usize, pos: usize, byte: u8) { + unsafe { shim_ldm_poke(self.c, buf as c_int, pos, byte) }; + self.r.poke(buf, pos, byte); + } + fn load_dict(&mut self, buf: usize, pos: usize, d: &[u8]) { + unsafe { shim_ldm_load_dict(self.c, buf as c_int, pos, d.as_ptr().cast(), d.len()) }; + self.r.load_dict(buf, pos, d).unwrap_or_else(|e| panic!("{}: load_dict {e:?}", self.ctx)); + self.check_window("load_dict"); + } + /// Whether it went without an error. + fn generate(&mut self, buf: usize, pos: usize, len: usize, reset: bool, stats: &mut Stats) -> bool { + let before = self.c_window(); + let expected = c::res(unsafe { shim_ldm_generate(self.c, buf as c_int, pos, len, reset as c_int) }).map(|_| ()); + let got = self.r.generate(pos, len, reset).map_err(|e| e as u32); + let at = format!("generate(buf {buf}, pos {pos}, len {len}, reset {reset}), window before {before:?}"); + assert_eq!(got, expected, "{}: result of {at}", self.ctx); + let mut nb = 0usize; + let c_seqs = unsafe { let p = shim_ldm_seqs(self.c, &mut nb); std::slice::from_raw_parts(p, nb) }; + let r_seqs = self.r.seqs(); + let same = r_seqs.len() == nb && r_seqs.iter().zip(c_seqs).all(|(a, b)| *a == (b.offset, b.lit_length, b.match_length)); + if !same { + let k = r_seqs.iter().zip(c_seqs).position(|(a, b)| *a != (b.offset, b.lit_length, b.match_length)).unwrap_or(nb.min(r_seqs.len())); + panic!("{}: {at}: sequence {k} differs: got {:?}, expected {:?} ({} vs {nb} sequences)", self.ctx, r_seqs.get(k), c_seqs.get(k), r_seqs.len()); + } + self.check_window(&at); + let after = self.c_window(); + stats.calls += 1; + stats.bytes += len as u64; + if reset { stats.seqs += nb as u64; } + if after[1] < after[0] { stats.calls_ext_dict += 1; if reset { stats.seqs_ext_dict += nb as u64; } } + stats.corrections += u64::from(after[4] - before[4]); + if expected.is_err() { stats.errors += 1; } + expected.is_ok() + } + fn check_tables(&self, p: &LdmParams) { + let table = unsafe { std::slice::from_raw_parts(shim_ldm_table(self.c), 1 << p.hash_log) }; + for (k, (a, b)) in self.r.table().zip(table).enumerate() { + assert_eq!(a, (b.offset, b.checksum), "{}: entry {k} of the hash table", self.ctx); + } + let offsets = unsafe { std::slice::from_raw_parts(shim_ldm_bucket_offsets(self.c), 1 << (p.hash_log - p.bucket_size_log)) }; + assert!(self.r.bucket_offsets() == offsets, "{}: bucket offsets", self.ctx); + } +} + +fn scenario(s: u64, max_len: usize, stats: &mut Stats) { + let mut rng = Rng(s); + let mut p = LdmParams { + enable: 1, + hash_log: if rng.chance(1, 12) { rng.range(17, 24) } else { rng.range(6, 16) } as u32, + bucket_size_log: rng.range(1, 8) as u32, + min_match_length: match rng.below(4) { 0 => rng.range(4, 12), 1 => *rng.pick(&[16usize, 32, 63, 64, 65, 128]), 2 => rng.range(4, 300), _ => rng.range(4, 4096) } as u32, + hash_rate_log: if rng.chance(1, 8) { rng.range(0, 25) } else { rng.range(0, 10) } as u32, + window_log: 0, + }; + let cp = CParams { window_log: rng.range(10, 27) as u32, chain_log: 6, hash_log: 6, search_log: 1, min_match: 4, target_length: 0, strategy: rng.range(1, 9) as u32 }; + if rng.chance(1, 6) { p.hash_log = 0; p.hash_rate_log = 0; } + if rng.chance(1, 6) { p.hash_rate_log = 0; } + if rng.chance(1, 6) { p.bucket_size_log = 0; } + if rng.chance(1, 6) { p.min_match_length = 0; } + unsafe { ZSTD_ldm_adjustParameters(&mut p, &cp) }; + if p.hash_log > 24 { p.hash_log = 24; } + + let mut len = match rng.below(5) { 0 => rng.size(5000), 1 => rng.size(300_000), 2 => rng.size(max_len), _ => rng.range(max_len / 4, max_len) } + 1; + while (len as u64 * u64::from(p.min_match_length)) >> p.hash_rate_log > 100 << 20 { len /= 2; } + let content = long_repeats(&mut rng, len); + + let layout = rng.below(5); + let dict_len = if layout == 1 || layout == 2 || rng.chance(1, 8) { rng.size(len / 2) } else { 0 }; + let ring = if layout == 4 { (rng.size(len) + 1).max(16) } else { 0 }; + let room = len + 64; + let sizes = if layout == 4 { [ring, 0, dict_len] } else { [room, room, room] }; + let start_index = match rng.below(8) { + 0 => (3500u32 << 20).wrapping_sub(rng.below(len + (2 << 20)) as u32).wrapping_add(rng.below(1 << 20) as u32), + 1 => 2 + rng.size(3400 << 20) as u32, + _ => 0, + }; + let reset = rng.chance(3, 4); + let capacity = if rng.chance(1, 10) { rng.below(6) } else { len / p.min_match_length as usize + 1 }; + let max_update = *rng.pick(&[1000usize, 128 << 10, 1 << 20, 5 << 20, usize::MAX >> 1]); + let pokes = if rng.chance(1, 8) { rng.range(1, 200) } else { 0 }; + let piece = *rng.pick(&[0usize, 128 << 10, 128 << 10, 1]); + let ctx = format!("seed {s}: {p:?} len {len} layout {layout} dict {dict_len} ring {ring} start_index {start_index} reset {reset} capacity {capacity} pokes {pokes}"); + let pa = arr(p); + let mut pair = Pair { c: unsafe { shim_ldm_new(pa.as_ptr(), sizes.as_ptr(), capacity, start_index) }, r: t::Ldm::new(pa, sizes, capacity, start_index), ctx }; + stats.scenarios += 1; + + let (mut buf, mut pos) = (if layout == 4 { 0 } else { 1 }, 0usize); + let mut done = 0usize; + if dict_len > 0 { + stats.dicts += 1; + if layout == 2 { pair.load_dict(buf, 0, &content[..dict_len]); pos = dict_len; } else { pair.load_dict(2, 0, &content[..dict_len]); } + done = dict_len; + } + 'all: while done < len { + let mut n = (1 + rng.size(max_update.min(len - done) - 1)).min(len - done); + match layout { + 3 if rng.chance(1, 3) => { buf = rng.below(3); pos = rng.below(room - n + 1); } + 4 => { n = n.min(ring); if pos + n > ring { pos = 0; } } + _ => {} + } + if pos + n > sizes[buf] { pos = 0; } + pair.update(buf, pos, &content[done..done + n]); + let end = pos + n; + for _ in 0..pokes { + let b = rng.below(3); + if sizes[b] > 0 { pair.poke(b, rng.below(sizes[b]), rng.next() as u8); } + } + while pos < end { + let m = match piece { 0 => end - pos, 1 => 1 + rng.size(end - pos - 1), k => k.min(end - pos) }; + if !pair.generate(buf, pos, m, reset, stats) { break 'all; } + pos += m; + } + done += n; + } + pair.check_tables(&p); +} + +pub fn run(seed: u64, iters: u64, max_len: usize) { + let mut stats = Stats::default(); + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + if i % 200 == 0 { eprintln!("ldm iter {i} (seed {s})"); } + scenario(s, max_len, &mut stats); + } + println!("ldm ok: {iters} scenarios from seed {seed}, up to {max_len} bytes: {stats:?}"); +} + +pub fn bench(len: usize) { + let mut rng = Rng(42); + for (name, content) in [("repeats", long_repeats(&mut rng, len)), ("text", { let mut v = Vec::new(); while v.len() < len { v.extend(data(&mut Rng(2 + v.len() as u64 % 3), 300_000)); } v.truncate(len); v })] { + let mut p = LdmParams { enable: 1, hash_log: 0, bucket_size_log: 0, min_match_length: 0, hash_rate_log: 0, window_log: 0 }; + let cp = CParams { window_log: 27, chain_log: 6, hash_log: 6, search_log: 1, min_match: 4, target_length: 0, strategy: 2 }; + unsafe { ZSTD_ldm_adjustParameters(&mut p, &cp) }; + let pa = arr(p); + let sizes = [0, len, 0]; + let capacity = len / 64 + 1; + for _ in 0..3 { + let c = unsafe { shim_ldm_new(pa.as_ptr(), sizes.as_ptr(), capacity, 0) }; + let mut r = t::Ldm::new(pa, sizes, capacity, 0); + unsafe { shim_ldm_update(c, 1, 0, content.as_ptr().cast(), len) }; + r.update(1, 0, &content); + let t0 = std::time::Instant::now(); + unsafe { shim_ldm_generate(c, 1, 0, len, 1) }; + let t1 = std::time::Instant::now(); + r.generate(0, len, true).unwrap(); + let t2 = std::time::Instant::now(); + println!("{name}: {p:?}: C {:.1} MB/s, Rust {:.1} MB/s, {} sequences", len as f64 / 1e6 / (t1 - t0).as_secs_f64(), len as f64 / 1e6 / (t2 - t1).as_secs_f64(), r.seqs().len()); + unsafe { shim_ldm_free(c) }; + } + } + let block = data(&mut Rng(7), 128 << 10); + let block = &block[..128 << 10]; + for level in 0..=4 { + let mut wksp = vec![0u64; 8208 / 8]; + let n = 2000; + let t0 = std::time::Instant::now(); + let mut a = 0; + for _ in 0..n { a += unsafe { ZSTD_splitBlock(std::hint::black_box(block).as_ptr().cast(), block.len(), level, wksp.as_mut_ptr().cast(), 8208) }; } + let t1 = std::time::Instant::now(); + let mut b = 0; + for _ in 0..n { b += t::split_block(std::hint::black_box(block), level).unwrap(); } + let t2 = std::time::Instant::now(); + println!("split level {level}: C {:.2} us, Rust {:.2} us ({a} {b})", (t1 - t0).as_secs_f64() * 1e6 / n as f64, (t2 - t1).as_secs_f64() * 1e6 / n as f64); + } +} + +// ── ZSTD_ldm_blockCompress ──────────────────────────────────────────────── + +#[repr(C)] +#[derive(Clone, Copy)] +struct SeqDef { off_base: u32, lit_length: u16, ml_base: u16 } +enum ShimBlk {} +unsafe extern "C" { + fn shim_blk_new(cp: *mut u32, use_row: c_int, dict: *const c_void, dict_size: usize, src: *const c_void, src_size: usize, seqs: *const RawSeq, nb: usize) -> *mut ShimBlk; + fn shim_blk_free(s: *mut ShimBlk); + fn shim_blk_max_block(s: *const ShimBlk) -> usize; + fn shim_blk_run(s: *mut ShimBlk, pos: usize, block_size: usize) -> usize; + fn shim_blk_seqs(s: *const ShimBlk, nb: *mut usize) -> *const SeqDef; + fn shim_blk_literals(s: *const ShimBlk, nb: *mut usize) -> *const u8; + fn shim_blk_state(s: *const ShimBlk, out: *mut u32); + fn shim_blk_store(s: *const ShimBlk) -> *const RawSeqStore; +} + +unsafe fn slice_of<'a, T>(p: *const T, n: usize) -> &'a [T] { if n == 0 { &[] } else { unsafe { std::slice::from_raw_parts(p, n) } } } + +pub fn blocks(seed: u64, iters: u64, strategies: &[u32], max_len: usize) { + let (mut nb_blocks, mut nb_seqs, mut nb_raw, mut nb_dict) = (0u64, 0u64, 0u64, 0u64); + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if i % 200 == 0 { eprintln!("ldmblocks iter {i} (seed {s})"); } + let len = match rng.below(3) { 0 => rng.size(20_000), 1 => rng.size(max_len), _ => rng.range(max_len / 4, max_len) } + 1; + let dict_len = if rng.chance(1, 3) { 8 + rng.size(len) } else { 0 }; // less than 8 bytes are not loaded + let content = long_repeats(&mut rng, dict_len + len); + let (dict, src) = content.split_at(dict_len); + let mut cp = [ + rng.range(10, 24) as u32, rng.range(6, 18) as u32, rng.range(6, 18) as u32, rng.range(1, 7) as u32, rng.range(3, 7) as u32, + *rng.pick(&[0u32, 1, 2, 8, 32, 64, 256, 999]), *rng.pick(strategies), + ]; + let use_row = rng.chance(1, 2); + + // The long distance matches of the whole input, or made-up sequences. + let raw: Vec = if rng.chance(2, 3) { + let mut p = LdmParams { + enable: 1, hash_log: rng.range(6, 16) as u32, bucket_size_log: rng.range(1, 6) as u32, + min_match_length: *rng.pick(&[4u32, 8, 16, 32, 64, 200]), hash_rate_log: rng.range(0, 7) as u32, window_log: 0, + }; + let c = CParams { window_log: cp[0], chain_log: 6, hash_log: 6, search_log: 1, min_match: 4, target_length: 0, strategy: cp[6] }; + unsafe { ZSTD_ldm_adjustParameters(&mut p, &c) }; + let mut l = t::Ldm::new(arr(p), [0, len, dict_len], len / p.min_match_length as usize + 1, 0); + if dict_len > 0 { l.load_dict(2, 0, dict).unwrap(); } + l.update(1, 0, src); + l.generate(0, len, true).unwrap(); + l.seqs() + } else { + let (max_ll, max_ml) = (*rng.pick(&[0usize, 10, 1000, 70_000, 300_000]), *rng.pick(&[10usize, 1000, 70_000, 300_000])); + let mut v = Vec::new(); + let mut at = 0usize; + let stop = rng.below(len + len / 4 + 1); + while at < stop { + let (ll, ml) = (rng.size(max_ll), 4 + rng.size(max_ml)); + at += ll + ml; + let beyond = rng.below(2); + v.push((1 + rng.size((at - ml + dict_len).max(1).min(1 << cp[0]) - 1 + beyond) as u32, ll as u32, ml as u32)); + } + v + }; + let c_raw: Vec = raw.iter().map(|q| RawSeq { offset: q.0, lit_length: q.1, match_length: q.2 }).collect(); + + let c = unsafe { shim_blk_new(cp.as_mut_ptr(), use_row as c_int, dict.as_ptr().cast(), dict.len(), src.as_ptr().cast(), src.len(), c_raw.as_ptr(), c_raw.len()) }; + let ctx = format!("seed {s}: cp {cp:?} row {use_row} dict {dict_len} src {len} raw sequences {}", raw.len()); + let mut r = t::LdmBlocks::new(cp, use_row, dict, src, &raw).unwrap_or_else(|e| panic!("{ctx}: {e:?}")); + let max_block = unsafe { shim_blk_max_block(c) }; + let full_blocks = rng.chance(1, 2); + let mut pos = 0usize; + nb_raw += raw.len() as u64; + nb_dict += u64::from(dict_len > 0); + while pos < len { + let n = if full_blocks { max_block } else { 1 + rng.size(max_block - 1) }.min(len - pos); + if std::env::var_os("LDM_TRACE").is_some() { eprintln!("{ctx}: block at {pos} of {n} (max {max_block})"); } + let last_ll = c::res(unsafe { shim_blk_run(c, pos, n) }).unwrap_or_else(|e| panic!("{ctx}: C error {e}")); + let got = r.run(pos, n).unwrap_or_else(|e| panic!("{ctx}: block at {pos} of {n}: {e:?}")); + let expected = unsafe { + let mut nb = 0usize; + let seqs = slice_of(shim_blk_seqs(c, &mut nb), nb).iter().map(|q| (q.off_base, q.lit_length, q.ml_base)).collect(); + let literals = slice_of(shim_blk_literals(c, &mut nb), nb).to_vec(); + let mut st = [0u32; 6]; + shim_blk_state(c, st.as_mut_ptr()); + let store = &*shim_blk_store(c); + t::BlockOut { + seqs, literals, last_ll, long_length: if st[0] == 0 { None } else { Some((st[0], st[1])) }, rep: [st[2], st[3], st[4]], next_to_update: st[5], + store: t::RawStore { + seq: slice_of(store.seq, store.capacity).iter().map(|q| (q.offset, q.lit_length, q.match_length)).collect(), + pos: store.pos, pos_in_sequence: store.pos_in_sequence, size: store.size, + }, + } + }; + if got != expected { + let k = got.seqs.iter().zip(&expected.seqs).position(|(a, b)| a != b).unwrap_or(got.seqs.len().min(expected.seqs.len())); + panic!("{ctx}: block at {pos} of {n}: sequence {k}: got {:?}, expected {:?} ({} vs {}); last_ll {} vs {}; rep {:?} vs {:?}; long {:?} vs {:?}; next_to_update {} vs {}; literals same {}; store same {} (pos {} vs {}, in sequence {} vs {})", + got.seqs.get(k), expected.seqs.get(k), got.seqs.len(), expected.seqs.len(), got.last_ll, expected.last_ll, got.rep, expected.rep, + got.long_length, expected.long_length, got.next_to_update, expected.next_to_update, got.literals == expected.literals, got.store == expected.store, + got.store.pos, expected.store.pos, got.store.pos_in_sequence, expected.store.pos_in_sequence); + } + nb_blocks += 1; + nb_seqs += got.seqs.len() as u64; + pos += n; + } + unsafe { shim_blk_free(c) }; + } + println!("ldm blocks ok: {iters} inputs from seed {seed}, strategies {strategies:?}, up to {max_len} bytes: {nb_blocks} blocks, {nb_seqs} sequences, {nb_raw} raw sequences, {nb_dict} with a prefix"); +} diff --git a/src/zstd/wip/work/ldm/harness/src/main.rs b/src/zstd/wip/work/ldm/harness/src/main.rs new file mode 100644 index 000000000000..8eaf738235fb --- /dev/null +++ b/src/zstd/wip/work/ldm/harness/src/main.rs @@ -0,0 +1,30 @@ +mod c; +mod data; +mod ldm; +mod seqs; + +fn main() { + let args: Vec = std::env::args().collect(); + let cmd = args.get(1).map(String::as_str).unwrap_or(""); + let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); + let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); + match cmd { + // seqs + "seqs" => { + let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); + seqs::run(seed, iters, &strategies, args.get(5).is_some_and(|s| s == "1")); + } + "presplit" => ldm::presplit(seed, iters), + "ldmparams" => ldm::params(seed, iters), + "ldmskip" => ldm::skip(seed, iters), + // ldm + "ldm" => ldm::run(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(4 << 20)), + // ldmblocks + "ldmblocks" => { + let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); + ldm::blocks(seed, iters, &strategies, args.get(5).and_then(|s| s.parse().ok()).unwrap_or(1 << 20)); + } + "ldmbench" => ldm::bench(seed as usize), + _ => eprintln!("unknown command"), + } +} diff --git a/src/zstd/wip/work/ldm/harness/src/seqs.rs b/src/zstd/wip/work/ldm/harness/src/seqs.rs new file mode 100644 index 000000000000..7c03d3b0f952 --- /dev/null +++ b/src/zstd/wip/work/ldm/harness/src/seqs.rs @@ -0,0 +1,73 @@ +//! One block through a Rust block compressor and through the C library (`ZSTD_generateSequences`). +use crate::c; +use crate::data::{Rng, data}; +use std::ffi::{c_uint, c_void}; + +#[repr(C)] +#[derive(Clone, Copy, Default, Debug)] +struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } +#[repr(C)] +#[derive(Clone, Copy, Debug)] +struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } +unsafe extern "C" { + fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; + fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; + fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; +} + +fn c_sequences(cp: [u32; 7], use_row: bool, dict: &[u8], src: &[u8]) -> Vec<(u32, u32, u32)> { + let mut cctx = c::CCtx::new(); + for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } + cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder + cctx.set(1010, 2).unwrap(); // no block splitter + if !dict.is_empty() { cctx.ref_prefix(dict).unwrap(); } + let mut out = vec![Sequence::default(); src.len() / 3 + 8]; + let n = c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }).unwrap(); + out[..n].iter().map(|s| (s.lit_length, s.match_length, s.offset)).collect() +} + +/// `strategies`: which of 1..=9 to test. `with_dict`: also test with a prefix, which is the extDict variants. +pub fn run(seed: u64, iters: u64, strategies: &[u32], with_dict: bool) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if i % 500 == 0 { eprintln!("seqs iter {i} (seed {s})"); } + let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071]); + let len = rng.range(7, max); + let mut src = data(&mut rng, len); + let dict = if with_dict && rng.chance(1, 2) { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; + if !dict.is_empty() && rng.chance(2, 3) { + for _ in 0..rng.range(1, 6) { + let n = rng.size(dict.len().min(src.len())); + let at = rng.below(dict.len() - n + 1); + let to = rng.below(src.len() - n + 1); + src[to..to + n].copy_from_slice(&dict[at..at + n]); + } + } + let strategy = *rng.pick(strategies); + let p = if rng.chance(1, 3) { + // the parameters of a level, with the strategy forced + let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; + p.strategy = strategy; + p + } else { + CParams { + window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, + search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, + target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, + } + }; + let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; + let cp = [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy]; + let use_row = rng.chance(1, 2); + let expected = c_sequences(cp, use_row, &dict, &src); + let got = bun_zstd::testing::first_block(cp, use_row, &dict, &src); + let got = got.unwrap_or_else(|e| panic!("seed {s}: error {e:?} (cp {cp:?} row {use_row} dict {} src {})", dict.len(), src.len())); + if got != expected { + let at = got.iter().zip(&expected).position(|(a, b)| a != b).unwrap_or(got.len().min(expected.len())); + panic!("seed {s}: cp {cp:?} row {use_row} dict {} src {}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", + dict.len(), src.len(), got.get(at), expected.get(at), got.len(), expected.len()); + } + } + println!("seqs ok: {iters} iterations from seed {seed}, strategies {strategies:?}, dict {with_dict}"); +} diff --git a/src/zstd/wip/work/ldm/snap/double_fast.rs b/src/zstd/wip/work/ldm/snap/double_fast.rs new file mode 100644 index 000000000000..fbf2d4120cad --- /dev/null +++ b/src/zstd/wip/work/ldm/snap/double_fast.rs @@ -0,0 +1,785 @@ +//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. + +use super::fast::{FAST_HASH_FILL_STEP, Located, count_backwards, restore_rep}; +use super::internal::{ + BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, + write_tagged_index, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, read_le32, read_le64}; + +/// `ZSTD_fillDoubleHashTableForCDict` +fn fill_double_hash_table_for_cdict( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits_l = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let h_bits_s = ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; + let mls = ms.c_params.min_match; + let hash_large = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + // The positions in between only go in the table of long hashes, where there is nothing yet. + while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { + let curr = prefix.index_of(ip); + for i in 0..FAST_HASH_FILL_STEP { + let sm_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; + let lg_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; + if i == 0 { + write_tagged_index(hash_small, sm_hash_and_tag, curr + i)?; + } + if i == 0 || hash_large.at(lg_hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { + write_tagged_index(hash_large, lg_hash_and_tag, curr + i)?; + } + if dtlm == DictTableLoadMethod::Fast { + break; + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillDoubleHashTableForCCtx` +fn fill_double_hash_table_for_cctx( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let mls = ms.c_params.min_match; + let hash_large = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + // The positions in between only go in the table of long hashes, where there is nothing yet. + while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { + let curr = prefix.index_of(ip); + for i in 0..FAST_HASH_FILL_STEP { + let sm_hash = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; + let lg_hash = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; + if i == 0 { + hash_small.put(sm_hash, curr + i)?; + } + if i == 0 || hash_large.at(lg_hash)? == 0 { + hash_large.put(lg_hash, curr + i)?; + } + if dtlm == DictTableLoadMethod::Fast { + break; + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_double_hash_table( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, + tfp: TableFillPurpose, +) -> Result<()> { + match tfp { + TableFillPurpose::ForCDict => fill_double_hash_table_for_cdict(ms, prefix, end, dtlm), + TableFillPurpose::ForCCtx => fill_double_hash_table_for_cctx(ms, prefix, end, dtlm), + } +} + +/// `ZSTD_compressBlock_doubleFast_noDict_generic` +fn compress_block_double_fast_no_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; + + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let window_log = ms.c_params.window_log; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, window_log); + let prefix_lowest = src.pos_of(prefix_lowest_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip = istart + usize::from(istart == prefix_lowest); + let [mut offset_1, mut offset_2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let current = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(current, window_log); + let max_rep = current.wrapping_sub(window_low); + if offset_2 > max_rep { + offset_saved2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved1 = offset_1; + offset_1 = 0; + } + } + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + // One turn per match. + 'start: loop { + let mut step = 1usize; + let mut next_step = ip + K_STEP_INCR; + let mut ip1 = ip + step; + if ip1 > ilimit { + break 'start; + } + + let mut hl0 = hash_ptr(buf, ip, h_bits_l, 8)?; + let mut idxl0 = hash_long.at(hl0)?; + let mut curr; + + let m_length = 'match_stored: { + // One turn per position. Ends with the offset and the length of a match, which `ip` is moved to the start + // of. + let (offset, m_length, hl1) = loop { + let hs0 = hash_ptr(buf, ip, h_bits_s, MLS)?; + let idxs0 = hash_small.at(hs0)?; + curr = src.index_of(ip); + hash_long.put(hl0, curr)?; + hash_small.put(hs0, curr)?; + + if (offset_1 > 0) + & (read_le32(buf, (ip + 1).wrapping_sub(offset_1 as usize))? + == read_le32(buf, ip + 1)?) + { + let m_length = count( + buf, + ip + 1 + 4, + buf, + (ip + 1 + 4).wrapping_sub(offset_1 as usize), + iend, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break 'match_stored m_length; + } + + let hl1 = hash_ptr(buf, ip1, h_bits_l, 8)?; + + let matchl0 = src.pos_of(idxl0); + if idxl0 >= prefix_lowest_index && read_le64(buf, matchl0)? == read_le64(buf, ip)? { + let m_length = count(buf, ip + 8, buf, matchl0 + 8, iend)? + 8; + let offset = ip.wrapping_sub(matchl0) as u32; + let back = count_backwards(buf, ip, anchor, buf, matchl0, prefix_lowest); + ip -= back; + break (offset, m_length + back, hl1); + } + + let idxl1 = hash_long.at(hl1)?; + + let mut matchs0 = src.pos_of(idxs0); + if idxs0 >= prefix_lowest_index && read_le32(buf, matchs0)? == read_le32(buf, ip)? { + // A short match: a longer one at the next position is preferred. + let mut m_length = count(buf, ip + 4, buf, matchs0 + 4, iend)? + 4; + let mut offset = ip.wrapping_sub(matchs0) as u32; + let matchl1 = src.pos_of(idxl1); + if idxl1 > prefix_lowest_index + && read_le64(buf, matchl1)? == read_le64(buf, ip1)? + { + let l1len = count(buf, ip1 + 8, buf, matchl1 + 8, iend)? + 8; + if l1len > m_length { + ip = ip1; + m_length = l1len; + offset = ip.wrapping_sub(matchl1) as u32; + matchs0 = matchl1; + } + } + let back = count_backwards(buf, ip, anchor, buf, matchs0, prefix_lowest); + ip -= back; + break (offset, m_length + back, hl1); + } + + if ip1 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + ip = ip1; + ip1 += step; + + hl0 = hl1; + idxl0 = idxl1; + if ip1 > ilimit { + break 'start; + } + }; + + offset_2 = offset_1; + offset_1 = offset; + // With a larger step, ip1 may be past the end of the match. + if step < 4 { + hash_long.put(hl1, src.index_of(ip1))?; + } + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + m_length + }; + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit + && (offset_2 > 0) + & (read_le32(buf, ip)? == read_le32(buf, ip.wrapping_sub(offset_2 as usize))?) + { + let r_length = count( + buf, + ip + 4, + buf, + (ip + 4).wrapping_sub(offset_2 as usize), + iend, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, src.index_of(ip))?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, src.index_of(ip))?; + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; + ip += r_length; + anchor = ip; + } + } + } + + restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` +fn compress_block_double_fast_dict_match_state_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, ms.c_params.window_log); + let prefix_lowest = src.pos_of(prefix_lowest_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + let [mut offset_1, mut offset_2, _] = *rep; + + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_hash_long = dms.ms.hash_table.as_slice(); + let dict_hash_small = dms.ms.chain_table.as_slice(); + let dict_start_index = dms.ms.window.dict_limit; + let dict_start = dms.pos_of(dict_start_index); + let dict_end = dms.end; + // What to subtract from an index below prefix_lowest_index to get the index in the dictionary. + let dict_index_delta = prefix_lowest_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); + let dict_h_bits_l = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let dict_h_bits_s = dms.ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; + let dict_and_prefix_length = istart + .wrapping_sub(prefix_lowest) + .wrapping_add(dict_end.wrapping_sub(dict_start)) as u32; + + let locate = |index: u32| { + if index < prefix_lowest_index { + Located { + buf: dms.buf, + pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_lowest, + end: iend, + } + } + }; + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip = istart + usize::from(dict_and_prefix_length == 0); + + 'search: while ip < ilimit { + let h2 = hash_ptr(buf, ip, h_bits_l, 8)?; + let h = hash_ptr(buf, ip, h_bits_s, MLS)?; + let dict_hash_and_tag_l = hash_ptr(buf, ip, dict_h_bits_l, 8)?; + let dict_hash_and_tag_s = hash_ptr(buf, ip, dict_h_bits_s, MLS)?; + let dict_match_index_and_tag_l = + dict_hash_long.at(dict_hash_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_match_index_and_tag_s = + dict_hash_small.at(dict_hash_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_tags_match_l = + compare_packed_tags(dict_match_index_and_tag_l as usize, dict_hash_and_tag_l); + let dict_tags_match_s = + compare_packed_tags(dict_match_index_and_tag_s as usize, dict_hash_and_tag_s); + let curr = src.index_of(ip); + let match_index_l = hash_long.at(h2)?; + let mut match_index_s = hash_small.at(h)?; + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + hash_long.put(h2, curr)?; + hash_small.put(h, curr)?; + + let m_length = 'match_stored: { + if index_overlap_check(prefix_lowest_index, rep_index) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? + { + let m_length = count_2segments( + buf, + ip + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_lowest, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break 'match_stored m_length; + } + + // The offset and the length of a match, which `ip` is moved to the start of. + let (offset, m_length) = 'match_found: { + let match_long = src.pos_of(match_index_l); + if match_index_l >= prefix_lowest_index + && read_le64(buf, match_long)? == read_le64(buf, ip)? + { + let m_length = count(buf, ip + 8, buf, match_long + 8, iend)? + 8; + let offset = ip.wrapping_sub(match_long) as u32; + let back = count_backwards(buf, ip, anchor, buf, match_long, prefix_lowest); + ip -= back; + break 'match_found (offset, m_length + back); + } else if dict_tags_match_l { + let dict_match_index_l = + dict_match_index_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match_l = dms.pos_of(dict_match_index_l); + if dict_match_index_l > dict_start_index + && read_le64(dms.buf, dict_match_l)? == read_le64(buf, ip)? + { + let m_length = count_2segments( + buf, + ip + 8, + dms.buf, + dict_match_l + 8, + iend, + dict_end, + prefix_lowest, + )? + 8; + let offset = curr + .wrapping_sub(dict_match_index_l) + .wrapping_sub(dict_index_delta); + let back = + count_backwards(buf, ip, anchor, dms.buf, dict_match_l, dict_start); + ip -= back; + break 'match_found (offset, m_length + back); + } + } + + let short_found = if match_index_s > prefix_lowest_index { + read_le32(buf, src.pos_of(match_index_s))? == read_le32(buf, ip)? + } else if dict_tags_match_s { + let dict_match_index_s = + dict_match_index_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS; + match_index_s = dict_match_index_s.wrapping_add(dict_index_delta); + dict_match_index_s > dict_start_index + && read_le32(dms.buf, dms.pos_of(dict_match_index_s))? + == read_le32(buf, ip)? + } else { + false + }; + if !short_found { + ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; + continue 'search; + } + + // A short match: a long one at the next position is preferred. + let hl3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; + let dict_hash_and_tag_l3 = hash_ptr(buf, ip + 1, dict_h_bits_l, 8)?; + let match_index_l3 = hash_long.at(hl3)?; + let dict_match_index_and_tag_l3 = + dict_hash_long.at(dict_hash_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_tags_match_l3 = + compare_packed_tags(dict_match_index_and_tag_l3 as usize, dict_hash_and_tag_l3); + let match_l3 = src.pos_of(match_index_l3); + hash_long.put(hl3, curr + 1)?; + + if match_index_l3 >= prefix_lowest_index + && read_le64(buf, match_l3)? == read_le64(buf, ip + 1)? + { + let m_length = count(buf, ip + 9, buf, match_l3 + 8, iend)? + 8; + ip += 1; + let offset = ip.wrapping_sub(match_l3) as u32; + let back = count_backwards(buf, ip, anchor, buf, match_l3, prefix_lowest); + ip -= back; + break 'match_found (offset, m_length + back); + } else if dict_tags_match_l3 { + let dict_match_index_l3 = + dict_match_index_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match_l3 = dms.pos_of(dict_match_index_l3); + if dict_match_index_l3 > dict_start_index + && read_le64(dms.buf, dict_match_l3)? == read_le64(buf, ip + 1)? + { + let m_length = count_2segments( + buf, + ip + 1 + 8, + dms.buf, + dict_match_l3 + 8, + iend, + dict_end, + prefix_lowest, + )? + 8; + ip += 1; + let offset = curr + .wrapping_add(1) + .wrapping_sub(dict_match_index_l3) + .wrapping_sub(dict_index_delta); + let back = + count_backwards(buf, ip, anchor, dms.buf, dict_match_l3, dict_start); + ip -= back; + break 'match_found (offset, m_length + back); + } + } + + let match_s = locate(match_index_s); + let m_length = count_2segments( + buf, + ip + 4, + match_s.buf, + match_s.pos + 4, + iend, + match_s.end, + prefix_lowest, + )? + 4; + let offset = curr.wrapping_sub(match_index_s); + let back = + count_backwards(buf, ip, anchor, match_s.buf, match_s.pos, match_s.start); + ip -= back; + (offset, m_length + back) + }; + + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + m_length + }; + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit { + let current2 = src.index_of(ip); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !(index_overlap_check(prefix_lowest_index, rep_index2) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_lowest, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; + ip += rep_length2; + anchor = ip; + } + } + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast` +pub(crate) fn compress_block_double_fast( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_no_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_no_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_no_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_no_dict_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState` +pub(crate) fn compress_block_double_fast_dict_match_state( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_doubleFast_extDict_generic` +fn compress_block_double_fast_ext_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); + let dict_start_index = low_limit; + let dict_limit = ms.window.dict_limit; + let prefix_start_index = dict_limit.max(low_limit); + let prefix_start = src.pos_of(prefix_start_index); + let dict_start = src.dict_pos_of(dict_start_index); + let dict_end = src.dict_pos_of(prefix_start_index); + let [mut offset_1, mut offset_2, _] = *rep; + + // Nothing of the older segment is within reach. + if prefix_start_index == dict_start_index { + return compress_block_double_fast(ms, seq_store, rep, src); + } + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: src.dict_buf, + pos: src.dict_pos_of(index), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip = istart; + + while ip < ilimit { + let h_small = hash_ptr(buf, ip, h_bits_s, MLS)?; + let match_index = hash_small.at(h_small)?; + let match_s = locate(match_index); + + let h_long = hash_ptr(buf, ip, h_bits_l, 8)?; + let match_long_index = hash_long.at(h_long)?; + let match_long = locate(match_long_index); + + let curr = src.index_of(ip); + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + hash_long.put(h_long, curr)?; + hash_small.put(h_small, curr)?; + + let m_length; + if (index_overlap_check(prefix_start_index, rep_index) + & (offset_1 <= curr.wrapping_add(1).wrapping_sub(dict_start_index))) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? + { + m_length = count_2segments( + buf, + ip + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_start, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + } else { + // The match, how much of it is known, and its offset. `ip` is moved to where it is for. + let (found, known, offset) = if match_long_index > dict_start_index + && read_le64(match_long.buf, match_long.pos)? == read_le64(buf, ip)? + { + (match_long, 8, curr.wrapping_sub(match_long_index)) + } else if match_index > dict_start_index + && read_le32(match_s.buf, match_s.pos)? == read_le32(buf, ip)? + { + // A short match: a long one at the next position is preferred. + let h3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; + let match_index3 = hash_long.at(h3)?; + let match3 = locate(match_index3); + hash_long.put(h3, curr + 1)?; + if match_index3 > dict_start_index + && read_le64(match3.buf, match3.pos)? == read_le64(buf, ip + 1)? + { + ip += 1; + (match3, 8, curr.wrapping_add(1).wrapping_sub(match_index3)) + } else { + (match_s, 4, curr.wrapping_sub(match_index)) + } + } else { + ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; + continue; + }; + let forwards = count_2segments( + buf, + ip + known, + found.buf, + found.pos + known, + iend, + found.end, + prefix_start, + )? + known; + let back = count_backwards(buf, ip, anchor, found.buf, found.pos, found.start); + ip -= back; + m_length = forwards + back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + } + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit { + let current2 = src.index_of(ip); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !((index_overlap_check(prefix_start_index, rep_index2) + & (offset_2 <= current2.wrapping_sub(dict_start_index))) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; + ip += rep_length2; + anchor = ip; + } + } + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast_extDict` +pub(crate) fn compress_block_double_fast_ext_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), + } +} diff --git a/src/zstd/wip/work/ldm/snap/fast.rs b/src/zstd/wip/work/ldm/snap/fast.rs new file mode 100644 index 000000000000..4bec128a6786 --- /dev/null +++ b/src/zstd/wip/work/ldm/snap/fast.rs @@ -0,0 +1,829 @@ +//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, + write_tagged_index, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, read_le32}; + +pub(super) const FAST_HASH_FILL_STEP: u32 = 3; + +/// Where an index leads, when it can be in one of two segments. +#[derive(Clone, Copy)] +pub(super) struct Located<'a> { + /// The buffer that the segment is in. + pub(super) buf: &'a [u8], + /// The position of the index in `buf`. + pub(super) pos: usize, + /// Where the segment starts in `buf`. + pub(super) start: usize, + /// Where it ends. + pub(super) end: usize, +} + +/// How many bytes before `ip`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. +#[inline(always)] +pub(super) fn count_backwards( + buf: &[u8], + ip: usize, + anchor: usize, + match_buf: &[u8], + match_pos: usize, + match_low: usize, +) -> usize { + let before_ip = buf.get(anchor..ip).unwrap_or_default(); + let before_match = match_buf.get(match_low..match_pos).unwrap_or_default(); + before_ip + .iter() + .rev() + .zip(before_match.iter().rev()) + .take_while(|(a, b)| a == b) + .count() +} + +/// The repeat offsets to leave at the end of a block, at the start of which those that reached too far were saved and +/// set to 0. +#[inline(always)] +pub(super) fn restore_rep( + rep: &mut [u32; ZSTD_REP_NUM], + offset_1: u32, + offset_2: u32, + offset_saved1: u32, + offset_saved2: u32, +) { + // If offset_1 started invalid and became valid, offset_saved1 is what rotates into the second place. + let offset_saved2 = if offset_saved1 != 0 && offset_1 != 0 { + offset_saved1 + } else { + offset_saved2 + }; + let [rep0, rep1, _] = rep; + *rep0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved1 + }; + *rep1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved2 + }; +} + +/// `ZSTD_fillHashTableForCDict` +fn fill_hash_table_for_cdict( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let mls = ms.c_params.min_match; + let hash_table = ms.hash_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { + let curr = prefix.index_of(ip); + write_tagged_index(hash_table, hash_ptr(buf, ip, h_bits, mls)?, curr)?; + if dtlm == DictTableLoadMethod::Full { + // The positions in between only go where there is nothing yet. + for p in 1..FAST_HASH_FILL_STEP { + let hash_and_tag = hash_ptr(buf, ip + p as usize, h_bits, mls)?; + if hash_table.at(hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { + write_tagged_index(hash_table, hash_and_tag, curr + p)?; + } + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillHashTableForCCtx` +fn fill_hash_table_for_cctx( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits = ms.c_params.hash_log; + let mls = ms.c_params.min_match; + let hash_table = ms.hash_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { + let curr = prefix.index_of(ip); + hash_table.put(hash_ptr(buf, ip, h_bits, mls)?, curr)?; + if dtlm == DictTableLoadMethod::Full { + // The positions in between only go where there is nothing yet. + for p in 1..FAST_HASH_FILL_STEP { + let hash = hash_ptr(buf, ip + p as usize, h_bits, mls)?; + if hash_table.at(hash)? == 0 { + hash_table.put(hash, curr + p)?; + } + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_hash_table( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, + tfp: TableFillPurpose, +) -> Result<()> { + match tfp { + TableFillPurpose::ForCDict => fill_hash_table_for_cdict(ms, prefix, end, dtlm), + TableFillPurpose::ForCCtx => fill_hash_table_for_cctx(ms, prefix, end, dtlm), + } +} + +/// `ZSTD_match4Found_branch`, and `ZSTD_match4Found_cmov`, which only differs in how it avoids reading at an index that +/// is not valid. +#[inline(always)] +fn match4_found( + buf: &[u8], + current: usize, + match_pos: usize, + match_idx: u32, + idx_low_limit: u32, +) -> Result { + Ok(match_idx >= idx_low_limit && read_le32(buf, current)? == read_le32(buf, match_pos)?) +} + +/// `ZSTD_compressBlock_fast_noDict_generic` +/// +/// Each turn of the inner loop looks at two positions, `ip0` and `ip1`, for a match, and at `ip2` for a repeat. The +/// hash of a position is computed, and the table is read, one position ahead of the comparison. +fn compress_block_fast_no_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); + + let hlog = ms.c_params.hash_log; + let window_log = ms.c_params.window_log; + let target_length = ms.c_params.target_length; + // The least step is 2, as two positions are looked at per turn. + let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_start_index = ms.get_lowest_prefix_index(end_index, window_log); + let prefix_start = src.pos_of(prefix_start_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip0 = istart + usize::from(istart == prefix_start); + let [mut rep_offset1, mut rep_offset2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let curr = src.index_of(ip0); + let window_low = ms.get_lowest_prefix_index(curr, window_log); + let max_rep = curr.wrapping_sub(window_low); + if rep_offset2 > max_rep { + offset_saved2 = rep_offset2; + rep_offset2 = 0; + } + if rep_offset1 > max_rep { + offset_saved1 = rep_offset1; + rep_offset1 = 0; + } + } + let hash_table = ms.hash_table.as_mut_slice(); + + 'start: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let mut ip1 = ip0 + 1; + let mut ip2 = ip0 + step; + let mut ip3 = ip2 + 1; + if ip3 >= ilimit { + break 'start; + } + + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let mut match_idx = hash_table.at(hash0)?; + let mut current0; + + // Ends with the start and the length so far of a repeat match, or with `None` for a match at `match_idx`. + let rep_match = loop { + let rval = read_le32(buf, ip2.wrapping_sub(rep_offset1 as usize))?; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if (read_le32(buf, ip2)? == rval) & (rep_offset1 > 0) { + ip0 = ip2; + let match0 = ip0.wrapping_sub(rep_offset1 as usize); + let back = usize::from(buf.at(ip0 - 1)? == buf.at(match0.wrapping_sub(1))?); + ip0 -= back; + // The hash of ip1 is there, and ip1 is before the end of the match. + hash_table.put(hash1, src.index_of(ip1))?; + break Some((match0.wrapping_sub(back), 4 + back)); + } + + if match4_found( + buf, + ip0, + src.pos_of(match_idx), + match_idx, + prefix_start_index, + )? { + hash_table.put(hash1, src.index_of(ip1))?; + break None; + } + + match_idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if match4_found( + buf, + ip0, + src.pos_of(match_idx), + match_idx, + prefix_start_index, + )? { + // ip1 is certain to be before the end of the match only if the step is small. + if step <= 4 { + hash_table.put(hash1, src.index_of(ip1))?; + } + break None; + } + + match_idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + if ip2 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + if ip3 >= ilimit { + break 'start; + } + }; + + let (offcode, match0, mut m_length) = match rep_match { + Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), + None => { + let match0 = src.pos_of(match_idx); + rep_offset2 = rep_offset1; + rep_offset1 = ip0.wrapping_sub(match0) as u32; + let back = count_backwards(buf, ip0, anchor, buf, match0, prefix_start); + ip0 -= back; + (offset_to_offbase(rep_offset1), match0 - back, 4 + back) + } + }; + + m_length += count(buf, ip0 + m_length, buf, match0 + m_length, iend)?; + seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; + ip0 += m_length; + anchor = ip0; + + if ip0 <= ilimit { + hash_table.put( + hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, + current0 + 2, + )?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + if rep_offset2 > 0 { + while ip0 <= ilimit + && read_le32(buf, ip0)? + == read_le32(buf, ip0.wrapping_sub(rep_offset2 as usize))? + { + let r_length = count( + buf, + ip0 + 4, + buf, + (ip0 + 4).wrapping_sub(rep_offset2 as usize), + iend, + )? + 4; + core::mem::swap(&mut rep_offset1, &mut rep_offset2); + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; + ip0 += r_length; + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; + anchor = ip0; + } + } + } + } + + restore_rep(rep, rep_offset1, rep_offset2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast` +pub(crate) fn compress_block_fast( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_no_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_no_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_no_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_no_dict_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_fast_dictMatchState_generic` +fn compress_block_fast_dict_match_state_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; + + let hlog = ms.c_params.hash_log; + let target_length = ms.c_params.target_length; + let step_size = (target_length + u32::from(target_length == 0)) as usize; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let prefix_start_index = ms.window.dict_limit; + let prefix_start = src.pos_of(prefix_start_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + let [mut offset_1, mut offset_2, _] = *rep; + + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_hash_table = dms.ms.hash_table.as_slice(); + let dict_start_index = dms.ms.window.dict_limit; + let dict_start = dms.pos_of(dict_start_index); + let dict_end = dms.end; + // What to subtract from an index below prefix_start_index to get the index in the dictionary. + let dict_index_delta = prefix_start_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); + let dict_and_prefix_length = istart + .wrapping_sub(prefix_start) + .wrapping_add(dict_end) + .wrapping_sub(dict_start) as u32; + let dict_h_bits = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: dms.buf, + pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + let hash_table = ms.hash_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip0 = istart + usize::from(dict_and_prefix_length == 0); + let mut ip1 = istart + step_size; + + 'search: while ip1 <= ilimit { + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + + let dict_hash_and_tag0 = hash_ptr(buf, ip0, dict_h_bits, MLS)?; + let mut dict_match_index_and_tag = + dict_hash_table.at(dict_hash_and_tag0 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let mut dict_tags_match = + compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag0); + + let mut match_index = hash_table.at(hash0)?; + let mut curr = src.index_of(ip0); + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let m_length = loop { + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + let hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let dict_hash_and_tag1 = hash_ptr(buf, ip1, dict_h_bits, MLS)?; + hash_table.put(hash0, curr)?; + + if index_overlap_check(prefix_start_index, rep_index) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip0 + 1)? + { + let m_length = count_2segments( + buf, + ip0 + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_start, + )? + 4; + ip0 += 1; + seq_store.store_seq(ip0 - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break m_length; + } + + if dict_tags_match { + let dict_match_index = dict_match_index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match = dms.pos_of(dict_match_index); + // A match in the dictionary is only taken if there is no candidate in the prefix. + if dict_match_index > dict_start_index + && read_le32(dms.buf, dict_match)? == read_le32(buf, ip0)? + && match_index <= prefix_start_index + { + let offset = curr + .wrapping_sub(dict_match_index) + .wrapping_sub(dict_index_delta); + let mut m_length = count_2segments( + buf, + ip0 + 4, + dms.buf, + dict_match + 4, + iend, + dict_end, + prefix_start, + )? + 4; + let back = count_backwards(buf, ip0, anchor, dms.buf, dict_match, dict_start); + ip0 -= back; + m_length += back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip0 - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + break m_length; + } + } + + let match_pos = src.pos_of(match_index); + if match4_found(buf, ip0, match_pos, match_index, prefix_start_index)? { + let offset = ip0.wrapping_sub(match_pos) as u32; + let mut m_length = count(buf, ip0 + 4, buf, match_pos + 4, iend)? + 4; + let back = count_backwards(buf, ip0, anchor, buf, match_pos, prefix_start); + ip0 -= back; + m_length += back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip0 - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + break m_length; + } + + dict_match_index_and_tag = + dict_hash_table.at(dict_hash_and_tag1 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + dict_tags_match = + compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag1); + match_index = hash_table.at(hash1)?; + + if ip1 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + ip0 = ip1; + ip1 += step; + if ip1 > ilimit { + break 'search; + } + + curr = src.index_of(ip0); + hash0 = hash1; + }; + + ip0 += m_length; + anchor = ip0; + + if ip0 <= ilimit { + hash_table.put(hash_ptr(buf, src.pos_of(curr + 2), hlog, MLS)?, curr + 2)?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + while ip0 <= ilimit { + let current2 = src.index_of(ip0); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !(index_overlap_check(prefix_start_index, rep_index2) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip0 + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, current2)?; + ip0 += rep_length2; + anchor = ip0; + } + } + + ip1 = ip0 + step_size; + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast_dictMatchState` +pub(crate) fn compress_block_fast_dict_match_state( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_fast_extDict_generic` +fn compress_block_fast_ext_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); + + let hlog = ms.c_params.hash_log; + let target_length = ms.c_params.target_length; + let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); + let dict_start_index = low_limit; + let dict_start = src.dict_pos_of(dict_start_index); + let dict_limit = ms.window.dict_limit; + let prefix_start_index = dict_limit.max(low_limit); + let prefix_start = src.pos_of(prefix_start_index); + let dict_end = src.dict_pos_of(prefix_start_index); + + // Nothing of the older segment is within reach. + if prefix_start_index == dict_start_index { + return compress_block_fast(ms, seq_store, rep, src); + } + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip0 = istart; + let [mut offset_1, mut offset_2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let curr = src.index_of(ip0); + let max_rep = curr.wrapping_sub(dict_start_index); + if offset_2 >= max_rep { + offset_saved2 = offset_2; + offset_2 = 0; + } + if offset_1 >= max_rep { + offset_saved1 = offset_1; + offset_1 = 0; + } + } + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: src.dict_buf, + pos: src.dict_pos_of(index), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + // Whether there is a match for the position `ip` at `idx`, which is from the table. + let match_found = |ip: usize, idx: u32| -> Result { + let candidate = locate(idx); + Ok(idx >= dict_start_index + && read_le32(candidate.buf, candidate.pos)? == read_le32(buf, ip)?) + }; + let hash_table = ms.hash_table.as_mut_slice(); + + 'start: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let mut ip1 = ip0 + 1; + let mut ip2 = ip0 + step; + let mut ip3 = ip2 + 1; + if ip3 >= ilimit { + break 'start; + } + + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let mut idx = hash_table.at(hash0)?; + let mut current0; + + // Ends with the start and the length so far of a repeat match, or with `None` for a match at `idx`. + let rep_match = loop { + let current2 = src.index_of(ip2); + let rep_index = current2.wrapping_sub(offset_1); + let mut rep_match = locate(rep_index); + // The 4 bytes have to be in one segment, with a byte before them. + let rep_found = ((prefix_start_index.wrapping_sub(rep_index) >= 4) & (offset_1 > 0)) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip2)?; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if rep_found { + ip0 = ip2; + let back = usize::from( + buf.at(ip0 - 1)? == rep_match.buf.at(rep_match.pos.wrapping_sub(1))?, + ); + ip0 -= back; + rep_match.pos = rep_match.pos.wrapping_sub(back); + break Some((rep_match, 4 + back)); + } + + if match_found(ip0, idx)? { + break None; + } + + idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if match_found(ip0, idx)? { + break None; + } + + idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + if ip2 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + if ip3 >= ilimit { + break 'start; + } + }; + + let (offcode, match0, mut m_length) = match rep_match { + Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), + None => { + let offset = current0.wrapping_sub(idx); + let mut match0 = locate(idx); + offset_2 = offset_1; + offset_1 = offset; + let back = count_backwards(buf, ip0, anchor, match0.buf, match0.pos, match0.start); + ip0 -= back; + match0.pos -= back; + (offset_to_offbase(offset), match0, 4 + back) + } + }; + + m_length += count_2segments( + buf, + ip0 + m_length, + match0.buf, + match0.pos + m_length, + iend, + match0.end, + prefix_start, + )?; + seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; + ip0 += m_length; + anchor = ip0; + + if ip1 < ip0 { + hash_table.put(hash1, src.index_of(ip1))?; + } + + if ip0 <= ilimit { + hash_table.put( + hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, + current0 + 2, + )?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + while ip0 <= ilimit { + let rep_index2 = src.index_of(ip0).wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !((index_overlap_check(prefix_start_index, rep_index2) & (offset_2 > 0)) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip0 + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; + ip0 += rep_length2; + anchor = ip0; + } + } + } + + restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast_extDict` +pub(crate) fn compress_block_fast_ext_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), + } +} diff --git a/src/zstd/wip/work/ldm/snap/lazy.rs b/src/zstd/wip/work/ldm/snap/lazy.rs new file mode 100644 index 000000000000..94dbe308ce79 --- /dev/null +++ b/src/zstd/wip/work/ldm/snap/lazy.rs @@ -0,0 +1,1353 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use super::internal::{ + BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, + SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, + hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, + offset_to_offbase, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +const K_LAZY_SKIPPING_STEP: usize = 8; + +// `ZSTD_dictMode_e`, as an argument of a const generic. +const NO_DICT: u8 = DictMode::NoDict as u8; +const EXT_DICT: u8 = DictMode::ExtDict as u8; +const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; + +/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. +#[inline(always)] +const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { + dms.end.wrapping_sub(dms.base) as u32 +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. +#[inline(always)] +fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { + let hash_log = ms.c_params.hash_log; + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let bt = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); + *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. +fn insert_dubt1( + ms: &mut MatchState, + src: &BlockSrc<'_>, + curr: u32, + input_end: usize, + mut nb_compares: u32, + bt_low: u32, +) -> Result<()> { + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let (ip_buf, ip, iend) = if curr >= dict_limit { + (src.buf, src.pos_of(curr), input_end) + } else { + (src.dict_buf, src.dict_pos_of(curr), dict_end) + }; + let window_valid = ms.window.low_limit; + let max_distance = 1u32 << ms.c_params.window_log; + let window_low = if curr.wrapping_sub(window_valid) > max_distance { + curr - max_distance + } else { + window_valid + }; + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + // The next sorted candidate. The other slot has the previous unsorted one. + let mut match_index = bt.at(smaller_ptr)?; + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let in_prefix = + DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; + // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = if in_prefix { + (src.buf, src.pos_of(match_index)) + } else { + (src.dict_buf, src.dict_pos_of(match_index)) + }; + + if in_prefix || curr < dict_limit { + // Both are in the same segment. + match_length += count( + ip_buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + ip_buf, + ip + match_length, + match_buf, + match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if match_index as usize + match_length >= dict_limit as usize { + (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); + } + } + + if ip + match_length == iend { + // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0) +} + +/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. +fn dubt_find_better_dict_match( + ms: &MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mut best_length: usize, + mut nb_compares: u32, + mls: u32, +) -> Result { + let dms = src.dms.ok_or(BUG)?; + let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(h)?; + + let prefix_start = src.pos_of(ms.window.dict_limit); + let curr = src.index_of(ip); + let dict_high_limit = dms_end_index(&dms); + let dict_low_limit = dms.ms.window.low_limit; + let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); + + let dict_bt = dms.ms.chain_table.as_slice(); + let bt_log = dms.ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { + dict_low_limit + } else { + dict_high_limit - bt_mask + }; + + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + + while nb_compares != 0 && dict_match_index > dict_low_limit { + let next_ptr = 2 * (dict_match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let match_index = dict_match_index.wrapping_add(dict_index_delta); + match_length += count_2segments( + src.buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + iend, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(offset_ptr.wrapping_add(1)) as i32 + { + best_length = match_length; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // No way to know which is smaller. + break; + } + } + + if dict_match_index <= bt_low { + // Beyond the tree. + break; + } + let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { + let match_pos = src.pos_of(match_index); + src.buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < src.buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dict_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dict_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + Ok(best_length) +} + +/// `ZSTD_DUBT_findBestMatch` +#[inline(always)] +fn dubt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + off_base_ptr: &mut u32, + mls: u32, +) -> Result { + let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; + let mut match_index = ms.hash_table.at(h)?; + + let curr = src.index_of(ip); + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = curr.saturating_sub(bt_mask); + let unsort_limit = bt_low.max(window_low); + let mut nb_compares = 1u32 << ms.c_params.search_log; + let mut nb_candidates = nb_compares; + let mut previous_candidate = 0u32; + + // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. + let bt = ms.chain_table.as_mut_slice(); + let mut next_candidate = 2 * (match_index & bt_mask) as usize; + while match_index > unsort_limit + && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK + && nb_candidates > 1 + { + bt.put(next_candidate + 1, previous_candidate)?; + previous_candidate = match_index; + match_index = bt.at(next_candidate)?; + next_candidate = 2 * (match_index & bt_mask) as usize; + nb_candidates -= 1; + } + + // If the last candidate is still unsorted, cut the list there. + if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { + *bt.array_at_mut(next_candidate)? = [0, 0]; + } + + // Sort the candidates, the oldest first. + match_index = previous_candidate; + while match_index != 0 { + let next_candidate_idx = ms + .chain_table + .at(2 * (match_index & bt_mask) as usize + 1)?; + insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; + match_index = next_candidate_idx; + nb_candidates += 1; + } + + // Find the longest match. + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + let mut match_end_idx = curr + 8 + 1; + let mut best_length = 0usize; + + match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); + + if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + src.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + let dict_match_pos = src.dict_pos_of(match_index); + match_length += count_2segments( + src.buf, + ip + match_length, + src.dict_buf, + dict_match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + (match_buf, match_pos) = (src.dict_buf, dict_match_pos); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index + match_length as u32; + } + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(*off_base_ptr) as i32 + { + best_length = match_length; + *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // Equal: no way to know which is smaller. + if DICT_MODE == DICT_MATCH_STATE { + // Nothing in the dictionary can be longer. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0)?; + + if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { + best_length = dubt_find_better_dict_match( + ms, + src, + ip, + iend, + off_base_ptr, + best_length, + nb_compares, + mls, + )?; + } + + // Skip repetitive patterns. + ms.next_to_update = match_end_idx - 8; + Ok(best_length) +} + +/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. +/// `off_base_ptr` is written to if there is one. +#[inline(never)] +fn bt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + off_base_ptr: &mut u32, +) -> Result { + if ip < src.pos_of(ms.next_to_update) { + // In an area that is skipped. + return Ok(0); + } + update_dubt(ms, src.prefix(), ip, MLS)?; + dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) +} + +// ── hash chains ─────────────────────────────────────────────────────────── + +/// `ZSTD_insertAndFindFirstIndex_internal` +#[inline(always)] +fn insert_and_find_first_index_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + lazy_skipping: bool, +) -> Result { + let hash_log = ms.c_params.hash_log; + let chain_mask = (1u32 << ms.c_params.chain_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let chain_table = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); + chain_table.put((idx & chain_mask) as usize, next_in_chain)?; + // While skipping, not every position is inserted. + if lazy_skipping { + break; + } + } + + ms.next_to_update = target; + hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) +} + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) +} + +/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn hc_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let chain_size = 1u32 << ms.c_params.chain_log; + let chain_mask = chain_size - 1; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let min_chain = curr.saturating_sub(chain_size); + let mut nb_attempts = 1u32 << ms.c_params.search_log; + let mut ml = 4 - 1; + + let mut match_index = + insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; + let chain_table = ms.chain_table.as_slice(); + + while match_index >= low_limit && nb_attempts > 0 { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + + if match_index <= min_chain { + break; + } + match_index = chain_table.at((match_index & chain_mask) as usize)?; + nb_attempts -= 1; + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_chain_table = dms.ms.chain_table.as_slice(); + let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; + let dms_chain_mask = dms_chain_size - 1; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_size = dms_end_index(&dms); + let dms_index_delta = dict_limit.wrapping_sub(dms_size); + let dms_min_chain = dms_size.saturating_sub(dms_chain_size); + + let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; + match_index = dms.ms.hash_table.at(h)?; + + while match_index >= dms_lowest_index && nb_attempts > 0 { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + + if match_index <= dms_min_chain { + break; + } + match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; + nb_attempts -= 1; + } + } + + Ok(ml) +} + +// ── row-based match finder ──────────────────────────────────────────────── +// +// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits +// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag +// table is the "head": which entry was written last. + +pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; +const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; +const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; + +/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. +#[inline(always)] +fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { + let mut next = u32::from(*head).wrapping_sub(1) & row_mask; + if next == 0 { + next = row_mask; + } + *head = next as u8; + next +} + +/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. +#[inline(always)] +fn row_fill_hash_cache( + ms: &mut MatchState, + prefix: Prefix<'_>, + mls: u32, + idx: u32, + i_limit: usize, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); + let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; + + for idx in idx..lim { + let hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + ms.hash_salt, + )? as u32; + let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; + ms.hash_cache.put(cached, hash)?; + } + Ok(()) +} + +/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes +/// further takes its place. +#[inline(always)] +fn row_next_cached_hash( + cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], + prefix: Prefix<'_>, + idx: u32, + hash_log: u32, + mls: u32, + hash_salt: u64, +) -> Result { + let new_hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32; + let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; + Ok(core::mem::replace(cached, new_hash)) +} + +/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. +#[inline(always)] +fn row_update_internal_impl( + ms: &mut MatchState, + prefix: Prefix<'_>, + update_start_idx: u32, + update_end_idx: u32, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let hash_salt = ms.hash_salt; + let hash_table = ms.hash_table.as_mut_slice(); + let tag_table = ms.tag_table.as_mut_slice(); + + for idx in update_start_idx..update_end_idx { + let hash = if use_cache { + row_next_cached_hash(&mut ms.hash_cache, prefix, idx, hash_log, mls, hash_salt)? + } else { + hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32 + }; + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; + tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; + hash_table.put(rel_row + pos, idx)?; + } + Ok(()) +} + +/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is +/// when compressing, not all of them if they are many. +#[inline(always)] +fn row_update_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + const K_SKIP_THRESHOLD: u32 = 384; + const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; + const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; + + let mut idx = ms.next_to_update; + let target = prefix.index_of(ip); + + if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { + let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; + row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; + idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; + row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; + } + row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the +/// row-based match finder. +pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { + let row_log = ms.c_params.search_log.clamp(4, 6); + let row_mask = (1u32 << row_log) - 1; + let mls = ms.c_params.min_match.min(6); + row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) +} + +/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. +#[inline(always)] +fn row_get_match_mask( + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + head: u32, +) -> u64 { + let mut matches = 0u64; + let (chunks, _) = tag_row.as_chunks::<16>(); + for (n, chunk) in chunks.iter().enumerate() { + let mut chunk_matches = 0u16; + for (i, &entry) in chunk.iter().enumerate() { + chunk_matches |= u16::from(entry == tag) << i; + } + matches |= u64::from(chunk_matches) << (16 * n); + } + match ROW_ENTRIES { + 16 => u64::from((matches as u16).rotate_right(head)), + 32 => u64::from((matches as u32).rotate_right(head)), + _ => matches.rotate_right(head), + } +} + +/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, +/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than +/// `low_limit`. Returns how many there are. +#[inline(always)] +fn row_collect_matches( + row: &[u32; ROW_ENTRIES], + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + low_limit: u32, + nb_attempts: &mut u32, + match_buffer: &mut [u32; ROW_ENTRIES], +) -> Result { + let row_mask = ROW_ENTRIES as u32 - 1; + let head = u32::from(tag_row.at(0)?) & row_mask; + let mut num_matches = 0usize; + let mut matches = row_get_match_mask(tag_row, tag, head); + + while matches > 0 && *nb_attempts > 0 { + let match_pos = (head + matches.trailing_zeros()) & row_mask; + matches &= matches - 1; + if match_pos == 0 { + // The head, which happens to be the same as the tag. + continue; + } + let match_index = row.at(match_pos as usize)?; + if match_index < low_limit { + break; + } + match_buffer.put(num_matches, match_index)?; + num_matches += 1; + *nb_attempts -= 1; + } + Ok(num_matches) +} + +/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn row_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let hash_log = ms.row_hash_log; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let row_log = ROW_ENTRIES.trailing_zeros(); + let row_mask = ROW_ENTRIES as u32 - 1; + // There are no more searches than entries in a row. + let capped_search_log = ms.c_params.search_log.min(row_log); + let hash_salt = ms.hash_salt; + let mut nb_attempts = 1u32 << capped_search_log; + let mut ml = 4 - 1; + let mut match_buffer = [0u32; ROW_ENTRIES]; + + // Update the tables up to, and not including, `ip`. + let hash = if ms.lazy_skipping { + // While skipping, not every position is inserted. + ms.next_to_update = curr; + hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 + } else { + let prefix = src.prefix(); + row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; + row_next_cached_hash(&mut ms.hash_cache, prefix, curr, hash_log, MLS, hash_salt)? + }; + ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); + + { + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; + let row = ms.hash_table.array_at_mut::(rel_row)?; + let tag_row = ms.tag_table.array_at_mut::(rel_row)?; + let num_matches = row_collect_matches( + row, + tag_row, + tag, + low_limit, + &mut nb_attempts, + &mut match_buffer, + )?; + + // Insert `ip`. + let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; + tag_row.put(pos, tag)?; + row.put(pos, ms.next_to_update)?; + ms.next_to_update += 1; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + } + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); + + let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + let dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; + let dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let num_matches = row_collect_matches( + dms.ms.hash_table.array_at::(dms_rel_row)?, + dms.ms.tag_table.array_at::(dms_rel_row)?, + (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, + dms_lowest_index, + &mut nb_attempts, + &mut match_buffer, + )?; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + } + } + + Ok(ml) +} + +// ── the parsers ─────────────────────────────────────────────────────────── + +// `searchMethod_e`, as an argument of a const generic. +const SEARCH_HASH_CHAIN: u8 = 0; +const SEARCH_BINARY_TREE: u8 = 1; +const SEARCH_ROW_HASH: u8 = 2; + +/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. +#[inline(always)] +fn search_max( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mls: u32, + row_log: u32, +) -> Result { + match SEARCH_METHOD { + SEARCH_HASH_CHAIN => match mls { + 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + SEARCH_BINARY_TREE => match mls { + 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + _ => match (mls, row_log) { + (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + } +} + +/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match +/// state is searched. +#[derive(Clone, Copy)] +struct DictSegment<'a> { + buf: &'a [u8], + /// Its sum with an index of the window is a position in `buf`. It wraps around. + base: usize, + /// Where the segment starts in `buf`. + start: usize, + /// Where it ends. + end: usize, +} + +/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. +#[inline(always)] +fn rep_length( + ms: &MatchState, + src: &BlockSrc<'_>, + dict: &DictSegment<'_>, + ip: usize, + offset: u32, +) -> Result { + let buf = src.buf; + let iend = src.end; + if DICT_MODE == NO_DICT { + let rep_match = ip.wrapping_sub(offset as usize); + if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { + return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); + } + return Ok(0); + } + + let dict_limit = ms.window.dict_limit; + let curr = src.index_of(ip); + let rep_index = curr.wrapping_sub(offset); + if !index_overlap_check(dict_limit, rep_index) { + return Ok(0); + } + if DICT_MODE == EXT_DICT { + // The offset may lead to what is no longer in the window. + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + if offset > curr.wrapping_sub(window_low) { + return Ok(0); + } + } + let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { + let rep_match = dict.base.wrapping_add(rep_index as usize); + (dict.buf, rep_match, dict.end) + } else { + (buf, src.pos_of(rep_index), iend) + }; + if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { + return Ok(0); + } + let prefix_start = src.pos_of(dict_limit); + Ok(count_2segments( + buf, + ip + 4, + rep_buf, + rep_match + 4, + iend, + rep_end, + prefix_start, + )? + 4) +} + +/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions +/// further a better match is looked for, once there is one. +#[inline(never)] +fn compress_block_lazy_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let prefix_lowest_index = ms.window.dict_limit; + let prefix_lowest = src.pos_of(prefix_lowest_index); + let mls = ms.c_params.min_match.clamp(4, 6); + let row_log = ms.c_params.search_log.clamp(4, 6); + + let [rep_0, rep_1, _] = rep; + let mut offset_1 = *rep_0; + let mut offset_2 = *rep_1; + let mut offset_saved_1 = 0u32; + let mut offset_saved_2 = 0u32; + + ms.lazy_skipping = false; + + let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { + 8 + ZSTD_ROW_HASH_CACHE_SIZE + } else { + 8 + }; + let Some(ilimit) = iend.checked_sub(margin) else { + // There is no position to search, nor to hash. + return Ok(iend - istart); + }; + + let dict = match DICT_MODE { + NO_DICT => DictSegment { + buf: &[], + base: 0, + start: 0, + end: 0, + }, + EXT_DICT => DictSegment { + buf: src.dict_buf, + base: src.dict_base, + start: src.dict_pos_of(ms.window.low_limit), + end: src.dict_pos_of(prefix_lowest_index), + }, + _ => { + let dms = src.dms.ok_or(BUG)?; + let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); + DictSegment { + buf: dms.buf, + base: dms.base.wrapping_sub(dict_index_delta as usize), + start: dms.pos_of(dms.ms.window.dict_limit), + end: dms.end, + } + } + }; + + if DICT_MODE == EXT_DICT { + ip += usize::from(ip == prefix_lowest); + } else { + let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); + ip += usize::from(dict_and_prefix_length == 0); + } + if DICT_MODE == NO_DICT { + let curr = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); + let max_rep = curr - window_low; + if offset_2 > max_rep { + offset_saved_2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved_1 = offset_1; + offset_1 = 0; + } + } + + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + + 'parse: while ip < ilimit { + let mut match_length; + let mut off_base = REPCODE1_TO_OFFBASE; + let mut start = ip + 1; + + 'found: { + // Check the repeat offset. + match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; + if DEPTH == 0 && match_length != 0 { + break 'found; + } + + // First search. + { + let mut off_base_found = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut off_base_found, + mls, + row_log, + )?; + if ml2 > match_length { + match_length = ml2; + start = ip; + off_base = off_base_found; + } + } + + if match_length < 4 { + // Jump faster over incompressible sections. + let step = (ip - anchor) >> K_SEARCH_STRENGTH; + ip += step + 1; + // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of + // no use out of the tables. As in the C code, it does not start at the same step in all modes. + ms.lazy_skipping = if DICT_MODE == EXT_DICT { + step > K_LAZY_SKIPPING_STEP + } else { + step + 1 > K_LAZY_SKIPPING_STEP + }; + continue 'parse; + } + + // Look for something better at the next positions. + if DEPTH >= 1 { + while ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 3) as i32; + let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + + // Nothing better: try one position further. + if DEPTH == 2 && ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 4) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + } + break; + } + } + + // Catch up: the match may start earlier. + if offbase_is_offset(off_base) { + let offset = offbase_to_offset(off_base); + let match_index = src.index_of(start).wrapping_sub(offset); + let (match_buf, mut match_pos, m_start) = + if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { + let match_pos = dict.base.wrapping_add(match_index as usize); + (dict.buf, match_pos, dict.start) + } else { + (buf, src.pos_of(match_index), prefix_lowest) + }; + while start > anchor + && match_pos > m_start + && buf.at(start - 1)? == match_buf.at(match_pos - 1)? + { + start -= 1; + match_pos -= 1; + match_length += 1; + } + offset_2 = offset_1; + offset_1 = offset; + } + } + + // Store the sequence. + seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; + ip = start + match_length; + anchor = ip; + + if ms.lazy_skipping { + // There was a match: every position is inserted again. + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + ms.lazy_skipping = false; + } + + // Check the second repeat offset right away. + while ip <= ilimit { + match_length = rep_length::(ms, src, &dict, ip, offset_2)?; + if match_length == 0 { + break; + } + (offset_1, offset_2) = (offset_2, offset_1); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; + ip += match_length; + anchor = ip; + } + } + + // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. + if offset_saved_1 != 0 && offset_1 != 0 { + offset_saved_2 = offset_saved_1; + } + + // Save the repeat offsets for the next block. + *rep_0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved_1 + }; + *rep_1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved_2 + }; + + // The size of the last literals. + Ok(iend - anchor) +} + +/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. +#[inline(always)] +fn compress_block_lazy_any_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + match dict_mode { + DictMode::NoDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::ExtDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + compress_block_lazy_generic::( + ms, seq_store, rep, src, + ) + } + } +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row: bool, + dict_mode: DictMode, +) -> Result { + match (strategy, use_row) { + (Strategy::Greedy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Greedy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( + ms, seq_store, rep, src, dict_mode, + ), + _ => Err(BUG), + } +} diff --git a/src/zstd/wip/work/ldm/snap/opt.rs b/src/zstd/wip/work/ldm/snap/opt.rs new file mode 100644 index 000000000000..a7a6022a3a45 --- /dev/null +++ b/src/zstd/wip/work/ldm/snap/opt.rs @@ -0,0 +1,1721 @@ +//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. + +use core::mem; + +use super::hist; +use super::huf::HufRepeat; +use super::internal::{ + BlockSrc, DictMode, EntropyCTables, Match, MatchState, OptPrice, OptState, Optimal, + ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, ZSTD_OPT_NUM, count, + count_2segments, hash_ptr, hash3_ptr, index_overlap_check, ll_code, ml_code, new_rep, + offset_to_offbase, +}; +use crate::common::{ + LL_BITS, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, +}; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +/// How much a literal adds to its frequency, so that frequencies adapt fast to new statistics. +const ZSTD_LITFREQ_ADD: u32 = 2; +const ZSTD_MAX_PRICE: i32 = 1 << 30; +/// A first block of at most this size is priced by fixed rules, not by statistics. +const ZSTD_PREDEF_THRESHOLD: usize = 8; + +const MAX_LIT: u32 = 255; + +// ── prices ──────────────────────────────────────────────────────────────── + +/// Prices are in units of `1 / BITCOST_MULTIPLIER` of a bit. +const BITCOST_ACCURACY: u32 = 8; +const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY; + +/// `WEIGHT`: approximate for `btopt`, accurate for `btultra`. +#[inline(always)] +const fn weight(stat: u32, opt_level: i32) -> u32 { + if opt_level != 0 { + frac_weight(stat) + } else { + bit_weight(stat) + } +} + +/// `ZSTD_bitWeight`: the cost of a statistic, in whole bits. +#[inline(always)] +const fn bit_weight(stat: u32) -> u32 { + highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER +} + +/// `ZSTD_fracWeight`: the cost of a statistic, with a fraction of a bit that is interpolated linearly. +#[inline(always)] +const fn frac_weight(raw_stat: u32) -> u32 { + let stat = raw_stat.wrapping_add(1); + let hb = highbit32(stat); + let b_weight = hb * BITCOST_MULTIPLIER; + // From 1 to 2, in fixed point. + let f_weight = (stat << BITCOST_ACCURACY) >> hb; + b_weight.wrapping_add(f_weight) +} + +/// `ZSTD_compressedLiterals` +#[inline(always)] +fn compressed_literals(opt_ptr: &OptState) -> bool { + opt_ptr.literal_compression_mode != ParamSwitch::Disable +} + +/// `ZSTD_setBasePrices` +#[inline(always)] +fn set_base_prices(opt_ptr: &mut OptState, opt_level: i32) { + if compressed_literals(opt_ptr) { + opt_ptr.lit_sum_base_price = weight(opt_ptr.lit_sum, opt_level); + } + opt_ptr.lit_length_sum_base_price = weight(opt_ptr.lit_length_sum, opt_level); + opt_ptr.match_length_sum_base_price = weight(opt_ptr.match_length_sum, opt_level); + opt_ptr.off_code_sum_base_price = weight(opt_ptr.off_code_sum, opt_level); +} + +/// `sum_u32` +fn sum_u32(table: &[u32]) -> u32 { + table.iter().fold(0, |total, &elt| total.wrapping_add(elt)) +} + +/// `base_directive_e`: the least a scaled down statistic can be. +#[derive(Clone, Copy, PartialEq, Eq)] +enum BaseDirective { + /// 0, if it is 0 already. + Base0Possible, + Base1Guaranteed, +} + +/// `ZSTD_downscaleStats`: returns the new sum of the elements. +fn downscale_stats( + table: &mut [u32], + last_elt_index: u32, + shift: u32, + base1: BaseDirective, +) -> Result { + let mut sum = 0u32; + for elt in table.sub_mut(..=last_elt_index as usize)? { + let base = u32::from(base1 == BaseDirective::Base1Guaranteed || *elt > 0); + let new_stat = base.wrapping_add(*elt >> (shift & 31)); + sum = sum.wrapping_add(new_stat); + *elt = new_stat; + } + Ok(sum) +} + +/// `ZSTD_scaleStats`: scales all the elements down if their sum is too large. Returns the sum that results. +fn scale_stats(table: &mut [u32], last_elt_index: u32, log_target: u32) -> Result { + let prevsum = sum_u32(table.sub(..=last_elt_index as usize)?); + let factor = prevsum >> log_target; + if factor <= 1 { + return Ok(prevsum); + } + downscale_stats( + table, + last_elt_index, + highbit32(factor), + BaseDirective::Base1Guaranteed, + ) +} + +/// Frequencies, out of about `1 << scale_log`, that make each symbol cost the bits `bit_cost` tells. Returns their sum. +fn freqs_from_bit_costs( + freqs: &mut [u32], + last_elt_index: u32, + scale_log: u32, + bit_cost: impl Fn(u32) -> u32, +) -> Result { + let mut sum = 0u32; + for (symbol, freq) in (0u32..).zip(freqs.sub_mut(..=last_elt_index as usize)?) { + let bit_cost = bit_cost(symbol); + // At least 1, to have a cost. + *freq = if bit_cost != 0 { + 1 << (scale_log.wrapping_sub(bit_cost) & 31) + } else { + 1 + }; + sum = sum.wrapping_add(*freq); + } + Ok(sum) +} + +/// `ZSTD_rescaleFreqs`: for the first block, which no statistics tell, starts them from the tables of the dictionary +/// if there is one, else from the literals of `src` and from fixed values for the sequences. For the next ones, scales +/// down what the blocks before have accumulated. +fn rescale_freqs( + opt_ptr: &mut OptState, + symbol_costs: &EntropyCTables, + src: &[u8], + opt_level: i32, +) -> Result<()> { + #[rustfmt::skip] + const BASE_LL_FREQS: [u32; MAX_LL as usize + 1] = [ + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, + ]; + #[rustfmt::skip] + const BASE_OFC_FREQS: [u32; MAX_OFF as usize + 1] = [ + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + ]; + + let compressed_literals = compressed_literals(opt_ptr); + opt_ptr.price_type = OptPrice::Dynamic; + + if opt_ptr.lit_length_sum == 0 { + if src.len() <= ZSTD_PREDEF_THRESHOLD { + opt_ptr.price_type = OptPrice::Predef; + } + + if symbol_costs.huf.repeat_mode == HufRepeat::Valid { + // A Huffman table that has all the values is presumed to come from a dictionary. + opt_ptr.price_type = OptPrice::Dynamic; + + if compressed_literals { + let huf = &symbol_costs.huf.c_table; + opt_ptr.lit_sum = + freqs_from_bit_costs(&mut opt_ptr.lit_freq, MAX_LIT, 11, |lit| { + huf.get_nb_bits(lit) + })?; + } + let fse = &symbol_costs.fse; + opt_ptr.lit_length_sum = + freqs_from_bit_costs(&mut opt_ptr.lit_length_freq, MAX_LL, 10, |ll| { + fse.litlength_c_table.get_max_nb_bits(ll) + })?; + opt_ptr.match_length_sum = + freqs_from_bit_costs(&mut opt_ptr.match_length_freq, MAX_ML, 10, |ml| { + fse.matchlength_c_table.get_max_nb_bits(ml) + })?; + opt_ptr.off_code_sum = + freqs_from_bit_costs(&mut opt_ptr.off_code_freq, MAX_OFF, 10, |of| { + fse.offcode_c_table.get_max_nb_bits(of) + })?; + } else { + if compressed_literals { + let mut lit = MAX_LIT; + hist::count_simple(&mut opt_ptr.lit_freq, &mut lit, src)?; + opt_ptr.lit_sum = downscale_stats( + &mut opt_ptr.lit_freq, + MAX_LIT, + 8, + BaseDirective::Base0Possible, + )?; + } + + opt_ptr.lit_length_freq.write_at(0, &BASE_LL_FREQS)?; + opt_ptr.lit_length_sum = sum_u32(&BASE_LL_FREQS); + + opt_ptr + .match_length_freq + .sub_mut(..=MAX_ML as usize)? + .fill(1); + opt_ptr.match_length_sum = MAX_ML + 1; + + opt_ptr.off_code_freq.write_at(0, &BASE_OFC_FREQS)?; + opt_ptr.off_code_sum = sum_u32(&BASE_OFC_FREQS); + } + } else { + if compressed_literals { + opt_ptr.lit_sum = scale_stats(&mut opt_ptr.lit_freq, MAX_LIT, 12)?; + } + opt_ptr.lit_length_sum = scale_stats(&mut opt_ptr.lit_length_freq, MAX_LL, 11)?; + opt_ptr.match_length_sum = scale_stats(&mut opt_ptr.match_length_freq, MAX_ML, 11)?; + opt_ptr.off_code_sum = scale_stats(&mut opt_ptr.off_code_freq, MAX_OFF, 11)?; + } + + set_base_prices(opt_ptr, opt_level); + Ok(()) +} + +/// `ZSTD_rawLiteralsCost`: the price of `literals`, without that of their length. +#[inline(always)] +fn raw_literals_cost(literals: &[u8], opt_ptr: &OptState, opt_level: i32) -> Result { + let lit_length = literals.len() as u32; + if lit_length == 0 { + return Ok(0); + } + + if !compressed_literals(opt_ptr) { + // 8 bits per literal. + return Ok((lit_length << 3).wrapping_mul(BITCOST_MULTIPLIER)); + } + + if opt_ptr.price_type == OptPrice::Predef { + // 6 bits per literal. + return Ok(lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER)); + } + + let mut price = opt_ptr.lit_sum_base_price.wrapping_mul(lit_length); + let lit_price_max = opt_ptr.lit_sum_base_price.wrapping_sub(BITCOST_MULTIPLIER); + for &literal in literals { + let lit_price = weight(opt_ptr.lit_freq.at(usize::from(literal))?, opt_level); + price = price.wrapping_sub(lit_price.min(lit_price_max)); + } + Ok(price) +} + +/// `ZSTD_litLengthPrice`: the price of the symbol for a length of literals. +#[inline(always)] +fn lit_length_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + if opt_ptr.price_type == OptPrice::Predef { + return Ok(weight(lit_length, opt_level)); + } + + // The format has no code for the length of a whole block of the largest size, which is then all literals. It is + // made to cost 1 bit more than the length before it. + let (lit_length, extra) = if lit_length == ZSTD_BLOCKSIZE_MAX as u32 { + (ZSTD_BLOCKSIZE_MAX as u32 - 1, BITCOST_MULTIPLIER) + } else { + (lit_length, 0) + }; + + let ll_code = ll_code(lit_length) as usize; + Ok(extra + .wrapping_add(u32::from(LL_BITS.at(ll_code)?) * BITCOST_MULTIPLIER) + .wrapping_add(opt_ptr.lit_length_sum_base_price) + .wrapping_sub(weight(opt_ptr.lit_length_freq.at(ll_code)?, opt_level))) +} + +/// `ZSTD_getMatchPrice`: the price of the offset and of the match length of a sequence. `off_base` is as in +/// [`SeqStore::store_seq`]. Below level 2, it favours small offsets, which are faster to decompress. +#[inline(always)] +fn get_match_price( + off_base: u32, + match_length: u32, + opt_ptr: &OptState, + opt_level: i32, +) -> Result { + let off_code = highbit32(off_base); + let ml_base = match_length.wrapping_sub(MINMATCH); + + if opt_ptr.price_type == OptPrice::Predef { + return Ok(weight(ml_base, opt_level).wrapping_add((16 + off_code) * BITCOST_MULTIPLIER)); + } + + let off_code_weight = weight(opt_ptr.off_code_freq.at(off_code as usize)?, opt_level); + let mut price = (off_code * BITCOST_MULTIPLIER).wrapping_add( + opt_ptr + .off_code_sum_base_price + .wrapping_sub(off_code_weight), + ); + if opt_level < 2 && off_code >= 20 { + price = price.wrapping_add((off_code - 19) * 2 * BITCOST_MULTIPLIER); + } + + let ml_code = ml_code(ml_base) as usize; + let ml_code_weight = weight(opt_ptr.match_length_freq.at(ml_code)?, opt_level); + price = price + .wrapping_add(u32::from(ML_BITS.at(ml_code)?) * BITCOST_MULTIPLIER) + .wrapping_add( + opt_ptr + .match_length_sum_base_price + .wrapping_sub(ml_code_weight), + ); + + // Matches cost a bit more, to have fewer sequences, which is faster to decompress. + Ok(price.wrapping_add(BITCOST_MULTIPLIER / 5)) +} + +/// `ZSTD_updateStats` +#[inline(always)] +fn update_stats( + opt_ptr: &mut OptState, + literals: &[u8], + off_base: u32, + match_length: u32, +) -> Result<()> { + #[inline(always)] + fn increment(freqs: &mut [u32], code: u32, sum: &mut u32) -> Result<()> { + let freq = freqs.at_mut(code as usize)?; + *freq = freq.wrapping_add(1); + *sum = sum.wrapping_add(1); + Ok(()) + } + + let lit_length = literals.len() as u32; + if compressed_literals(opt_ptr) { + for &literal in literals { + let freq = opt_ptr.lit_freq.at_mut(usize::from(literal))?; + *freq = freq.wrapping_add(ZSTD_LITFREQ_ADD); + } + opt_ptr.lit_sum = opt_ptr + .lit_sum + .wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD)); + } + + increment( + &mut opt_ptr.lit_length_freq, + ll_code(lit_length), + &mut opt_ptr.lit_length_sum, + )?; + increment( + &mut opt_ptr.off_code_freq, + highbit32(off_base), + &mut opt_ptr.off_code_sum, + )?; + increment( + &mut opt_ptr.match_length_freq, + ml_code(match_length.wrapping_sub(MINMATCH)), + &mut opt_ptr.match_length_sum, + ) +} + +/// `ZSTD_readMINMATCH`: `length` bytes, 3 or 4, to compare with others. Reads 4. +#[inline(always)] +fn read_minmatch(buf: &[u8], pos: usize, length: u32) -> Result { + let value = read_le32(buf, pos)?; + Ok(if length == 3 { value << 8 } else { value }) +} + +/// `ZSTD_insertAndFindFirstIndexHash3`: indexes the 3-byte hashes up to `ip`, which is not included, and looks it up. +/// All of it is in the current segment. +fn insert_and_find_first_index_hash3( + ms: &mut MatchState, + next_to_update3: &mut u32, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + let hash_table3 = ms.hash_table3.as_mut_slice(); + let hash_log3 = ms.hash_log3; + let target = prefix.index_of(ip); + let hash3 = hash3_ptr(prefix.buf, ip, hash_log3)?; + + for idx in *next_to_update3..target { + hash_table3.put(hash3_ptr(prefix.buf, prefix.pos_of(idx), hash_log3)?, idx)?; + } + + *next_to_update3 = target; + hash_table3.at(hash3) +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `*ptr = value`, where `ptr` is an element of `bt`, or `None` for a place that nothing reads. +#[inline(always)] +fn store(bt: &mut [u32], ptr: Option, value: u32) -> Result<()> { + match ptr { + Some(ptr) => bt.put(ptr, value), + None => Ok(()), + } +} + +/// `ZSTD_insertBt1`: adds the position `ip`, which is at most `iend - 8`, to the tree. `target` is up to where the +/// caller is filling it, and `dict` the older segment, which is looked at if `ext_dict`. Returns how many positions +/// to move forward. +#[inline(always)] +fn insert_bt1( + ms: &mut MatchState, + prefix: Prefix<'_>, + dict: Prefix<'_>, + ip: usize, + iend: usize, + target: u32, + mls: u32, + ext_dict: bool, +) -> Result { + let c_params = ms.c_params; + let h = hash_ptr(prefix.buf, ip, c_params.hash_log, mls)?; + let bt_log = c_params.chain_log.wrapping_sub(1); + let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = dict.pos_of(dict_limit); + let prefix_start = prefix.pos_of(dict_limit); + let curr = prefix.index_of(ip); + let bt_low = curr.saturating_sub(bt_mask); + let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); + let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); + // Only the positions that are still in the window once the tree is filled are of use. + let window_low = ms.get_lowest_match_index(target, c_params.window_log); + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut best_length = 8usize; + let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); + + let bt = ms.chain_table.as_mut_slice(); + let hash_entry = ms.hash_table.at_mut(h)?; + let mut match_index = *hash_entry; + *hash_entry = curr; + + while nb_compares != 0 && match_index >= window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + // What is sure to be common. + let mut match_length = common_length_smaller.min(common_length_larger); + + // A position of the older segment, as one of the current segment, only makes sense with a length that + // brings it there. + let mut match_buf = prefix.buf; + let mut match_pos = prefix.pos_of(match_index); + if !ext_dict || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + prefix.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + prefix.buf, + ip + match_length, + dict.buf, + dict.pos_of(match_index) + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + match_buf = dict.buf; + match_pos = dict.pos_of(match_index); + } + } + + if match_length > best_length { + best_length = match_length; + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + } + + if ip + match_length == iend { + // They are equal: there is no telling which is smaller. Dropping it keeps the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < prefix.buf.at(ip + match_length)? { + // All the smaller ones will have at least this much in common. + store(bt, smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. + smaller_ptr = None; + break; + } + smaller_ptr = Some(next_ptr + 1); + match_index = bt.at(next_ptr + 1)?; + } else { + store(bt, larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = None; + break; + } + larger_ptr = Some(next_ptr); + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + store(bt, smaller_ptr, 0)?; + store(bt, larger_ptr, 0)?; + + let positions = if best_length > 384 { + 192.min((best_length - 384) as u32) + } else { + 0 + }; + Ok(positions.max(match_end_idx.wrapping_sub(curr.wrapping_add(8)))) +} + +/// `ZSTD_updateTree_internal` +#[inline(always)] +fn update_tree_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + dict: Prefix<'_>, + ip: usize, + iend: usize, + mls: u32, + dict_mode: DictMode, +) -> Result<()> { + let target = prefix.index_of(ip); + let mut idx = ms.next_to_update; + + while idx < target { + let forward = insert_bt1( + ms, + prefix, + dict, + prefix.pos_of(idx), + iend, + target, + mls, + dict_mode == DictMode::ExtDict, + )?; + idx = idx.wrapping_add(forward); + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. +/// Matches are not compared past the position `iend`. +pub(crate) fn update_tree( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + iend: usize, +) -> Result<()> { + let no_dict = Prefix { buf: &[], base: 0 }; + let mls = ms.c_params.min_match; + update_tree_internal(ms, prefix, no_dict, ip, iend, mls, DictMode::NoDict) +} + +/// `ZSTD_insertBtAndGetAllMatches`: adds the position `ip` to the tree, and stores in `matches` the matches found on +/// the way, each longer than the one before and all at least `length_to_beat` long. `ll0` is whether no literals +/// come before `ip`. Returns how many there are. +#[inline(always)] +fn insert_bt_and_get_all_matches( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + dict_mode: DictMode, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, + mls: u32, +) -> Result { + let buf = src.buf; + let c_params = ms.c_params; + let sufficient_len = c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); + let curr = src.index_of(ip); + let min_match = if mls == 3 { 3 } else { 4 }; + let h = hash_ptr(buf, ip, c_params.hash_log, mls)?; + let bt_log = c_params.chain_log.wrapping_sub(1); + let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt_low = curr.saturating_sub(bt_mask); + let window_low = ms.get_lowest_match_index(curr, c_params.window_log); + let match_low = window_low.max(1); + let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); + let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); + // The farthest position any match refers to, which detects repetitive patterns. + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut mnum = 0u32; + let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); + + let dms = if dict_mode == DictMode::DictMatchState { + Some(src.dms.ok_or(BUG)?) + } else { + None + }; + let dms_high_limit = dms.map_or(0, |dms| dms.end.wrapping_sub(dms.base) as u32); + let dms_low_limit = dms.map_or(0, |dms| dms.ms.window.low_limit); + let dms_index_delta = dms.map_or(0, |_| window_low.wrapping_sub(dms_high_limit)); + + let mut best_length = length_to_beat.wrapping_sub(1) as usize; + + // The repeat offsets. + let ll0 = u32::from(ll0); + let [rep0, _, _] = *rep; + for rep_code in ll0..ZSTD_REP_NUM as u32 + ll0 { + let rep_offset = match rep.get(rep_code as usize) { + Some(&offset) => offset, + None => rep0.wrapping_sub(1), + }; + let rep_index = curr.wrapping_sub(rep_offset); + let mut rep_len = 0u32; + // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. + if rep_offset.wrapping_sub(1) < curr.wrapping_sub(dict_limit) { + // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. + let rep_match = ip.wrapping_sub(rep_offset as usize); + if rep_index >= window_low + && read_minmatch(buf, ip, min_match)? == read_minmatch(buf, rep_match, min_match)? + { + rep_len = count( + buf, + ip + min_match as usize, + buf, + rep_match + min_match as usize, + i_limit, + )? as u32 + + min_match; + } + } else if dict_mode == DictMode::ExtDict { + // `curr > rep_index >= window_low` + if rep_offset.wrapping_sub(1) < curr.wrapping_sub(window_low) + && index_overlap_check(dict_limit, rep_index) + { + let rep_match = src.dict_pos_of(rep_index); + if read_minmatch(buf, ip, min_match)? + == read_minmatch(src.dict_buf, rep_match, min_match)? + { + rep_len = count_2segments( + buf, + ip + min_match as usize, + src.dict_buf, + rep_match + min_match as usize, + i_limit, + dict_end, + prefix_start, + )? as u32 + + min_match; + } + } + } else if let Some(dms) = dms { + // `curr > rep_index >= dms_low_limit` + if rep_offset.wrapping_sub(1) + < curr.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta)) + && index_overlap_check(dict_limit, rep_index) + { + let rep_match = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); + if read_minmatch(buf, ip, min_match)? + == read_minmatch(dms.buf, rep_match, min_match)? + { + rep_len = count_2segments( + buf, + ip + min_match as usize, + dms.buf, + rep_match + min_match as usize, + i_limit, + dms.end, + prefix_start, + )? as u32 + + min_match; + } + } + } + if rep_len as usize > best_length { + best_length = rep_len as usize; + matches.put( + mnum as usize, + Match { + off: rep_code - ll0 + 1, + len: rep_len, + }, + )?; + mnum += 1; + if rep_len > sufficient_len || ip + rep_len as usize == i_limit { + // The best there can be. + return Ok(mnum); + } + } + } + + // The table of 3-byte hashes. That of a dictionary is not filled. + if mls == 3 && best_length < mls as usize { + let match_index3 = + insert_and_find_first_index_hash3(ms, next_to_update3, src.prefix(), ip)?; + // Farther than that is likely to cost too much. + if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) { + let mlen = if dict_mode != DictMode::ExtDict || match_index3 >= dict_limit { + count(buf, ip, buf, src.pos_of(match_index3), i_limit)? + } else { + count_2segments( + buf, + ip, + src.dict_buf, + src.dict_pos_of(match_index3), + i_limit, + dict_end, + prefix_start, + )? + }; + + if mlen >= mls as usize { + best_length = mlen; + matches.put( + 0, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index3)), + len: mlen as u32, + }, + )?; + mnum = 1; + if mlen > sufficient_len as usize || ip + mlen == i_limit { + // The best there can be: `ip` is not inserted. + ms.next_to_update = curr.wrapping_add(1); + return Ok(1); + } + } + } + } + + let bt = ms.chain_table.as_mut_slice(); + let hash_entry = ms.hash_table.at_mut(h)?; + let mut match_index = *hash_entry; + *hash_entry = curr; + + while nb_compares != 0 && match_index >= match_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + // What is sure to be common. + let mut match_length = common_length_smaller.min(common_length_larger); + + // A position of the older segment, as one of the current segment, only makes sense with a length that + // brings it there. + let mut match_buf = buf; + let mut match_pos = src.pos_of(match_index); + if dict_mode != DictMode::ExtDict + || match_index as usize + match_length >= dict_limit as usize + { + match_length += count( + buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + i_limit, + )?; + } else { + match_length += count_2segments( + buf, + ip + match_length, + src.dict_buf, + src.dict_pos_of(match_index) + match_length, + i_limit, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + match_buf = src.dict_buf; + match_pos = src.dict_pos_of(match_index); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + best_length = match_length; + matches.put( + mnum as usize, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index)), + len: match_length as u32, + }, + )?; + mnum += 1; + if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { + // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. + if dict_mode == DictMode::DictMatchState { + // The dictionary is not to be searched either. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < buf.at(ip + match_length)? { + // All the smaller ones will have at least this much in common. + store(bt, smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. + smaller_ptr = None; + break; + } + smaller_ptr = Some(next_ptr + 1); + match_index = bt.at(next_ptr + 1)?; + } else { + store(bt, larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = None; + break; + } + larger_ptr = Some(next_ptr); + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + store(bt, smaller_ptr, 0)?; + store(bt, larger_ptr, 0)?; + + if let Some(dms) = dms + && nb_compares != 0 + { + let dms_c_params = &dms.ms.c_params; + let dms_bt_log = dms_c_params.chain_log.wrapping_sub(1); + let dms_bt_mask = 1u32.wrapping_shl(dms_bt_log).wrapping_sub(1); + let dms_bt_low = if dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit) { + dms_high_limit.wrapping_sub(dms_bt_mask) + } else { + dms_low_limit + }; + let dms_h = hash_ptr(buf, ip, dms_c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(dms_h)?; + let dms_bt = dms.ms.chain_table.as_slice(); + common_length_smaller = 0; + common_length_larger = 0; + while nb_compares != 0 && dict_match_index > dms_low_limit { + let next_ptr = 2 * (dict_match_index & dms_bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + match_length += count_2segments( + buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + i_limit, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + match_index = dict_match_index.wrapping_add(dms_index_delta); + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + best_length = match_length; + matches.put( + mnum as usize, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index)), + len: match_length as u32, + }, + )?; + mnum += 1; + if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { + break; + } + } + + if dict_match_index <= dms_bt_low { + // Beyond the tree. + break; + } + // A match that reaches the end of the dictionary goes on at the start of the current segment. + let match_byte = if dict_match_index as usize + match_length >= dms_high_limit as usize + { + let match_pos = src.pos_of(dict_match_index.wrapping_add(dms_index_delta)); + buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dms_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dms_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + } + + // Repetitive patterns are skipped. + ms.next_to_update = match_end_idx.wrapping_sub(8); + Ok(mnum) +} + +/// `ZSTD_getAllMatchesFn`: the arguments are `matches`, `ms`, `next_to_update3`, `src`, `ip`, `i_high_limit`, `rep`, +/// `ll0` and `length_to_beat`. +type GetAllMatchesFn = fn( + &mut [Match], + &mut MatchState, + &mut u32, + &BlockSrc<'_>, + usize, + usize, + &[u32; ZSTD_REP_NUM], + bool, + u32, +) -> Result; + +/// `ZSTD_btGetAllMatches_internal` +#[inline(always)] +fn bt_get_all_matches_internal( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, + dict_mode: DictMode, + mls: u32, +) -> Result { + if src.index_of(ip) < ms.next_to_update { + // An area that is skipped. + return Ok(0); + } + let dict = Prefix { + buf: src.dict_buf, + base: src.dict_base, + }; + update_tree_internal(ms, src.prefix(), dict, ip, i_high_limit, mls, dict_mode)?; + insert_bt_and_get_all_matches( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + dict_mode, + rep, + ll0, + length_to_beat, + mls, + ) +} + +/// `ZSTD_btGetAllMatches_noDict_3` to `ZSTD_btGetAllMatches_noDict_6` +fn bt_get_all_matches_no_dict( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::NoDict, + MLS, + ) +} + +/// `ZSTD_btGetAllMatches_extDict_3` to `ZSTD_btGetAllMatches_extDict_6` +fn bt_get_all_matches_ext_dict( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::ExtDict, + MLS, + ) +} + +/// `ZSTD_btGetAllMatches_dictMatchState_3` to `ZSTD_btGetAllMatches_dictMatchState_6` +fn bt_get_all_matches_dict_match_state( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::DictMatchState, + MLS, + ) +} + +/// `ZSTD_selectBtGetAllMatches` +fn select_bt_get_all_matches(ms: &MatchState, dict_mode: DictMode) -> GetAllMatchesFn { + match (dict_mode, ms.c_params.min_match) { + (DictMode::NoDict, ..=3) => bt_get_all_matches_no_dict::<3>, + (DictMode::NoDict, 4) => bt_get_all_matches_no_dict::<4>, + (DictMode::NoDict, 5) => bt_get_all_matches_no_dict::<5>, + (DictMode::NoDict, 6..) => bt_get_all_matches_no_dict::<6>, + (DictMode::ExtDict, ..=3) => bt_get_all_matches_ext_dict::<3>, + (DictMode::ExtDict, 4) => bt_get_all_matches_ext_dict::<4>, + (DictMode::ExtDict, 5) => bt_get_all_matches_ext_dict::<5>, + (DictMode::ExtDict, 6..) => bt_get_all_matches_ext_dict::<6>, + (DictMode::DictMatchState, ..=3) => bt_get_all_matches_dict_match_state::<3>, + (DictMode::DictMatchState, 4) => bt_get_all_matches_dict_match_state::<4>, + (DictMode::DictMatchState, 5) => bt_get_all_matches_dict_match_state::<5>, + (DictMode::DictMatchState, 6..) => bt_get_all_matches_dict_match_state::<6>, + } +} + +// ── candidates from long distance matching ──────────────────────────────── + +/// `ZSTD_optLdm_t`: what it takes to decide whether to use a long distance match. +struct OptLdm<'a> { + /// `seqStore.seq`: the `seqStore.size` candidates for this block and the next ones. + seq: &'a [RawSeq], + /// `seqStore.pos` + pos: usize, + /// `seqStore.posInSequence` + pos_in_sequence: usize, + /// Where the current candidate starts. + start_pos_in_block: u32, + /// Where it ends. + end_pos_in_block: u32, + offset: u32, +} + +impl<'a> OptLdm<'a> { + fn new(seq_store: Option<&'a RawSeqStore>) -> Self { + Self { + seq: seq_store + .and_then(|store| store.seq.get(..store.size)) + .unwrap_or_default(), + pos: seq_store.map_or(0, |store| store.pos), + pos_in_sequence: seq_store.map_or(0, |store| store.pos_in_sequence), + start_pos_in_block: 0, + end_pos_in_block: 0, + offset: 0, + } + } + + /// `ZSTD_optLdm_skipRawSeqStoreBytes`: moves forward by `nb_bytes`. + fn skip_raw_seq_store_bytes(&mut self, nb_bytes: usize) { + let mut curr_pos = self.pos_in_sequence.wrapping_add(nb_bytes) as u32; + while curr_pos != 0 { + let Some(curr_seq) = self.seq.get(self.pos) else { + break; + }; + let seq_length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); + if curr_pos >= seq_length { + curr_pos -= seq_length; + self.pos += 1; + } else { + self.pos_in_sequence = curr_pos as usize; + break; + } + } + if curr_pos == 0 || self.pos == self.seq.len() { + self.pos_in_sequence = 0; + } + } + + /// `ZSTD_opt_getNextMatchAndUpdateSeqStore`: finds where the next candidate starts and ends in the block, and + /// moves past it. + fn get_next_match_and_update_seq_store( + &mut self, + curr_pos_in_block: u32, + block_bytes_remaining: u32, + ) { + let Some(&curr_seq) = self.seq.get(self.pos) else { + // No position gets there, so that there is no candidate in this block. + self.start_pos_in_block = u32::MAX; + self.end_pos_in_block = u32::MAX; + return; + }; + // What is left of the literals and of the match. + let pos_in_sequence = self.pos_in_sequence as u32; + let curr_block_end_pos = curr_pos_in_block.wrapping_add(block_bytes_remaining); + let literals_bytes_remaining = curr_seq.lit_length.saturating_sub(pos_in_sequence); + let match_bytes_remaining = if literals_bytes_remaining == 0 { + curr_seq + .match_length + .wrapping_sub(pos_in_sequence.wrapping_sub(curr_seq.lit_length)) + } else { + curr_seq.match_length + }; + + if literals_bytes_remaining >= block_bytes_remaining { + // The match starts after the block. + self.start_pos_in_block = u32::MAX; + self.end_pos_in_block = u32::MAX; + self.skip_raw_seq_store_bytes(block_bytes_remaining as usize); + return; + } + + // What is left of a match can be too short to use, which is checked later. + self.start_pos_in_block = curr_pos_in_block.wrapping_add(literals_bytes_remaining); + self.end_pos_in_block = self.start_pos_in_block.wrapping_add(match_bytes_remaining); + self.offset = curr_seq.offset; + + if self.end_pos_in_block > curr_block_end_pos { + // The match ends after the block, which can only use its start. + self.end_pos_in_block = curr_block_end_pos; + self.skip_raw_seq_store_bytes( + curr_block_end_pos.wrapping_sub(curr_pos_in_block) as usize + ); + } else { + self.skip_raw_seq_store_bytes( + literals_bytes_remaining.wrapping_add(match_bytes_remaining) as usize, + ); + } + } + + /// `ZSTD_optLdm_maybeAddMatch`: adds what is left of the candidate at `curr_pos_in_block` to `matches`, if it is + /// long enough and longer than all of them, since they are in order of length. + fn maybe_add_match( + &self, + matches: &mut [Match], + nb_matches: &mut u32, + curr_pos_in_block: u32, + min_match: u32, + ) -> Result<()> { + let pos_diff = curr_pos_in_block.wrapping_sub(self.start_pos_in_block); + let candidate_match_length = self + .end_pos_in_block + .wrapping_sub(self.start_pos_in_block) + .wrapping_sub(pos_diff); + + if curr_pos_in_block < self.start_pos_in_block + || curr_pos_in_block >= self.end_pos_in_block + || candidate_match_length < min_match + { + return Ok(()); + } + + if *nb_matches == 0 + || (candidate_match_length > matches.at(*nb_matches as usize - 1)?.len + && (*nb_matches as usize) < ZSTD_OPT_NUM) + { + matches.put( + *nb_matches as usize, + Match { + off: offset_to_offbase(self.offset), + len: candidate_match_length, + }, + )?; + *nb_matches += 1; + } + Ok(()) + } + + /// `ZSTD_optLdm_processMatchCandidate` + #[inline(always)] + fn process_match_candidate( + &mut self, + matches: &mut [Match], + nb_matches: &mut u32, + curr_pos_in_block: u32, + remaining_bytes: u32, + min_match: u32, + ) -> Result<()> { + if self.pos >= self.seq.len() { + return Ok(()); + } + + if curr_pos_in_block >= self.end_pos_in_block { + if curr_pos_in_block > self.end_pos_in_block { + // The parser does not stop at the end of each candidate: it is often some bytes past it. + let pos_overshoot = curr_pos_in_block - self.end_pos_in_block; + self.skip_raw_seq_store_bytes(pos_overshoot as usize); + } + self.get_next_match_and_update_seq_store(curr_pos_in_block, remaining_bytes); + } + self.maybe_add_match(matches, nb_matches, curr_pos_in_block, min_match) + } +} + +// ── the optimal parser ──────────────────────────────────────────────────── + +/// `LIT_PRICE`: the price of the literal at `pos`. +#[inline(always)] +fn lit_price(buf: &[u8], pos: usize, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok(raw_literals_cost(buf.span(pos, 1)?, opt_ptr, opt_level)? as i32) +} + +/// `LL_PRICE` +#[inline(always)] +fn ll_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok(lit_length_price(lit_length, opt_ptr, opt_level)? as i32) +} + +/// `LL_INCPRICE`: what one more literal, which makes `lit_length` of them, adds to the price of their length. +#[inline(always)] +fn ll_inc_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok( + ll_price(lit_length, opt_ptr, opt_level)?.wrapping_sub(ll_price( + lit_length.wrapping_sub(1), + opt_ptr, + opt_level, + )?), + ) +} + +/// `ZSTD_compressBlock_opt_generic` +#[inline(always)] +fn compress_block_opt_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + opt_level: i32, + dict_mode: DictMode, +) -> Result { + // The match finders borrow the whole of `ms`. + let mut price_table = mem::take(&mut ms.opt.price_table); + let mut match_table = mem::take(&mut ms.opt.match_table); + let result = parse_block( + ms, + seq_store, + rep, + src, + &mut price_table, + &mut match_table, + opt_level, + dict_mode, + ); + ms.opt.price_table = price_table; + ms.opt.match_table = match_table; + result +} + +/// The body of `ZSTD_compressBlock_opt_generic`. `opt` is `ms.opt.price_table` and `matches` is `ms.opt.match_table`. +#[inline(always)] +fn parse_block( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + opt: &mut [Optimal], + matches: &mut [Match], + opt_level: i32, + dict_mode: DictMode, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let ilimit = iend.saturating_sub(8); + let prefix_start = src.pos_of(ms.window.dict_limit); + + let get_all_matches = select_bt_get_all_matches(ms, dict_mode); + + let sufficient_len = ms.c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); + let min_match = if ms.c_params.min_match == 3 { 3 } else { 4 }; + let mut next_to_update3 = ms.next_to_update; + + let mut last_stretch = Optimal::default(); + let mut opt_ldm = OptLdm::new(src.ldm_seq_store); + opt_ldm.get_next_match_and_update_seq_store(0, (iend - ip) as u32); + + rescale_freqs( + &mut ms.opt, + src.symbol_costs, + buf.sub(istart..iend)?, + opt_level, + )?; + ip += usize::from(ip == prefix_start); + + 'series: while ip < ilimit { + let mut last_pos: u32; + + // The forward pass. `opt` holds stretches, each a match followed by literals, unlike a sequence, which is + // literals followed by a match: that way each position of a run of literals can have another match before it. + // It ends with the last stretch in `last_stretch`, which starts at `cur` and ends at `last_pos`. + let mut cur = 'forward: { + // Find a first match. + { + let litlen = (ip - anchor) as u32; + let mut nb_matches = get_all_matches( + matches, + ms, + &mut next_to_update3, + src, + ip, + iend, + rep, + litlen == 0, + min_match, + )?; + opt_ldm.process_match_candidate( + matches, + &mut nb_matches, + (ip - istart) as u32, + (iend - ip) as u32, + min_match, + )?; + if nb_matches == 0 { + ip += 1; + continue 'series; + } + let matches = matches.sub(..nb_matches as usize)?; + + // The price of the literals before the first match is the same whatever comes next, so it is left + // out. But how the price of their length changes depends on their length. + let first_price = ll_price(litlen, &ms.opt, opt_level)?; + { + let first = opt.at_mut(0)?; + first.mlen = 0; + first.litlen = litlen; + first.price = first_price; + first.rep = *rep; + } + + // A large match is encoded at once. + let longest = matches.at(matches.len() - 1)?; + if longest.len > sufficient_len { + last_stretch.litlen = 0; + last_stretch.mlen = longest.len; + last_stretch.off = longest.off; + last_pos = longest.len; + break 'forward 0; + } + + // Set the prices of the first matches. + let ll0_price = ll_price(0, &ms.opt, opt_level)?; + let mut pos = 1u32; + while pos < min_match { + let stretch = opt.at_mut(pos as usize)?; + stretch.price = ZSTD_MAX_PRICE; + stretch.mlen = 0; + stretch.litlen = litlen.wrapping_add(pos); + pos += 1; + } + for candidate in matches { + while pos <= candidate.len { + let match_price = + get_match_price(candidate.off, pos, &ms.opt, opt_level)? as i32; + let sequence_price = first_price.wrapping_add(match_price); + let stretch = opt.at_mut(pos as usize)?; + stretch.mlen = pos; + stretch.off = candidate.off; + stretch.litlen = 0; + stretch.price = sequence_price.wrapping_add(ll0_price); + pos += 1; + } + } + last_pos = pos - 1; + opt.at_mut(pos as usize)?.price = ZSTD_MAX_PRICE; + } + + // Check the positions that follow. + let mut cur = 0u32; + while cur < last_pos { + cur += 1; + let inr = ip + cur as usize; + + // Get to this position by a literal, if it is cheaper. + { + let before = opt.at(cur as usize - 1)?; + let litlen = before.litlen.wrapping_add(1); + let price = before + .price + .wrapping_add(lit_price(buf, inr - 1, &ms.opt, opt_level)?) + .wrapping_add(ll_inc_price(litlen, &ms.opt, opt_level)?); + let prev_match = opt.at(cur as usize)?; + if price <= prev_match.price { + opt.put( + cur as usize, + Optimal { + litlen, + price, + ..before + }, + )?; + // If this replaces a match, and 1 literal is cheaper than none, the match followed by a + // literal may be a cheaper way to the next position. + if opt_level >= 1 + && prev_match.litlen == 0 + && ll_inc_price(1, &ms.opt, opt_level)? < 0 + && inr < iend + { + let next_lit_price = lit_price(buf, inr, &ms.opt, opt_level)?; + let with_1_literal = prev_match + .price + .wrapping_add(next_lit_price) + .wrapping_add(ll_inc_price(1, &ms.opt, opt_level)?); + let with_more_literals = price + .wrapping_add(next_lit_price) + .wrapping_add(ll_inc_price( + litlen.wrapping_add(1), + &ms.opt, + opt_level, + )?); + if with_1_literal < with_more_literals + && with_1_literal < opt.at(cur as usize + 1)?.price + { + let prev = opt.at(cur.wrapping_sub(prev_match.mlen) as usize)?; + opt.put( + cur as usize + 1, + Optimal { + rep: new_rep(&prev.rep, prev_match.off, prev.litlen == 0), + litlen: 1, + price: with_1_literal, + ..prev_match + }, + )?; + last_pos = last_pos.max(cur + 1); + } + } + } + } + + // The repeat offsets are not updated while matches are compared, but here, now that one is selected. + let mut here = opt.at(cur as usize)?; + if here.litlen == 0 { + let prev = opt.at(cur.wrapping_sub(here.mlen) as usize)?; + here.rep = new_rep(&prev.rep, here.off, prev.litlen == 0); + opt.at_mut(cur as usize)?.rep = here.rep; + } + + // The last match starts at least 8 bytes before the end. + if inr > ilimit { + continue; + } + + if cur == last_pos { + break; + } + + if opt_level == 0 + && opt.at(cur as usize + 1)?.price + <= here.price.wrapping_add(BITCOST_MULTIPLIER as i32 / 2) + { + // The next position is cheaper to get to: this one is not promising. + continue; + } + + { + let base_price = here.price.wrapping_add(ll_price(0, &ms.opt, opt_level)?); + let mut nb_matches = get_all_matches( + matches, + ms, + &mut next_to_update3, + src, + inr, + iend, + &here.rep, + here.litlen == 0, + min_match, + )?; + opt_ldm.process_match_candidate( + matches, + &mut nb_matches, + (inr - istart) as u32, + (iend - inr) as u32, + min_match, + )?; + if nb_matches == 0 { + continue; + } + let matches = matches.sub(..nb_matches as usize)?; + + let longest = matches.at(matches.len() - 1)?; + if longest.len > sufficient_len + || cur.wrapping_add(longest.len) as usize >= ZSTD_OPT_NUM + || inr + longest.len as usize >= iend + { + last_stretch.mlen = longest.len; + last_stretch.off = longest.off; + last_stretch.litlen = 0; + last_pos = cur.wrapping_add(longest.len); + break 'forward cur; + } + + // Set the prices that the matches found here give. + let mut start_ml = min_match; + for candidate in matches { + let mut mlen = candidate.len; + while mlen >= start_ml { + let pos = cur + mlen; + let price = base_price.wrapping_add(get_match_price( + candidate.off, + mlen, + &ms.opt, + opt_level, + )? + as i32); + + if pos > last_pos || price < opt.at(pos as usize)?.price { + while last_pos < pos { + // Positions that nothing leads to yet. + last_pos += 1; + let stretch = opt.at_mut(last_pos as usize)?; + stretch.price = ZSTD_MAX_PRICE; + // Not 0, which would be the end of a match. + stretch.litlen = 1; + } + let stretch = opt.at_mut(pos as usize)?; + stretch.mlen = mlen; + stretch.off = candidate.off; + stretch.litlen = 0; + stretch.price = price; + } else if opt_level == 0 { + // Shorter lengths are not tried. + break; + } + mlen -= 1; + } + start_ml = candidate.len + 1; + } + } + opt.at_mut(last_pos as usize + 1)?.price = ZSTD_MAX_PRICE; + } + + last_stretch = opt.at(last_pos as usize)?; + last_pos.wrapping_sub(last_stretch.mlen) + }; + + if last_stretch.mlen == 0 { + // All the matches have been turned into literals. + ip += last_pos as usize; + continue; + } + + if last_stretch.litlen == 0 { + // It ends with a match. + let prev = opt.at(cur as usize)?; + *rep = new_rep(&prev.rep, last_stretch.off, prev.litlen == 0); + } else { + *rep = last_stretch.rep; + cur = cur.wrapping_sub(last_stretch.litlen); + } + + // Write the cheapest path in `opt`, backwards from `store_end`, over the stretches that are not on it, and as + // sequences. The literals that end the last stretch are in no sequence: they are parsed again. (The C code + // starts by making them a last sequence without a match, and overwrites it at once.) + let store_end = cur.wrapping_add(2) as usize; + let mut store_start = store_end; + let mut stretch_pos = cur; + opt.put(store_end, last_stretch)?; + loop { + let next_stretch = opt.at(stretch_pos as usize)?; + opt.at_mut(store_start)?.litlen = next_stretch.litlen; + if next_stretch.mlen == 0 { + // The start of the series. + break; + } + store_start = store_start.wrapping_sub(1); + opt.put(store_start, next_stretch)?; + stretch_pos = + stretch_pos.wrapping_sub(next_stretch.litlen.wrapping_add(next_stretch.mlen)); + } + + for sequence in opt.sub(store_start..=store_end)? { + let llen = sequence.litlen as usize; + let mlen = sequence.mlen as usize; + update_stats( + &mut ms.opt, + buf.span(anchor, llen)?, + sequence.off, + sequence.mlen, + )?; + seq_store.store_seq(llen, buf, anchor, sequence.off, mlen)?; + anchor += llen + mlen; + ip = anchor; + } + + set_base_prices(&mut ms.opt, opt_level); + } + + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_opt0` +fn compress_block_opt0( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode) +} + +/// `ZSTD_compressBlock_opt2` +fn compress_block_opt2( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode) +} + +/// `ZSTD_initStats_ultra`: a first pass over the block, only to have better statistics to start from. It has to be the +/// first block, with no dictionary and no long distance matching. Returns the block as the window then sees it, since +/// its indices change. +fn init_stats_ultra<'a>( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &[u32; ZSTD_REP_NUM], + src: &BlockSrc<'a>, +) -> Result> { + // The repeat offsets it leaves are of no use. + let mut tmp_rep = *rep; + compress_block_opt2(ms, seq_store, &mut tmp_rep, src, DictMode::NoDict)?; + + // Of the first pass, only the statistics are kept: what the tables have is made out of reach. + let src_size = src.end - src.start; + seq_store.reset(); + ms.window.base = ms.window.base.wrapping_sub(src_size); + ms.window.dict_limit = ms.window.dict_limit.wrapping_add(src_size as u32); + ms.window.low_limit = ms.window.dict_limit; + ms.next_to_update = ms.window.dict_limit; + Ok(BlockSrc { + base: src.base.wrapping_sub(src_size), + ..*src + }) +} + +/// `ZSTD_compressBlock_btultra2` +fn compress_block_btultra2( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let curr = src.index_of(src.start); + + // The first block is parsed twice, the first time to collect statistics for the second. Since the first pass is + // then forgotten, nothing must be in the tables before it: no dictionary, no prefix, no long distance matching. + if ms.opt.lit_length_sum == 0 + && seq_store.sequences.is_empty() + && ms.window.dict_limit == ms.window.low_limit + && curr == ms.window.dict_limit + && src.end - src.start > ZSTD_PREDEF_THRESHOLD + { + let src = init_stats_ultra(ms, seq_store, rep, src)?; + return compress_block_opt2(ms, seq_store, rep, &src, DictMode::NoDict); + } + + compress_block_opt2(ms, seq_store, rep, src, DictMode::NoDict) +} + +/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` +/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. +/// +/// `btultra2` can move `ms.window.base`, which `src.base` comes from. +pub(crate) fn compress_block_opt( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + dict_mode: DictMode, +) -> Result { + match (strategy, dict_mode) { + (Strategy::BtUltra2, DictMode::NoDict) => compress_block_btultra2(ms, seq_store, rep, src), + (Strategy::BtUltra | Strategy::BtUltra2, _) => { + compress_block_opt2(ms, seq_store, rep, src, dict_mode) + } + _ => compress_block_opt0(ms, seq_store, rep, src, dict_mode), + } +} diff --git a/src/zstd/wip/work/ldm/stubs/double_fast.rs b/src/zstd/wip/work/ldm/stubs/double_fast.rs new file mode 100644 index 000000000000..65d401675d1b --- /dev/null +++ b/src/zstd/wip/work/ldm/stubs/double_fast.rs @@ -0,0 +1,48 @@ +//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_double_hash_table( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _end: usize, + _dtlm: DictTableLoadMethod, + _tfp: TableFillPurpose, +) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast` +pub(crate) fn compress_block_double_fast( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState` +pub(crate) fn compress_block_double_fast_dict_match_state( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast_extDict` +pub(crate) fn compress_block_double_fast_ext_dict( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/ldm/stubs/fast.rs b/src/zstd/wip/work/ldm/stubs/fast.rs new file mode 100644 index 000000000000..1f4788af28b4 --- /dev/null +++ b/src/zstd/wip/work/ldm/stubs/fast.rs @@ -0,0 +1,48 @@ +//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_hash_table( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _end: usize, + _dtlm: DictTableLoadMethod, + _tfp: TableFillPurpose, +) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast` +pub(crate) fn compress_block_fast( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast_dictMatchState` +pub(crate) fn compress_block_fast_dict_match_state( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast_extDict` +pub(crate) fn compress_block_fast_ext_dict( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/ldm/stubs/lazy.rs b/src/zstd/wip/work/ldm/stubs/lazy.rs new file mode 100644 index 000000000000..9f653fd684f3 --- /dev/null +++ b/src/zstd/wip/work/ldm/stubs/lazy.rs @@ -0,0 +1,34 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _ip: usize, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_row_update`: the same, in the rows of the row-based match finder. +pub(crate) fn row_update(_ms: &mut MatchState, _prefix: Prefix<'_>, _ip: usize) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, + _strategy: Strategy, + _use_row: bool, + _dict_mode: DictMode, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/ldm/stubs/opt.rs b/src/zstd/wip/work/ldm/stubs/opt.rs new file mode 100644 index 000000000000..d055a44cf982 --- /dev/null +++ b/src/zstd/wip/work/ldm/stubs/opt.rs @@ -0,0 +1,29 @@ +//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. + +use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. +/// Matches are not compared past the position `iend`. +pub(crate) fn update_tree( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _ip: usize, + _iend: usize, +) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` +/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. +pub(crate) fn compress_block_opt( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, + _strategy: Strategy, + _dict_mode: DictMode, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/ldm/zstd/compress/double_fast.rs b/src/zstd/wip/work/ldm/zstd/compress/double_fast.rs new file mode 100644 index 000000000000..fbf2d4120cad --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/compress/double_fast.rs @@ -0,0 +1,785 @@ +//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. + +use super::fast::{FAST_HASH_FILL_STEP, Located, count_backwards, restore_rep}; +use super::internal::{ + BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, + write_tagged_index, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, read_le32, read_le64}; + +/// `ZSTD_fillDoubleHashTableForCDict` +fn fill_double_hash_table_for_cdict( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits_l = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let h_bits_s = ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; + let mls = ms.c_params.min_match; + let hash_large = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + // The positions in between only go in the table of long hashes, where there is nothing yet. + while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { + let curr = prefix.index_of(ip); + for i in 0..FAST_HASH_FILL_STEP { + let sm_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; + let lg_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; + if i == 0 { + write_tagged_index(hash_small, sm_hash_and_tag, curr + i)?; + } + if i == 0 || hash_large.at(lg_hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { + write_tagged_index(hash_large, lg_hash_and_tag, curr + i)?; + } + if dtlm == DictTableLoadMethod::Fast { + break; + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillDoubleHashTableForCCtx` +fn fill_double_hash_table_for_cctx( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let mls = ms.c_params.min_match; + let hash_large = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + // The positions in between only go in the table of long hashes, where there is nothing yet. + while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { + let curr = prefix.index_of(ip); + for i in 0..FAST_HASH_FILL_STEP { + let sm_hash = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; + let lg_hash = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; + if i == 0 { + hash_small.put(sm_hash, curr + i)?; + } + if i == 0 || hash_large.at(lg_hash)? == 0 { + hash_large.put(lg_hash, curr + i)?; + } + if dtlm == DictTableLoadMethod::Fast { + break; + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_double_hash_table( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, + tfp: TableFillPurpose, +) -> Result<()> { + match tfp { + TableFillPurpose::ForCDict => fill_double_hash_table_for_cdict(ms, prefix, end, dtlm), + TableFillPurpose::ForCCtx => fill_double_hash_table_for_cctx(ms, prefix, end, dtlm), + } +} + +/// `ZSTD_compressBlock_doubleFast_noDict_generic` +fn compress_block_double_fast_no_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; + + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let window_log = ms.c_params.window_log; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, window_log); + let prefix_lowest = src.pos_of(prefix_lowest_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip = istart + usize::from(istart == prefix_lowest); + let [mut offset_1, mut offset_2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let current = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(current, window_log); + let max_rep = current.wrapping_sub(window_low); + if offset_2 > max_rep { + offset_saved2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved1 = offset_1; + offset_1 = 0; + } + } + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + // One turn per match. + 'start: loop { + let mut step = 1usize; + let mut next_step = ip + K_STEP_INCR; + let mut ip1 = ip + step; + if ip1 > ilimit { + break 'start; + } + + let mut hl0 = hash_ptr(buf, ip, h_bits_l, 8)?; + let mut idxl0 = hash_long.at(hl0)?; + let mut curr; + + let m_length = 'match_stored: { + // One turn per position. Ends with the offset and the length of a match, which `ip` is moved to the start + // of. + let (offset, m_length, hl1) = loop { + let hs0 = hash_ptr(buf, ip, h_bits_s, MLS)?; + let idxs0 = hash_small.at(hs0)?; + curr = src.index_of(ip); + hash_long.put(hl0, curr)?; + hash_small.put(hs0, curr)?; + + if (offset_1 > 0) + & (read_le32(buf, (ip + 1).wrapping_sub(offset_1 as usize))? + == read_le32(buf, ip + 1)?) + { + let m_length = count( + buf, + ip + 1 + 4, + buf, + (ip + 1 + 4).wrapping_sub(offset_1 as usize), + iend, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break 'match_stored m_length; + } + + let hl1 = hash_ptr(buf, ip1, h_bits_l, 8)?; + + let matchl0 = src.pos_of(idxl0); + if idxl0 >= prefix_lowest_index && read_le64(buf, matchl0)? == read_le64(buf, ip)? { + let m_length = count(buf, ip + 8, buf, matchl0 + 8, iend)? + 8; + let offset = ip.wrapping_sub(matchl0) as u32; + let back = count_backwards(buf, ip, anchor, buf, matchl0, prefix_lowest); + ip -= back; + break (offset, m_length + back, hl1); + } + + let idxl1 = hash_long.at(hl1)?; + + let mut matchs0 = src.pos_of(idxs0); + if idxs0 >= prefix_lowest_index && read_le32(buf, matchs0)? == read_le32(buf, ip)? { + // A short match: a longer one at the next position is preferred. + let mut m_length = count(buf, ip + 4, buf, matchs0 + 4, iend)? + 4; + let mut offset = ip.wrapping_sub(matchs0) as u32; + let matchl1 = src.pos_of(idxl1); + if idxl1 > prefix_lowest_index + && read_le64(buf, matchl1)? == read_le64(buf, ip1)? + { + let l1len = count(buf, ip1 + 8, buf, matchl1 + 8, iend)? + 8; + if l1len > m_length { + ip = ip1; + m_length = l1len; + offset = ip.wrapping_sub(matchl1) as u32; + matchs0 = matchl1; + } + } + let back = count_backwards(buf, ip, anchor, buf, matchs0, prefix_lowest); + ip -= back; + break (offset, m_length + back, hl1); + } + + if ip1 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + ip = ip1; + ip1 += step; + + hl0 = hl1; + idxl0 = idxl1; + if ip1 > ilimit { + break 'start; + } + }; + + offset_2 = offset_1; + offset_1 = offset; + // With a larger step, ip1 may be past the end of the match. + if step < 4 { + hash_long.put(hl1, src.index_of(ip1))?; + } + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + m_length + }; + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit + && (offset_2 > 0) + & (read_le32(buf, ip)? == read_le32(buf, ip.wrapping_sub(offset_2 as usize))?) + { + let r_length = count( + buf, + ip + 4, + buf, + (ip + 4).wrapping_sub(offset_2 as usize), + iend, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, src.index_of(ip))?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, src.index_of(ip))?; + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; + ip += r_length; + anchor = ip; + } + } + } + + restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` +fn compress_block_double_fast_dict_match_state_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, ms.c_params.window_log); + let prefix_lowest = src.pos_of(prefix_lowest_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + let [mut offset_1, mut offset_2, _] = *rep; + + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_hash_long = dms.ms.hash_table.as_slice(); + let dict_hash_small = dms.ms.chain_table.as_slice(); + let dict_start_index = dms.ms.window.dict_limit; + let dict_start = dms.pos_of(dict_start_index); + let dict_end = dms.end; + // What to subtract from an index below prefix_lowest_index to get the index in the dictionary. + let dict_index_delta = prefix_lowest_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); + let dict_h_bits_l = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let dict_h_bits_s = dms.ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; + let dict_and_prefix_length = istart + .wrapping_sub(prefix_lowest) + .wrapping_add(dict_end.wrapping_sub(dict_start)) as u32; + + let locate = |index: u32| { + if index < prefix_lowest_index { + Located { + buf: dms.buf, + pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_lowest, + end: iend, + } + } + }; + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip = istart + usize::from(dict_and_prefix_length == 0); + + 'search: while ip < ilimit { + let h2 = hash_ptr(buf, ip, h_bits_l, 8)?; + let h = hash_ptr(buf, ip, h_bits_s, MLS)?; + let dict_hash_and_tag_l = hash_ptr(buf, ip, dict_h_bits_l, 8)?; + let dict_hash_and_tag_s = hash_ptr(buf, ip, dict_h_bits_s, MLS)?; + let dict_match_index_and_tag_l = + dict_hash_long.at(dict_hash_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_match_index_and_tag_s = + dict_hash_small.at(dict_hash_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_tags_match_l = + compare_packed_tags(dict_match_index_and_tag_l as usize, dict_hash_and_tag_l); + let dict_tags_match_s = + compare_packed_tags(dict_match_index_and_tag_s as usize, dict_hash_and_tag_s); + let curr = src.index_of(ip); + let match_index_l = hash_long.at(h2)?; + let mut match_index_s = hash_small.at(h)?; + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + hash_long.put(h2, curr)?; + hash_small.put(h, curr)?; + + let m_length = 'match_stored: { + if index_overlap_check(prefix_lowest_index, rep_index) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? + { + let m_length = count_2segments( + buf, + ip + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_lowest, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break 'match_stored m_length; + } + + // The offset and the length of a match, which `ip` is moved to the start of. + let (offset, m_length) = 'match_found: { + let match_long = src.pos_of(match_index_l); + if match_index_l >= prefix_lowest_index + && read_le64(buf, match_long)? == read_le64(buf, ip)? + { + let m_length = count(buf, ip + 8, buf, match_long + 8, iend)? + 8; + let offset = ip.wrapping_sub(match_long) as u32; + let back = count_backwards(buf, ip, anchor, buf, match_long, prefix_lowest); + ip -= back; + break 'match_found (offset, m_length + back); + } else if dict_tags_match_l { + let dict_match_index_l = + dict_match_index_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match_l = dms.pos_of(dict_match_index_l); + if dict_match_index_l > dict_start_index + && read_le64(dms.buf, dict_match_l)? == read_le64(buf, ip)? + { + let m_length = count_2segments( + buf, + ip + 8, + dms.buf, + dict_match_l + 8, + iend, + dict_end, + prefix_lowest, + )? + 8; + let offset = curr + .wrapping_sub(dict_match_index_l) + .wrapping_sub(dict_index_delta); + let back = + count_backwards(buf, ip, anchor, dms.buf, dict_match_l, dict_start); + ip -= back; + break 'match_found (offset, m_length + back); + } + } + + let short_found = if match_index_s > prefix_lowest_index { + read_le32(buf, src.pos_of(match_index_s))? == read_le32(buf, ip)? + } else if dict_tags_match_s { + let dict_match_index_s = + dict_match_index_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS; + match_index_s = dict_match_index_s.wrapping_add(dict_index_delta); + dict_match_index_s > dict_start_index + && read_le32(dms.buf, dms.pos_of(dict_match_index_s))? + == read_le32(buf, ip)? + } else { + false + }; + if !short_found { + ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; + continue 'search; + } + + // A short match: a long one at the next position is preferred. + let hl3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; + let dict_hash_and_tag_l3 = hash_ptr(buf, ip + 1, dict_h_bits_l, 8)?; + let match_index_l3 = hash_long.at(hl3)?; + let dict_match_index_and_tag_l3 = + dict_hash_long.at(dict_hash_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let dict_tags_match_l3 = + compare_packed_tags(dict_match_index_and_tag_l3 as usize, dict_hash_and_tag_l3); + let match_l3 = src.pos_of(match_index_l3); + hash_long.put(hl3, curr + 1)?; + + if match_index_l3 >= prefix_lowest_index + && read_le64(buf, match_l3)? == read_le64(buf, ip + 1)? + { + let m_length = count(buf, ip + 9, buf, match_l3 + 8, iend)? + 8; + ip += 1; + let offset = ip.wrapping_sub(match_l3) as u32; + let back = count_backwards(buf, ip, anchor, buf, match_l3, prefix_lowest); + ip -= back; + break 'match_found (offset, m_length + back); + } else if dict_tags_match_l3 { + let dict_match_index_l3 = + dict_match_index_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match_l3 = dms.pos_of(dict_match_index_l3); + if dict_match_index_l3 > dict_start_index + && read_le64(dms.buf, dict_match_l3)? == read_le64(buf, ip + 1)? + { + let m_length = count_2segments( + buf, + ip + 1 + 8, + dms.buf, + dict_match_l3 + 8, + iend, + dict_end, + prefix_lowest, + )? + 8; + ip += 1; + let offset = curr + .wrapping_add(1) + .wrapping_sub(dict_match_index_l3) + .wrapping_sub(dict_index_delta); + let back = + count_backwards(buf, ip, anchor, dms.buf, dict_match_l3, dict_start); + ip -= back; + break 'match_found (offset, m_length + back); + } + } + + let match_s = locate(match_index_s); + let m_length = count_2segments( + buf, + ip + 4, + match_s.buf, + match_s.pos + 4, + iend, + match_s.end, + prefix_lowest, + )? + 4; + let offset = curr.wrapping_sub(match_index_s); + let back = + count_backwards(buf, ip, anchor, match_s.buf, match_s.pos, match_s.start); + ip -= back; + (offset, m_length + back) + }; + + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + m_length + }; + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit { + let current2 = src.index_of(ip); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !(index_overlap_check(prefix_lowest_index, rep_index2) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_lowest, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; + ip += rep_length2; + anchor = ip; + } + } + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast` +pub(crate) fn compress_block_double_fast( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_no_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_no_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_no_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_no_dict_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState` +pub(crate) fn compress_block_double_fast_dict_match_state( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_doubleFast_extDict_generic` +fn compress_block_double_fast_ext_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); + let dict_start_index = low_limit; + let dict_limit = ms.window.dict_limit; + let prefix_start_index = dict_limit.max(low_limit); + let prefix_start = src.pos_of(prefix_start_index); + let dict_start = src.dict_pos_of(dict_start_index); + let dict_end = src.dict_pos_of(prefix_start_index); + let [mut offset_1, mut offset_2, _] = *rep; + + // Nothing of the older segment is within reach. + if prefix_start_index == dict_start_index { + return compress_block_double_fast(ms, seq_store, rep, src); + } + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: src.dict_buf, + pos: src.dict_pos_of(index), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip = istart; + + while ip < ilimit { + let h_small = hash_ptr(buf, ip, h_bits_s, MLS)?; + let match_index = hash_small.at(h_small)?; + let match_s = locate(match_index); + + let h_long = hash_ptr(buf, ip, h_bits_l, 8)?; + let match_long_index = hash_long.at(h_long)?; + let match_long = locate(match_long_index); + + let curr = src.index_of(ip); + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + hash_long.put(h_long, curr)?; + hash_small.put(h_small, curr)?; + + let m_length; + if (index_overlap_check(prefix_start_index, rep_index) + & (offset_1 <= curr.wrapping_add(1).wrapping_sub(dict_start_index))) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? + { + m_length = count_2segments( + buf, + ip + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_start, + )? + 4; + ip += 1; + seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + } else { + // The match, how much of it is known, and its offset. `ip` is moved to where it is for. + let (found, known, offset) = if match_long_index > dict_start_index + && read_le64(match_long.buf, match_long.pos)? == read_le64(buf, ip)? + { + (match_long, 8, curr.wrapping_sub(match_long_index)) + } else if match_index > dict_start_index + && read_le32(match_s.buf, match_s.pos)? == read_le32(buf, ip)? + { + // A short match: a long one at the next position is preferred. + let h3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; + let match_index3 = hash_long.at(h3)?; + let match3 = locate(match_index3); + hash_long.put(h3, curr + 1)?; + if match_index3 > dict_start_index + && read_le64(match3.buf, match3.pos)? == read_le64(buf, ip + 1)? + { + ip += 1; + (match3, 8, curr.wrapping_add(1).wrapping_sub(match_index3)) + } else { + (match_s, 4, curr.wrapping_sub(match_index)) + } + } else { + ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; + continue; + }; + let forwards = count_2segments( + buf, + ip + known, + found.buf, + found.pos + known, + iend, + found.end, + prefix_start, + )? + known; + let back = count_backwards(buf, ip, anchor, found.buf, found.pos, found.start); + ip -= back; + m_length = forwards + back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + } + + ip += m_length; + anchor = ip; + + if ip <= ilimit { + let index_to_insert = curr + 2; + let to_insert = src.pos_of(index_to_insert); + hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; + hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; + hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + + while ip <= ilimit { + let current2 = src.index_of(ip); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !((index_overlap_check(prefix_start_index, rep_index2) + & (offset_2 <= current2.wrapping_sub(dict_start_index))) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; + hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; + ip += rep_length2; + anchor = ip; + } + } + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_doubleFast_extDict` +pub(crate) fn compress_block_double_fast_ext_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_double_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_double_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_double_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_double_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), + } +} diff --git a/src/zstd/wip/work/ldm/zstd/compress/fast.rs b/src/zstd/wip/work/ldm/zstd/compress/fast.rs new file mode 100644 index 000000000000..4bec128a6786 --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/compress/fast.rs @@ -0,0 +1,829 @@ +//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, + write_tagged_index, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, read_le32}; + +pub(super) const FAST_HASH_FILL_STEP: u32 = 3; + +/// Where an index leads, when it can be in one of two segments. +#[derive(Clone, Copy)] +pub(super) struct Located<'a> { + /// The buffer that the segment is in. + pub(super) buf: &'a [u8], + /// The position of the index in `buf`. + pub(super) pos: usize, + /// Where the segment starts in `buf`. + pub(super) start: usize, + /// Where it ends. + pub(super) end: usize, +} + +/// How many bytes before `ip`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. +#[inline(always)] +pub(super) fn count_backwards( + buf: &[u8], + ip: usize, + anchor: usize, + match_buf: &[u8], + match_pos: usize, + match_low: usize, +) -> usize { + let before_ip = buf.get(anchor..ip).unwrap_or_default(); + let before_match = match_buf.get(match_low..match_pos).unwrap_or_default(); + before_ip + .iter() + .rev() + .zip(before_match.iter().rev()) + .take_while(|(a, b)| a == b) + .count() +} + +/// The repeat offsets to leave at the end of a block, at the start of which those that reached too far were saved and +/// set to 0. +#[inline(always)] +pub(super) fn restore_rep( + rep: &mut [u32; ZSTD_REP_NUM], + offset_1: u32, + offset_2: u32, + offset_saved1: u32, + offset_saved2: u32, +) { + // If offset_1 started invalid and became valid, offset_saved1 is what rotates into the second place. + let offset_saved2 = if offset_saved1 != 0 && offset_1 != 0 { + offset_saved1 + } else { + offset_saved2 + }; + let [rep0, rep1, _] = rep; + *rep0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved1 + }; + *rep1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved2 + }; +} + +/// `ZSTD_fillHashTableForCDict` +fn fill_hash_table_for_cdict( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let mls = ms.c_params.min_match; + let hash_table = ms.hash_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { + let curr = prefix.index_of(ip); + write_tagged_index(hash_table, hash_ptr(buf, ip, h_bits, mls)?, curr)?; + if dtlm == DictTableLoadMethod::Full { + // The positions in between only go where there is nothing yet. + for p in 1..FAST_HASH_FILL_STEP { + let hash_and_tag = hash_ptr(buf, ip + p as usize, h_bits, mls)?; + if hash_table.at(hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { + write_tagged_index(hash_table, hash_and_tag, curr + p)?; + } + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillHashTableForCCtx` +fn fill_hash_table_for_cctx( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, +) -> Result<()> { + let h_bits = ms.c_params.hash_log; + let mls = ms.c_params.min_match; + let hash_table = ms.hash_table.as_mut_slice(); + let buf = prefix.buf; + let mut ip = prefix.pos_of(ms.next_to_update); + let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { + return Ok(()); + }; + + while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { + let curr = prefix.index_of(ip); + hash_table.put(hash_ptr(buf, ip, h_bits, mls)?, curr)?; + if dtlm == DictTableLoadMethod::Full { + // The positions in between only go where there is nothing yet. + for p in 1..FAST_HASH_FILL_STEP { + let hash = hash_ptr(buf, ip + p as usize, h_bits, mls)?; + if hash_table.at(hash)? == 0 { + hash_table.put(hash, curr + p)?; + } + } + } + ip += FAST_HASH_FILL_STEP as usize; + } + Ok(()) +} + +/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_hash_table( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + dtlm: DictTableLoadMethod, + tfp: TableFillPurpose, +) -> Result<()> { + match tfp { + TableFillPurpose::ForCDict => fill_hash_table_for_cdict(ms, prefix, end, dtlm), + TableFillPurpose::ForCCtx => fill_hash_table_for_cctx(ms, prefix, end, dtlm), + } +} + +/// `ZSTD_match4Found_branch`, and `ZSTD_match4Found_cmov`, which only differs in how it avoids reading at an index that +/// is not valid. +#[inline(always)] +fn match4_found( + buf: &[u8], + current: usize, + match_pos: usize, + match_idx: u32, + idx_low_limit: u32, +) -> Result { + Ok(match_idx >= idx_low_limit && read_le32(buf, current)? == read_le32(buf, match_pos)?) +} + +/// `ZSTD_compressBlock_fast_noDict_generic` +/// +/// Each turn of the inner loop looks at two positions, `ip0` and `ip1`, for a match, and at `ip2` for a repeat. The +/// hash of a position is computed, and the table is read, one position ahead of the comparison. +fn compress_block_fast_no_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); + + let hlog = ms.c_params.hash_log; + let window_log = ms.c_params.window_log; + let target_length = ms.c_params.target_length; + // The least step is 2, as two positions are looked at per turn. + let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let prefix_start_index = ms.get_lowest_prefix_index(end_index, window_log); + let prefix_start = src.pos_of(prefix_start_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip0 = istart + usize::from(istart == prefix_start); + let [mut rep_offset1, mut rep_offset2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let curr = src.index_of(ip0); + let window_low = ms.get_lowest_prefix_index(curr, window_log); + let max_rep = curr.wrapping_sub(window_low); + if rep_offset2 > max_rep { + offset_saved2 = rep_offset2; + rep_offset2 = 0; + } + if rep_offset1 > max_rep { + offset_saved1 = rep_offset1; + rep_offset1 = 0; + } + } + let hash_table = ms.hash_table.as_mut_slice(); + + 'start: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let mut ip1 = ip0 + 1; + let mut ip2 = ip0 + step; + let mut ip3 = ip2 + 1; + if ip3 >= ilimit { + break 'start; + } + + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let mut match_idx = hash_table.at(hash0)?; + let mut current0; + + // Ends with the start and the length so far of a repeat match, or with `None` for a match at `match_idx`. + let rep_match = loop { + let rval = read_le32(buf, ip2.wrapping_sub(rep_offset1 as usize))?; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if (read_le32(buf, ip2)? == rval) & (rep_offset1 > 0) { + ip0 = ip2; + let match0 = ip0.wrapping_sub(rep_offset1 as usize); + let back = usize::from(buf.at(ip0 - 1)? == buf.at(match0.wrapping_sub(1))?); + ip0 -= back; + // The hash of ip1 is there, and ip1 is before the end of the match. + hash_table.put(hash1, src.index_of(ip1))?; + break Some((match0.wrapping_sub(back), 4 + back)); + } + + if match4_found( + buf, + ip0, + src.pos_of(match_idx), + match_idx, + prefix_start_index, + )? { + hash_table.put(hash1, src.index_of(ip1))?; + break None; + } + + match_idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if match4_found( + buf, + ip0, + src.pos_of(match_idx), + match_idx, + prefix_start_index, + )? { + // ip1 is certain to be before the end of the match only if the step is small. + if step <= 4 { + hash_table.put(hash1, src.index_of(ip1))?; + } + break None; + } + + match_idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + if ip2 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + if ip3 >= ilimit { + break 'start; + } + }; + + let (offcode, match0, mut m_length) = match rep_match { + Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), + None => { + let match0 = src.pos_of(match_idx); + rep_offset2 = rep_offset1; + rep_offset1 = ip0.wrapping_sub(match0) as u32; + let back = count_backwards(buf, ip0, anchor, buf, match0, prefix_start); + ip0 -= back; + (offset_to_offbase(rep_offset1), match0 - back, 4 + back) + } + }; + + m_length += count(buf, ip0 + m_length, buf, match0 + m_length, iend)?; + seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; + ip0 += m_length; + anchor = ip0; + + if ip0 <= ilimit { + hash_table.put( + hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, + current0 + 2, + )?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + if rep_offset2 > 0 { + while ip0 <= ilimit + && read_le32(buf, ip0)? + == read_le32(buf, ip0.wrapping_sub(rep_offset2 as usize))? + { + let r_length = count( + buf, + ip0 + 4, + buf, + (ip0 + 4).wrapping_sub(rep_offset2 as usize), + iend, + )? + 4; + core::mem::swap(&mut rep_offset1, &mut rep_offset2); + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; + ip0 += r_length; + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; + anchor = ip0; + } + } + } + } + + restore_rep(rep, rep_offset1, rep_offset2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast` +pub(crate) fn compress_block_fast( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_no_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_no_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_no_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_no_dict_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_fast_dictMatchState_generic` +fn compress_block_fast_dict_match_state_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; + + let hlog = ms.c_params.hash_log; + let target_length = ms.c_params.target_length; + let step_size = (target_length + u32::from(target_length == 0)) as usize; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let prefix_start_index = ms.window.dict_limit; + let prefix_start = src.pos_of(prefix_start_index); + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + let [mut offset_1, mut offset_2, _] = *rep; + + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_hash_table = dms.ms.hash_table.as_slice(); + let dict_start_index = dms.ms.window.dict_limit; + let dict_start = dms.pos_of(dict_start_index); + let dict_end = dms.end; + // What to subtract from an index below prefix_start_index to get the index in the dictionary. + let dict_index_delta = prefix_start_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); + let dict_and_prefix_length = istart + .wrapping_sub(prefix_start) + .wrapping_add(dict_end) + .wrapping_sub(dict_start) as u32; + let dict_h_bits = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: dms.buf, + pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + let hash_table = ms.hash_table.as_mut_slice(); + + let mut anchor = istart; + let mut ip0 = istart + usize::from(dict_and_prefix_length == 0); + let mut ip1 = istart + step_size; + + 'search: while ip1 <= ilimit { + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + + let dict_hash_and_tag0 = hash_ptr(buf, ip0, dict_h_bits, MLS)?; + let mut dict_match_index_and_tag = + dict_hash_table.at(dict_hash_and_tag0 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + let mut dict_tags_match = + compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag0); + + let mut match_index = hash_table.at(hash0)?; + let mut curr = src.index_of(ip0); + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let m_length = loop { + let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); + let rep_match = locate(rep_index); + let hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let dict_hash_and_tag1 = hash_ptr(buf, ip1, dict_h_bits, MLS)?; + hash_table.put(hash0, curr)?; + + if index_overlap_check(prefix_start_index, rep_index) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip0 + 1)? + { + let m_length = count_2segments( + buf, + ip0 + 1 + 4, + rep_match.buf, + rep_match.pos + 4, + iend, + rep_match.end, + prefix_start, + )? + 4; + ip0 += 1; + seq_store.store_seq(ip0 - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + break m_length; + } + + if dict_tags_match { + let dict_match_index = dict_match_index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + let dict_match = dms.pos_of(dict_match_index); + // A match in the dictionary is only taken if there is no candidate in the prefix. + if dict_match_index > dict_start_index + && read_le32(dms.buf, dict_match)? == read_le32(buf, ip0)? + && match_index <= prefix_start_index + { + let offset = curr + .wrapping_sub(dict_match_index) + .wrapping_sub(dict_index_delta); + let mut m_length = count_2segments( + buf, + ip0 + 4, + dms.buf, + dict_match + 4, + iend, + dict_end, + prefix_start, + )? + 4; + let back = count_backwards(buf, ip0, anchor, dms.buf, dict_match, dict_start); + ip0 -= back; + m_length += back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip0 - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + break m_length; + } + } + + let match_pos = src.pos_of(match_index); + if match4_found(buf, ip0, match_pos, match_index, prefix_start_index)? { + let offset = ip0.wrapping_sub(match_pos) as u32; + let mut m_length = count(buf, ip0 + 4, buf, match_pos + 4, iend)? + 4; + let back = count_backwards(buf, ip0, anchor, buf, match_pos, prefix_start); + ip0 -= back; + m_length += back; + offset_2 = offset_1; + offset_1 = offset; + seq_store.store_seq( + ip0 - anchor, + buf, + anchor, + offset_to_offbase(offset), + m_length, + )?; + break m_length; + } + + dict_match_index_and_tag = + dict_hash_table.at(dict_hash_and_tag1 >> ZSTD_SHORT_CACHE_TAG_BITS)?; + dict_tags_match = + compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag1); + match_index = hash_table.at(hash1)?; + + if ip1 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + ip0 = ip1; + ip1 += step; + if ip1 > ilimit { + break 'search; + } + + curr = src.index_of(ip0); + hash0 = hash1; + }; + + ip0 += m_length; + anchor = ip0; + + if ip0 <= ilimit { + hash_table.put(hash_ptr(buf, src.pos_of(curr + 2), hlog, MLS)?, curr + 2)?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + while ip0 <= ilimit { + let current2 = src.index_of(ip0); + let rep_index2 = current2.wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !(index_overlap_check(prefix_start_index, rep_index2) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip0 + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, current2)?; + ip0 += rep_length2; + anchor = ip0; + } + } + + ip1 = ip0 + step_size; + } + + let [rep0, rep1, _] = rep; + *rep0 = offset_1; + *rep1 = offset_2; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast_dictMatchState` +pub(crate) fn compress_block_fast_dict_match_state( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), + } +} + +/// `ZSTD_compressBlock_fast_extDict_generic` +fn compress_block_fast_ext_dict_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); + + let hlog = ms.c_params.hash_log; + let target_length = ms.c_params.target_length; + let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; + let buf = src.buf; + let istart = src.start; + let iend = src.end; + let end_index = src.index_of(iend); + let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); + let dict_start_index = low_limit; + let dict_start = src.dict_pos_of(dict_start_index); + let dict_limit = ms.window.dict_limit; + let prefix_start_index = dict_limit.max(low_limit); + let prefix_start = src.pos_of(prefix_start_index); + let dict_end = src.dict_pos_of(prefix_start_index); + + // Nothing of the older segment is within reach. + if prefix_start_index == dict_start_index { + return compress_block_fast(ms, seq_store, rep, src); + } + // Nothing is looked for in a block this short. + let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { + return Ok(iend - istart); + }; + + let mut anchor = istart; + let mut ip0 = istart; + let [mut offset_1, mut offset_2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + { + let curr = src.index_of(ip0); + let max_rep = curr.wrapping_sub(dict_start_index); + if offset_2 >= max_rep { + offset_saved2 = offset_2; + offset_2 = 0; + } + if offset_1 >= max_rep { + offset_saved1 = offset_1; + offset_1 = 0; + } + } + + let locate = |index: u32| { + if index < prefix_start_index { + Located { + buf: src.dict_buf, + pos: src.dict_pos_of(index), + start: dict_start, + end: dict_end, + } + } else { + Located { + buf, + pos: src.pos_of(index), + start: prefix_start, + end: iend, + } + } + }; + // Whether there is a match for the position `ip` at `idx`, which is from the table. + let match_found = |ip: usize, idx: u32| -> Result { + let candidate = locate(idx); + Ok(idx >= dict_start_index + && read_le32(candidate.buf, candidate.pos)? == read_le32(buf, ip)?) + }; + let hash_table = ms.hash_table.as_mut_slice(); + + 'start: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + + let mut ip1 = ip0 + 1; + let mut ip2 = ip0 + step; + let mut ip3 = ip2 + 1; + if ip3 >= ilimit { + break 'start; + } + + let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let mut idx = hash_table.at(hash0)?; + let mut current0; + + // Ends with the start and the length so far of a repeat match, or with `None` for a match at `idx`. + let rep_match = loop { + let current2 = src.index_of(ip2); + let rep_index = current2.wrapping_sub(offset_1); + let mut rep_match = locate(rep_index); + // The 4 bytes have to be in one segment, with a byte before them. + let rep_found = ((prefix_start_index.wrapping_sub(rep_index) >= 4) & (offset_1 > 0)) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip2)?; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if rep_found { + ip0 = ip2; + let back = usize::from( + buf.at(ip0 - 1)? == rep_match.buf.at(rep_match.pos.wrapping_sub(1))?, + ); + ip0 -= back; + rep_match.pos = rep_match.pos.wrapping_sub(back); + break Some((rep_match, 4 + back)); + } + + if match_found(ip0, idx)? { + break None; + } + + idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + current0 = src.index_of(ip0); + hash_table.put(hash0, current0)?; + + if match_found(ip0, idx)? { + break None; + } + + idx = hash_table.at(hash1)?; + hash0 = hash1; + hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + if ip2 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + if ip3 >= ilimit { + break 'start; + } + }; + + let (offcode, match0, mut m_length) = match rep_match { + Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), + None => { + let offset = current0.wrapping_sub(idx); + let mut match0 = locate(idx); + offset_2 = offset_1; + offset_1 = offset; + let back = count_backwards(buf, ip0, anchor, match0.buf, match0.pos, match0.start); + ip0 -= back; + match0.pos -= back; + (offset_to_offbase(offset), match0, 4 + back) + } + }; + + m_length += count_2segments( + buf, + ip0 + m_length, + match0.buf, + match0.pos + m_length, + iend, + match0.end, + prefix_start, + )?; + seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; + ip0 += m_length; + anchor = ip0; + + if ip1 < ip0 { + hash_table.put(hash1, src.index_of(ip1))?; + } + + if ip0 <= ilimit { + hash_table.put( + hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, + current0 + 2, + )?; + hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + + while ip0 <= ilimit { + let rep_index2 = src.index_of(ip0).wrapping_sub(offset_2); + let rep_match2 = locate(rep_index2); + if !((index_overlap_check(prefix_start_index, rep_index2) & (offset_2 > 0)) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) + { + break; + } + let rep_length2 = count_2segments( + buf, + ip0 + 4, + rep_match2.buf, + rep_match2.pos + 4, + iend, + rep_match2.end, + prefix_start, + )? + 4; + core::mem::swap(&mut offset_1, &mut offset_2); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; + ip0 += rep_length2; + anchor = ip0; + } + } + } + + restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_fast_extDict` +pub(crate) fn compress_block_fast_ext_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + match ms.c_params.min_match { + 5 => compress_block_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), + 6 => compress_block_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), + 7 => compress_block_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), + _ => compress_block_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), + } +} diff --git a/src/zstd/wip/work/ldm/zstd/compress/fse.rs b/src/zstd/wip/work/ldm/zstd/compress/fse.rs new file mode 100644 index 000000000000..1b8f7cbf3173 --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/compress/fse.rs @@ -0,0 +1,70 @@ +//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. + +use crate::common::{MAX_SEQ, ML_FSE_LOG}; +use crate::util::SliceExt; + +/// `FSE_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum FseRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `FSE_symbolCompressionTransform` +#[derive(Clone, Copy, Default)] +pub(crate) struct FseSymbolTransform { + pub(crate) delta_find_state: i32, + pub(crate) delta_nb_bits: u32, +} + +/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. +#[derive(Clone)] +pub(crate) struct FseCTable { + pub(crate) table_log: u32, + pub(crate) max_symbol_value: u32, + pub(crate) state_table: [u16; 1 << ML_FSE_LOG], + pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], +} + +impl FseCTable { + pub(crate) const fn new() -> Self { + Self { + table_log: 0, + max_symbol_value: 0, + state_table: [0; 1 << ML_FSE_LOG], + symbol_tt: [FseSymbolTransform { + delta_find_state: 0, + delta_nb_bits: 0, + }; MAX_SEQ as usize + 1], + } + } + + /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log + /// if the table has no code for it. + #[inline(always)] + pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 + } + + /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. + /// The table has a code for it. + #[inline(always)] + pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + let min_nb_bits = transform.delta_nb_bits >> 16; + let threshold = (min_nb_bits + 1) << 16; + let table_size = 1u32 << self.table_log; + let delta_from_threshold = + threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); + // A linear interpolation, which is very approximate. + let normalized_delta_from_threshold = + (delta_from_threshold << accuracy_log) >> self.table_log; + let bit_multiplier = 1u32 << accuracy_log; + ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) + } +} diff --git a/src/zstd/wip/work/ldm/zstd/compress/hist.rs b/src/zstd/wip/work/ldm/zstd/compress/hist.rs new file mode 100644 index 000000000000..becf42516867 --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/compress/hist.rs @@ -0,0 +1,33 @@ +//! `hist.c`: how many times each byte value occurs. + +use crate::error::Result; +use crate::util::SliceExt; + +/// `HIST_add`: `count` has room for every value in `src`. +pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { + for &byte in src { + *count.at_mut(usize::from(byte))? += 1; + } + Ok(()) +} + +/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. +/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. +pub(crate) fn count_simple( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + let mut max_symbol = *max_symbol_value as usize; + count.sub_mut(..=max_symbol)?.fill(0); + if src.is_empty() { + *max_symbol_value = 0; + return Ok(0); + } + add(count, src)?; + while count.at(max_symbol)? == 0 { + max_symbol -= 1; + } + *max_symbol_value = max_symbol as u32; + Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) +} diff --git a/src/zstd/wip/work/ldm/zstd/compress/huf.rs b/src/zstd/wip/work/ldm/zstd/compress/huf.rs new file mode 100644 index 000000000000..78373a89ebf0 --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/compress/huf.rs @@ -0,0 +1,38 @@ +//! `huf_compress.c`: Huffman coding of literals. + +use crate::common::HUF_SYMBOLVALUE_MAX; +use crate::util::SliceExt; + +/// `HUF_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum HufRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the +/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in +/// its highest bits. +#[derive(Clone)] +pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); + +impl HufCTable { + pub(crate) const fn new() -> Self { + Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) + } + + /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. + #[inline] + pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { + let [header, elements @ ..] = &self.0; + let max_symbol_value = (header >> 8) as u8; + if symbol_value > u32::from(max_symbol_value) { + return 0; + } + (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 + } +} diff --git a/src/zstd/wip/work/ldm/zstd/compress/internal.rs b/src/zstd/wip/work/ldm/zstd/compress/internal.rs new file mode 100644 index 000000000000..51157ea6c1ce --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/compress/internal.rs @@ -0,0 +1,1212 @@ +//! `zstd_compress_internal.h`: what the parts of the compressor share. + +use super::fse::{FseCTable, FseRepeat}; +use super::huf::{HufCTable, HufRepeat}; +use crate::common::{ + BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, + ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; + +pub(crate) const K_SEARCH_STRENGTH: u32 = 8; +pub(crate) const HASH_READ_SIZE: usize = 8; +/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. +pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; + +pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; +pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; + +pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; +pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; + +/// `ZSTD_strategy` +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub(crate) enum Strategy { + Fast = 1, + DFast = 2, + Greedy = 3, + Lazy = 4, + Lazy2 = 5, + BtLazy2 = 6, + BtOpt = 7, + BtUltra = 8, + BtUltra2 = 9, +} + +impl Strategy { + pub(crate) const fn from_int(value: i32) -> Option { + Some(match value { + 1 => Self::Fast, + 2 => Self::DFast, + 3 => Self::Greedy, + 4 => Self::Lazy, + 5 => Self::Lazy2, + 6 => Self::BtLazy2, + 7 => Self::BtOpt, + 8 => Self::BtUltra, + 9 => Self::BtUltra2, + _ => return None, + }) + } +} + +/// `ZSTD_ParamSwitch_e` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum ParamSwitch { + Auto = 0, + Enable = 1, + Disable = 2, +} + +/// `ZSTD_compressionParameters` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct CompressionParameters { + pub(crate) window_log: u32, + pub(crate) chain_log: u32, + pub(crate) hash_log: u32, + pub(crate) search_log: u32, + pub(crate) min_match: u32, + pub(crate) target_length: u32, + /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. + pub(crate) strategy: Option, +} + +impl CompressionParameters { + /// The strategy of parameters that are ready to be applied. + #[inline(always)] + pub(crate) fn strat(&self) -> Strategy { + self.strategy.unwrap_or(Strategy::Fast) + } +} + +/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum DictMode { + NoDict, + ExtDict, + DictMatchState, +} + +/// `ZSTD_dictTableLoadMethod_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DictTableLoadMethod { + Fast, + Full, +} + +/// `ZSTD_tableFillPurpose_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TableFillPurpose { + ForCCtx, + ForCDict, +} + +// ── entropy tables ──────────────────────────────────────────────────────── + +/// `ZSTD_hufCTables_t` +#[derive(Clone)] +pub(crate) struct HufCTables { + pub(crate) c_table: HufCTable, + pub(crate) repeat_mode: HufRepeat, +} + +/// `ZSTD_fseCTables_t` +#[derive(Clone)] +pub(crate) struct FseCTables { + pub(crate) offcode_c_table: FseCTable, + pub(crate) matchlength_c_table: FseCTable, + pub(crate) litlength_c_table: FseCTable, + pub(crate) offcode_repeat_mode: FseRepeat, + pub(crate) matchlength_repeat_mode: FseRepeat, + pub(crate) litlength_repeat_mode: FseRepeat, +} + +/// `ZSTD_entropyCTables_t` +#[derive(Clone)] +pub(crate) struct EntropyCTables { + pub(crate) huf: HufCTables, + pub(crate) fse: FseCTables, +} + +/// `ZSTD_compressedBlockState_t` +#[derive(Clone)] +pub(crate) struct CompressedBlockState { + pub(crate) entropy: EntropyCTables, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +impl CompressedBlockState { + /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. + pub(crate) fn new() -> Self { + Self { + entropy: EntropyCTables { + huf: HufCTables { + c_table: HufCTable::new(), + repeat_mode: HufRepeat::None, + }, + fse: FseCTables { + offcode_c_table: FseCTable::new(), + matchlength_c_table: FseCTable::new(), + litlength_c_table: FseCTable::new(), + offcode_repeat_mode: FseRepeat::None, + matchlength_repeat_mode: FseRepeat::None, + litlength_repeat_mode: FseRepeat::None, + }, + }, + rep: REP_START_VALUE, + } + } + + /// `ZSTD_reset_compressedBlockState` + pub(crate) fn reset(&mut self) { + self.rep = REP_START_VALUE; + self.entropy.huf.repeat_mode = HufRepeat::None; + self.entropy.fse.offcode_repeat_mode = FseRepeat::None; + self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; + self.entropy.fse.litlength_repeat_mode = FseRepeat::None; + } +} + +/// `ZSTD_hufCTablesMetadata_t` +pub(crate) struct HufCTablesMetadata { + pub(crate) h_type: SymbolEncodingType, + pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], + pub(crate) huf_des_size: usize, +} + +/// `ZSTD_fseCTablesMetadata_t` +pub(crate) struct FseCTablesMetadata { + pub(crate) ll_type: SymbolEncodingType, + pub(crate) of_type: SymbolEncodingType, + pub(crate) ml_type: SymbolEncodingType, + pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], + pub(crate) fse_tables_size: usize, + /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. + pub(crate) last_count_size: usize, +} + +/// `ZSTD_entropyCTablesMetadata_t` +pub(crate) struct EntropyCTablesMetadata { + pub(crate) huf_metadata: HufCTablesMetadata, + pub(crate) fse_metadata: FseCTablesMetadata, +} + +impl EntropyCTablesMetadata { + pub(crate) const fn new() -> Self { + Self { + huf_metadata: HufCTablesMetadata { + h_type: SymbolEncodingType::Basic, + huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], + huf_des_size: 0, + }, + fse_metadata: FseCTablesMetadata { + ll_type: SymbolEncodingType::Basic, + of_type: SymbolEncodingType::Basic, + ml_type: SymbolEncodingType::Basic, + fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], + fse_tables_size: 0, + last_count_size: 0, + }, + } + } +} + +// ── sequences ───────────────────────────────────────────────────────────── + +/// `SeqDef` +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct SeqDef { + /// A repeat code (1 to 3), or an offset plus 3. + pub(crate) off_base: u32, + pub(crate) lit_length: u16, + /// The match length minus [`MINMATCH`]. + pub(crate) ml_base: u16, +} + +/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum LongLengthType { + None, + LiteralLength, + MatchLength, +} + +/// `SeqStore_t`: the sequences and the literals of a block. +pub(crate) struct SeqStore { + /// Never grows beyond the capacity it is given by [`Self::reserve`]. + pub(crate) sequences: Vec, + /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. + pub(crate) lit_buffer: Vec, + /// How much of `lit_buffer` is literals. + pub(crate) lit_len: usize, + pub(crate) ll_code: Vec, + pub(crate) ml_code: Vec, + pub(crate) of_code: Vec, + pub(crate) max_nb_seq: usize, + pub(crate) max_nb_lit: usize, + pub(crate) long_length_type: LongLengthType, + /// Which sequence has the long length. + pub(crate) long_length_pos: u32, +} + +/// `ZSTD_SequenceLength` +#[derive(Clone, Copy)] +pub(crate) struct SequenceLength { + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +impl SeqStore { + pub(crate) const fn new() -> Self { + Self { + sequences: Vec::new(), + lit_buffer: Vec::new(), + lit_len: 0, + ll_code: Vec::new(), + ml_code: Vec::new(), + of_code: Vec::new(), + max_nb_seq: 0, + max_nb_lit: 0, + long_length_type: LongLengthType::None, + long_length_pos: 0, + } + } + + /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. + pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { + fn grow(vec: &mut Vec, len: usize) -> Result<()> { + vec.clear(); + vec.try_reserve_exact(len) + .map_err(|_| ErrorCode::MemoryAllocation)?; + vec.resize(len, T::default()); + Ok(()) + } + grow(&mut self.sequences, max_nb_seq)?; + grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; + grow(&mut self.ll_code, max_nb_seq)?; + grow(&mut self.ml_code, max_nb_seq)?; + grow(&mut self.of_code, max_nb_seq)?; + self.max_nb_seq = max_nb_seq; + self.max_nb_lit = max_nb_lit; + self.reset(); + Ok(()) + } + + /// `ZSTD_resetSeqStore` + #[inline] + pub(crate) fn reset(&mut self) { + self.lit_len = 0; + self.sequences.clear(); + self.long_length_type = LongLengthType::None; + } + + #[inline] + pub(crate) fn literals(&self) -> &[u8] { + self.lit_buffer.get(..self.lit_len).unwrap_or_default() + } + + /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. + #[inline(always)] + pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { + debug_assert!(self.sequences.len() < self.max_nb_seq); + if lit_length > 0xFFFF { + self.long_length_type = LongLengthType::LiteralLength; + self.long_length_pos = self.sequences.len() as u32; + } + let ml_base = match_length.wrapping_sub(MINMATCH as usize); + if ml_base > 0xFFFF { + self.long_length_type = LongLengthType::MatchLength; + self.long_length_pos = self.sequences.len() as u32; + } + self.sequences.push(SeqDef { + off_base, + lit_length: lit_length as u16, + ml_base: ml_base as u16, + }); + } + + /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. + #[inline(always)] + pub(crate) fn store_seq( + &mut self, + lit_length: usize, + buf: &[u8], + literals: usize, + off_base: u32, + match_length: usize, + ) -> Result<()> { + match ( + buf.array_at::<16>(literals), + self.lit_buffer.array_at_mut::<16>(self.lit_len), + ) { + (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, + _ => self + .lit_buffer + .write_at(self.lit_len, buf.span(literals, lit_length)?)?, + } + self.lit_len += lit_length; + self.store_seq_only(lit_length, off_base, match_length); + Ok(()) + } + + /// Appends the literals that follow the last sequence of a block. + #[inline] + pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { + self.lit_buffer.write_at(self.lit_len, literals)?; + self.lit_len += literals.len(); + Ok(()) + } +} + +pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; +pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; +pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; + +/// `OFFSET_TO_OFFBASE`: `offset` is not 0. +#[inline(always)] +pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { + offset.wrapping_add(ZSTD_REP_NUM as u32) +} + +/// `OFFBASE_IS_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { + off_base > ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_IS_REPCODE` +#[inline(always)] +pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { + 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_TO_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { + off_base.wrapping_sub(ZSTD_REP_NUM as u32) +} + +/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. +#[inline(always)] +pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { + let [rep0, rep1, rep2] = *rep; + if offbase_is_offset(off_base) { + *rep = [offbase_to_offset(off_base), rep0, rep1]; + } else { + match off_base - 1 + u32::from(ll0) { + 0 => {} + 1 => *rep = [rep1, rep0, rep2], + 2 => *rep = [rep2, rep0, rep1], + _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], + } + } +} + +/// `ZSTD_newRep` +#[inline(always)] +pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { + let mut new_reps = *rep; + update_rep(&mut new_reps, off_base, ll0); + new_reps +} + +/// `ZSTD_LLcode` +#[inline(always)] +pub(crate) fn ll_code(lit_length: u32) -> u32 { + #[rustfmt::skip] + const LL_CODE: [u8; 64] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + ]; + const LL_DELTA_CODE: u32 = 19; + match LL_CODE.get(lit_length as usize) { + Some(&code) => u32::from(code), + None => highbit32(lit_length) + LL_DELTA_CODE, + } +} + +/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. +#[inline(always)] +pub(crate) fn ml_code(ml_base: u32) -> u32 { + #[rustfmt::skip] + const ML_CODE: [u8; 128] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ]; + const ML_DELTA_CODE: u32 = 36; + match ML_CODE.get(ml_base as usize) { + Some(&code) => u32::from(code), + None => highbit32(ml_base) + ML_DELTA_CODE, + } +} + +/// `ZSTD_noCompressBlock` +pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { + let c_block_header24 = + u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); + if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header24)?; + dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; + Ok(ZSTD_BLOCKHEADERSIZE + src.len()) +} + +/// `ZSTD_rleCompressBlock` +pub(crate) fn rle_compress_block( + dst: &mut [u8], + byte: u8, + src_size: usize, + last_block: bool, +) -> Result { + let c_block_header = + u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); + if dst.len() < 4 { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header)?; + dst.put(3, byte)?; + Ok(4) +} + +/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. +#[inline] +pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { + let minlog = if strat >= Strategy::BtUltra { + strat as u32 - 1 + } else { + 6 + }; + (src_size >> minlog) + 2 +} + +// ── matching ────────────────────────────────────────────────────────────── + +/// How many bytes `a` and `b` have in common at their start. +#[inline(always)] +pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { + let (a_words, _) = a.as_chunks::<8>(); + let (b_words, _) = b.as_chunks::<8>(); + let mut len = 0usize; + for (a_word, b_word) in a_words.iter().zip(b_words) { + let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); + if diff != 0 { + return len + (diff.trailing_zeros() >> 3) as usize; + } + len += 8; + } + let a_tail = a.get(len..).unwrap_or_default(); + let b_tail = b.get(len..).unwrap_or_default(); + len + a_tail + .iter() + .zip(b_tail) + .take_while(|(a_byte, b_byte)| a_byte == b_byte) + .count() +} + +/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. +#[inline(always)] +pub(crate) fn count( + buf: &[u8], + p_in: usize, + match_buf: &[u8], + p_match: usize, + p_in_limit: usize, +) -> Result { + Ok(common_prefix( + buf.sub(p_in..p_in_limit)?, + match_buf.sub(p_match..)?, + )) +} + +/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at +/// `i_start` in `buf`. +#[inline(always)] +pub(crate) fn count_2segments( + buf: &[u8], + ip: usize, + match_buf: &[u8], + match_pos: usize, + i_end: usize, + m_end: usize, + i_start: usize, +) -> Result { + let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); + if match_pos + match_length != m_end { + return Ok(match_length); + } + Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) +} + +const PRIME_3_BYTES: u32 = 506_832_829; +const PRIME_4_BYTES: u32 = 2_654_435_761; +const PRIME_5_BYTES: u64 = 889_523_592_379; +const PRIME_6_BYTES: u64 = 227_718_039_650_203; +const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; +pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; + +/// `ZSTD_hash3` +#[inline(always)] +const fn hash3(u: u32, h: u32, s: u32) -> u32 { + (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash4(u: u32, h: u32, s: u32) -> u32 { + ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash5(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash6(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash7(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +/// `ZSTD_hash8` +#[inline(always)] +pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { + (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} + +/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. +#[inline(always)] +pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { + Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) +} + +/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, +/// else 8. +#[inline(always)] +pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { + hash_ptr_salted(buf, pos, h_bits, mls, 0) +} + +/// `ZSTD_hashPtrSalted` +#[inline(always)] +pub(crate) fn hash_ptr_salted( + buf: &[u8], + pos: usize, + h_bits: u32, + mls: u32, + hash_salt: u64, +) -> Result { + Ok(match mls { + 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), + 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), + 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), + 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), + _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, + }) +} + +// ── the window ──────────────────────────────────────────────────────────── + +/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are +/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the +/// others, and an address is turned into a position in the buffer only to read. +pub(crate) type Addr = usize; + +/// The buffers that a window can have a segment in. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum BufId { + /// Where an empty window points. + None = 0, + /// The context's own buffer of input. + InBuff = 1, + /// What the caller gave to the call in progress. + User = 2, + /// The content of the dictionary, or the prefix. + Dict = 3, +} + +impl BufId { + /// The address of the first byte of the buffer. + #[inline(always)] + pub(crate) const fn start(self) -> Addr { + (self as usize + 1) << 44 + } +} + +/// The buffers themselves, for as long as a call lasts. +#[derive(Clone, Copy)] +pub(crate) struct Buffers<'a> { + pub(crate) in_buff: &'a [u8], + pub(crate) user: &'a [u8], + pub(crate) dict: &'a [u8], +} + +impl<'a> Buffers<'a> { + #[inline] + pub(crate) fn get(&self, id: BufId) -> &'a [u8] { + match id { + BufId::None => &[], + BufId::InBuff => self.in_buff, + BufId::User => self.user, + BufId::Dict => self.dict, + } + } +} + +/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. +pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; + +/// The index past which indices are brought down. +pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; +/// The most input that can be indexed at once. +pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; + +/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the +/// index is less than `dict_limit`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Window { + /// Where input has to continue to be contiguous. + pub(crate) next_src: Addr, + pub(crate) base: Addr, + pub(crate) dict_base: Addr, + /// The buffer of the segment that starts at `dict_limit`. + pub(crate) base_buf: BufId, + /// The buffer of the segment from `low_limit` to `dict_limit`. + pub(crate) dict_buf: BufId, + pub(crate) dict_limit: u32, + /// Below this, there is nothing valid. + pub(crate) low_limit: u32, + pub(crate) nb_overflow_corrections: u32, +} + +impl Window { + /// `ZSTD_window_init` + pub(crate) const fn new() -> Self { + let base = BufId::None.start(); + Self { + next_src: base + ZSTD_WINDOW_START_INDEX as usize, + base, + dict_base: base, + base_buf: BufId::None, + dict_buf: BufId::None, + dict_limit: ZSTD_WINDOW_START_INDEX, + low_limit: ZSTD_WINDOW_START_INDEX, + nb_overflow_corrections: 0, + } + } + + /// The index of `addr`, which is in the current segment or at its end. + #[inline(always)] + pub(crate) const fn index_of(&self, addr: Addr) -> u32 { + addr.wrapping_sub(self.base) as u32 + } + + /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. + pub(crate) fn clear(&mut self) { + let end = self.index_of(self.next_src); + self.low_limit = end; + self.dict_limit = end; + } + + /// `ZSTD_window_isEmpty` + pub(crate) fn is_empty(&self) -> bool { + self.dict_limit == ZSTD_WINDOW_START_INDEX + && self.low_limit == ZSTD_WINDOW_START_INDEX + && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize + } + + /// `ZSTD_window_hasExtDict` + #[inline] + pub(crate) fn has_ext_dict(&self) -> bool { + self.low_limit < self.dict_limit + } + + /// `ZSTD_window_needOverflowCorrection` + #[inline] + pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { + self.index_of(src_end) > ZSTD_CURRENT_MAX + } + + /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns + /// what to subtract from every index that is stored. + pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { + let cycle_size = 1u32 << cycle_log; + let cycle_mask = cycle_size - 1; + let curr = self.index_of(src); + let current_cycle = curr & cycle_mask; + // Indices must not fall below the start index. + let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { + cycle_size.max(ZSTD_WINDOW_START_INDEX) + } else { + 0 + }; + let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); + let correction = curr.wrapping_sub(new_current); + + self.base = self.base.wrapping_add(correction as usize); + self.dict_base = self.dict_base.wrapping_add(correction as usize); + self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.low_limit - correction + }; + self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.dict_limit - correction + }; + self.nb_overflow_corrections += 1; + correction + } + + /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the + /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. + pub(crate) fn enforce_max_dist( + &mut self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { + let new_low_limit = block_end_idx - max_dist; + if self.low_limit < new_low_limit { + self.low_limit = new_low_limit; + } + if self.dict_limit < self.low_limit { + self.dict_limit = self.low_limit; + } + return true; + } + false + } + + /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not + /// right before the current segment, and is to be forgotten. + pub(crate) fn dict_is_invalid( + &self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit + } + + /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not + /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. + pub(crate) fn update( + &mut self, + src: Addr, + src_buf: BufId, + src_size: usize, + force_non_contiguous: bool, + ) -> bool { + let mut contiguous = true; + if src_size == 0 { + return contiguous; + } + if src != self.next_src || force_non_contiguous { + let distance_from_base = self.next_src.wrapping_sub(self.base); + self.low_limit = self.dict_limit; + self.dict_limit = distance_from_base as u32; + self.dict_base = self.base; + self.dict_buf = self.base_buf; + self.base = src.wrapping_sub(distance_from_base); + self.base_buf = src_buf; + if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { + // Too small to be of use. + self.low_limit = self.dict_limit; + } + contiguous = false; + } + self.next_src = src + src_size; + // Where the input overlaps the older segment, it has overwritten it. + if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) + & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) + { + let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); + self.low_limit = if high_input_idx > self.dict_limit as usize { + self.dict_limit + } else { + high_input_idx as u32 + }; + } + contiguous + } +} + +/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes +/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. +#[inline(always)] +pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { + prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 +} + +pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; +pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; + +/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. +#[inline(always)] +pub(crate) fn write_tagged_index( + hash_table: &mut [u32], + hash_and_tag: usize, + index: u32, +) -> Result<()> { + let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; + hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) +} + +/// `ZSTD_comparePackedTags` +#[inline(always)] +pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { + (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) + == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) +} + +// ── the state of the match finders ──────────────────────────────────────── + +/// `ZSTD_match_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Match { + /// As [`SeqDef::off_base`]. + pub(crate) off: u32, + pub(crate) len: u32, +} + +/// `rawSeq`: a sequence found by long distance matching. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct RawSeq { + pub(crate) offset: u32, + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +/// `RawSeqStore_t` +#[derive(Clone, Default)] +pub(crate) struct RawSeqStore { + /// `capacity` of them, of which `size` are in use. + pub(crate) seq: Vec, + /// The sequence that is being read. + pub(crate) pos: usize, + /// How many of its bytes, literals first, are read. + pub(crate) pos_in_sequence: usize, + pub(crate) size: usize, + pub(crate) capacity: usize, +} + +/// `ZSTD_optimal_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Optimal { + /// The cost of getting to this position. + pub(crate) price: i32, + pub(crate) off: u32, + pub(crate) mlen: u32, + pub(crate) litlen: u32, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +/// `ZSTD_OptPrice_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum OptPrice { + Dynamic, + Predef, +} + +/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. +pub(crate) struct OptState { + pub(crate) lit_freq: Vec, + pub(crate) lit_length_freq: Vec, + pub(crate) match_length_freq: Vec, + pub(crate) off_code_freq: Vec, + pub(crate) match_table: Vec, + pub(crate) price_table: Vec, + pub(crate) lit_sum: u32, + pub(crate) lit_length_sum: u32, + pub(crate) match_length_sum: u32, + pub(crate) off_code_sum: u32, + pub(crate) lit_sum_base_price: u32, + pub(crate) lit_length_sum_base_price: u32, + pub(crate) match_length_sum_base_price: u32, + pub(crate) off_code_sum_base_price: u32, + pub(crate) price_type: OptPrice, + pub(crate) literal_compression_mode: ParamSwitch, +} + +impl OptState { + pub(crate) const fn new() -> Self { + Self { + lit_freq: Vec::new(), + lit_length_freq: Vec::new(), + match_length_freq: Vec::new(), + off_code_freq: Vec::new(), + match_table: Vec::new(), + price_table: Vec::new(), + lit_sum: 0, + lit_length_sum: 0, + match_length_sum: 0, + off_code_sum: 0, + lit_sum_base_price: 0, + lit_length_sum_base_price: 0, + match_length_sum_base_price: 0, + off_code_sum_base_price: 0, + price_type: OptPrice::Dynamic, + literal_compression_mode: ParamSwitch::Auto, + } + } +} + +pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; + +/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures +/// own, are in [`BlockSrc`]. +pub(crate) struct MatchState { + pub(crate) window: Window, + /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. + pub(crate) loaded_dict_end: u32, + /// The index from which the tables are still to be filled. + pub(crate) next_to_update: u32, + /// 0 if there is no table of 3-byte hashes. + pub(crate) hash_log3: u32, + pub(crate) row_hash_log: u32, + /// For the row-based match finder: a byte of hash for each entry of `hash_table`. + pub(crate) tag_table: Vec, + pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], + /// Makes what a previous frame left in `tag_table` meaningless. + pub(crate) hash_salt: u64, + pub(crate) hash_salt_entropy: u32, + pub(crate) hash_table: Vec, + pub(crate) hash_table3: Vec, + pub(crate) chain_table: Vec, + pub(crate) force_non_contiguous: bool, + pub(crate) opt: OptState, + /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. + pub(crate) has_dict_match_state: bool, + pub(crate) c_params: CompressionParameters, + pub(crate) lazy_skipping: bool, +} + +impl MatchState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + loaded_dict_end: 0, + next_to_update: 0, + hash_log3: 0, + row_hash_log: 0, + tag_table: Vec::new(), + hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + hash_salt: 0, + hash_salt_entropy: 0, + hash_table: Vec::new(), + hash_table3: Vec::new(), + chain_table: Vec::new(), + force_non_contiguous: false, + opt: OptState::new(), + has_dict_match_state: false, + c_params: CompressionParameters { + window_log: 0, + chain_log: 0, + hash_log: 0, + search_log: 0, + min_match: 0, + target_length: 0, + strategy: None, + }, + lazy_skipping: false, + } + } + + /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. + pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { + if self.window.has_ext_dict() { + return DictMode::ExtDict; + } + if dms.is_some() { + DictMode::DictMatchState + } else { + DictMode::NoDict + } + } + + /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. + #[inline(always)] + pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.low_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + // A dictionary is valid as a whole until it is invalidated as a whole. + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } + + /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. + #[inline(always)] + pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.dict_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } +} + +/// The match state of a dictionary, and where its indices lead. +#[derive(Clone, Copy)] +pub(crate) struct DictMatchState<'a> { + pub(crate) ms: &'a MatchState, + /// The buffer its window is in, which is one segment. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// `window.nextSrc`, as a position in `buf`. + pub(crate) end: usize, +} + +/// The current segment of a window, for the functions that index a dictionary. +#[derive(Clone, Copy)] +pub(crate) struct Prefix<'a> { + /// The buffer that the segment is in. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, +} + +impl Prefix<'_> { + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. +#[derive(Clone, Copy)] +pub(crate) struct BlockSrc<'a> { + /// The buffer that the current segment of the window is in, and so the block. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// The buffer that the older segment of the window is in. + pub(crate) dict_buf: &'a [u8], + /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. + pub(crate) dict_base: usize, + /// Where the block starts in `buf`. + pub(crate) start: usize, + /// Where it ends. + pub(crate) end: usize, + pub(crate) dms: Option>, + /// `opt.symbolCosts`: the tables of the previous block. + pub(crate) symbol_costs: &'a EntropyCTables, + /// `ldmSeqStore`: matches that the optimal parser is to consider. + pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, +} + +impl<'a> BlockSrc<'a> { + #[inline(always)] + pub(crate) const fn prefix(&self) -> Prefix<'a> { + Prefix { + buf: self.buf, + base: self.base, + } + } + + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`, which is in the current segment. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } + + /// The position in `dict_buf` of `index`, which is in the older segment. + #[inline(always)] + pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { + self.dict_base.wrapping_add(index as usize) + } +} + +impl DictMatchState<'_> { + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +// ── long distance matching ──────────────────────────────────────────────── + +/// `ldmEntry_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct LdmEntry { + pub(crate) offset: u32, + pub(crate) checksum: u32, +} + +/// `ldmParams_t` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct LdmParams { + pub(crate) enable_ldm: ParamSwitch, + pub(crate) hash_log: u32, + pub(crate) bucket_size_log: u32, + pub(crate) min_match_length: u32, + pub(crate) hash_rate_log: u32, + pub(crate) window_log: u32, +} + +/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. +pub(crate) struct LdmState { + pub(crate) window: Window, + pub(crate) hash_table: Vec, + pub(crate) loaded_dict_end: u32, + /// The next entry to replace in each bucket. + pub(crate) bucket_offsets: Vec, +} + +impl LdmState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + hash_table: Vec::new(), + loaded_dict_end: 0, + bucket_offsets: Vec::new(), + } + } +} diff --git a/src/zstd/wip/work/ldm/zstd/compress/lazy.rs b/src/zstd/wip/work/ldm/zstd/compress/lazy.rs new file mode 100644 index 000000000000..94dbe308ce79 --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/compress/lazy.rs @@ -0,0 +1,1353 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use super::internal::{ + BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, + SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, + hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, + offset_to_offbase, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +const K_LAZY_SKIPPING_STEP: usize = 8; + +// `ZSTD_dictMode_e`, as an argument of a const generic. +const NO_DICT: u8 = DictMode::NoDict as u8; +const EXT_DICT: u8 = DictMode::ExtDict as u8; +const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; + +/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. +#[inline(always)] +const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { + dms.end.wrapping_sub(dms.base) as u32 +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. +#[inline(always)] +fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { + let hash_log = ms.c_params.hash_log; + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let bt = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); + *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. +fn insert_dubt1( + ms: &mut MatchState, + src: &BlockSrc<'_>, + curr: u32, + input_end: usize, + mut nb_compares: u32, + bt_low: u32, +) -> Result<()> { + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let (ip_buf, ip, iend) = if curr >= dict_limit { + (src.buf, src.pos_of(curr), input_end) + } else { + (src.dict_buf, src.dict_pos_of(curr), dict_end) + }; + let window_valid = ms.window.low_limit; + let max_distance = 1u32 << ms.c_params.window_log; + let window_low = if curr.wrapping_sub(window_valid) > max_distance { + curr - max_distance + } else { + window_valid + }; + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + // The next sorted candidate. The other slot has the previous unsorted one. + let mut match_index = bt.at(smaller_ptr)?; + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let in_prefix = + DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; + // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = if in_prefix { + (src.buf, src.pos_of(match_index)) + } else { + (src.dict_buf, src.dict_pos_of(match_index)) + }; + + if in_prefix || curr < dict_limit { + // Both are in the same segment. + match_length += count( + ip_buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + ip_buf, + ip + match_length, + match_buf, + match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if match_index as usize + match_length >= dict_limit as usize { + (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); + } + } + + if ip + match_length == iend { + // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0) +} + +/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. +fn dubt_find_better_dict_match( + ms: &MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mut best_length: usize, + mut nb_compares: u32, + mls: u32, +) -> Result { + let dms = src.dms.ok_or(BUG)?; + let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(h)?; + + let prefix_start = src.pos_of(ms.window.dict_limit); + let curr = src.index_of(ip); + let dict_high_limit = dms_end_index(&dms); + let dict_low_limit = dms.ms.window.low_limit; + let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); + + let dict_bt = dms.ms.chain_table.as_slice(); + let bt_log = dms.ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { + dict_low_limit + } else { + dict_high_limit - bt_mask + }; + + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + + while nb_compares != 0 && dict_match_index > dict_low_limit { + let next_ptr = 2 * (dict_match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + let match_index = dict_match_index.wrapping_add(dict_index_delta); + match_length += count_2segments( + src.buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + iend, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(offset_ptr.wrapping_add(1)) as i32 + { + best_length = match_length; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // No way to know which is smaller. + break; + } + } + + if dict_match_index <= bt_low { + // Beyond the tree. + break; + } + let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { + let match_pos = src.pos_of(match_index); + src.buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < src.buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dict_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dict_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + Ok(best_length) +} + +/// `ZSTD_DUBT_findBestMatch` +#[inline(always)] +fn dubt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + off_base_ptr: &mut u32, + mls: u32, +) -> Result { + let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; + let mut match_index = ms.hash_table.at(h)?; + + let curr = src.index_of(ip); + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + + let bt_log = ms.c_params.chain_log - 1; + let bt_mask = (1u32 << bt_log) - 1; + let bt_low = curr.saturating_sub(bt_mask); + let unsort_limit = bt_low.max(window_low); + let mut nb_compares = 1u32 << ms.c_params.search_log; + let mut nb_candidates = nb_compares; + let mut previous_candidate = 0u32; + + // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. + let bt = ms.chain_table.as_mut_slice(); + let mut next_candidate = 2 * (match_index & bt_mask) as usize; + while match_index > unsort_limit + && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK + && nb_candidates > 1 + { + bt.put(next_candidate + 1, previous_candidate)?; + previous_candidate = match_index; + match_index = bt.at(next_candidate)?; + next_candidate = 2 * (match_index & bt_mask) as usize; + nb_candidates -= 1; + } + + // If the last candidate is still unsorted, cut the list there. + if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { + *bt.array_at_mut(next_candidate)? = [0, 0]; + } + + // Sort the candidates, the oldest first. + match_index = previous_candidate; + while match_index != 0 { + let next_candidate_idx = ms + .chain_table + .at(2 * (match_index & bt_mask) as usize + 1)?; + insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; + match_index = next_candidate_idx; + nb_candidates += 1; + } + + // Find the longest match. + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt = ms.chain_table.as_mut_slice(); + let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut larger_ptr = smaller_ptr + 1; + let mut match_end_idx = curr + 8 + 1; + let mut best_length = 0usize; + + match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); + + while nb_compares != 0 && match_index > window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. + let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); + + if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + src.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + let dict_match_pos = src.dict_pos_of(match_index); + match_length += count_2segments( + src.buf, + ip + match_length, + src.dict_buf, + dict_match_pos + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + (match_buf, match_pos) = (src.dict_buf, dict_match_pos); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index + match_length as u32; + } + if 4 * (match_length - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(*off_base_ptr) as i32 + { + best_length = match_length; + *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if ip + match_length == iend { + // Equal: no way to know which is smaller. + if DICT_MODE == DICT_MATCH_STATE { + // Nothing in the dictionary can be longer. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { + bt.put(smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. This slot keeps its value: only the other one is cleared. + smaller_ptr = larger_ptr; + break; + } + smaller_ptr = next_ptr + 1; + match_index = bt.at(next_ptr + 1)?; + } else { + bt.put(larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = smaller_ptr; + break; + } + larger_ptr = next_ptr; + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + bt.put(smaller_ptr, 0)?; + bt.put(larger_ptr, 0)?; + + if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { + best_length = dubt_find_better_dict_match( + ms, + src, + ip, + iend, + off_base_ptr, + best_length, + nb_compares, + mls, + )?; + } + + // Skip repetitive patterns. + ms.next_to_update = match_end_idx - 8; + Ok(best_length) +} + +/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. +/// `off_base_ptr` is written to if there is one. +#[inline(never)] +fn bt_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + off_base_ptr: &mut u32, +) -> Result { + if ip < src.pos_of(ms.next_to_update) { + // In an area that is skipped. + return Ok(0); + } + update_dubt(ms, src.prefix(), ip, MLS)?; + dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) +} + +// ── hash chains ─────────────────────────────────────────────────────────── + +/// `ZSTD_insertAndFindFirstIndex_internal` +#[inline(always)] +fn insert_and_find_first_index_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + lazy_skipping: bool, +) -> Result { + let hash_log = ms.c_params.hash_log; + let chain_mask = (1u32 << ms.c_params.chain_log) - 1; + let target = prefix.index_of(ip); + let hash_table = ms.hash_table.as_mut_slice(); + let chain_table = ms.chain_table.as_mut_slice(); + + for idx in ms.next_to_update..target { + let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; + let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); + chain_table.put((idx & chain_mask) as usize, next_in_chain)?; + // While skipping, not every position is inserted. + if lazy_skipping { + break; + } + } + + ms.next_to_update = target; + hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) +} + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) +} + +/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn hc_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let chain_size = 1u32 << ms.c_params.chain_log; + let chain_mask = chain_size - 1; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let min_chain = curr.saturating_sub(chain_size); + let mut nb_attempts = 1u32 << ms.c_params.search_log; + let mut ml = 4 - 1; + + let mut match_index = + insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; + let chain_table = ms.chain_table.as_slice(); + + while match_index >= low_limit && nb_attempts > 0 { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + + if match_index <= min_chain { + break; + } + match_index = chain_table.at((match_index & chain_mask) as usize)?; + nb_attempts -= 1; + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_chain_table = dms.ms.chain_table.as_slice(); + let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; + let dms_chain_mask = dms_chain_size - 1; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_size = dms_end_index(&dms); + let dms_index_delta = dict_limit.wrapping_sub(dms_size); + let dms_min_chain = dms_size.saturating_sub(dms_chain_size); + + let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; + match_index = dms.ms.hash_table.at(h)?; + + while match_index >= dms_lowest_index && nb_attempts > 0 { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + + if match_index <= dms_min_chain { + break; + } + match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; + nb_attempts -= 1; + } + } + + Ok(ml) +} + +// ── row-based match finder ──────────────────────────────────────────────── +// +// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits +// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag +// table is the "head": which entry was written last. + +pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; +const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; +const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; + +/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. +#[inline(always)] +fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { + let mut next = u32::from(*head).wrapping_sub(1) & row_mask; + if next == 0 { + next = row_mask; + } + *head = next as u8; + next +} + +/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. +#[inline(always)] +fn row_fill_hash_cache( + ms: &mut MatchState, + prefix: Prefix<'_>, + mls: u32, + idx: u32, + i_limit: usize, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); + let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; + + for idx in idx..lim { + let hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + ms.hash_salt, + )? as u32; + let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; + ms.hash_cache.put(cached, hash)?; + } + Ok(()) +} + +/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes +/// further takes its place. +#[inline(always)] +fn row_next_cached_hash( + cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], + prefix: Prefix<'_>, + idx: u32, + hash_log: u32, + mls: u32, + hash_salt: u64, +) -> Result { + let new_hash = hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32; + let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; + Ok(core::mem::replace(cached, new_hash)) +} + +/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. +#[inline(always)] +fn row_update_internal_impl( + ms: &mut MatchState, + prefix: Prefix<'_>, + update_start_idx: u32, + update_end_idx: u32, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + let hash_log = ms.row_hash_log; + let hash_salt = ms.hash_salt; + let hash_table = ms.hash_table.as_mut_slice(); + let tag_table = ms.tag_table.as_mut_slice(); + + for idx in update_start_idx..update_end_idx { + let hash = if use_cache { + row_next_cached_hash(&mut ms.hash_cache, prefix, idx, hash_log, mls, hash_salt)? + } else { + hash_ptr_salted( + prefix.buf, + prefix.pos_of(idx), + hash_log + ZSTD_ROW_HASH_TAG_BITS, + mls, + hash_salt, + )? as u32 + }; + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; + tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; + hash_table.put(rel_row + pos, idx)?; + } + Ok(()) +} + +/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is +/// when compressing, not all of them if they are many. +#[inline(always)] +fn row_update_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + mls: u32, + row_log: u32, + row_mask: u32, + use_cache: bool, +) -> Result<()> { + const K_SKIP_THRESHOLD: u32 = 384; + const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; + const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; + + let mut idx = ms.next_to_update; + let target = prefix.index_of(ip); + + if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { + let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; + row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; + idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; + row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; + } + row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the +/// row-based match finder. +pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { + let row_log = ms.c_params.search_log.clamp(4, 6); + let row_mask = (1u32 << row_log) - 1; + let mls = ms.c_params.min_match.min(6); + row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) +} + +/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. +#[inline(always)] +fn row_get_match_mask( + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + head: u32, +) -> u64 { + let mut matches = 0u64; + let (chunks, _) = tag_row.as_chunks::<16>(); + for (n, chunk) in chunks.iter().enumerate() { + let mut chunk_matches = 0u16; + for (i, &entry) in chunk.iter().enumerate() { + chunk_matches |= u16::from(entry == tag) << i; + } + matches |= u64::from(chunk_matches) << (16 * n); + } + match ROW_ENTRIES { + 16 => u64::from((matches as u16).rotate_right(head)), + 32 => u64::from((matches as u32).rotate_right(head)), + _ => matches.rotate_right(head), + } +} + +/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, +/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than +/// `low_limit`. Returns how many there are. +#[inline(always)] +fn row_collect_matches( + row: &[u32; ROW_ENTRIES], + tag_row: &[u8; ROW_ENTRIES], + tag: u8, + low_limit: u32, + nb_attempts: &mut u32, + match_buffer: &mut [u32; ROW_ENTRIES], +) -> Result { + let row_mask = ROW_ENTRIES as u32 - 1; + let head = u32::from(tag_row.at(0)?) & row_mask; + let mut num_matches = 0usize; + let mut matches = row_get_match_mask(tag_row, tag, head); + + while matches > 0 && *nb_attempts > 0 { + let match_pos = (head + matches.trailing_zeros()) & row_mask; + matches &= matches - 1; + if match_pos == 0 { + // The head, which happens to be the same as the tag. + continue; + } + let match_index = row.at(match_pos as usize)?; + if match_index < low_limit { + break; + } + match_buffer.put(num_matches, match_index)?; + num_matches += 1; + *nb_attempts -= 1; + } + Ok(num_matches) +} + +/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there +/// is one. +#[inline(never)] +fn row_find_best_match( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + offset_ptr: &mut u32, +) -> Result { + let buf = src.buf; + let hash_log = ms.row_hash_log; + let dict_limit = ms.window.dict_limit; + let prefix_start = src.pos_of(dict_limit); + let dict_end = src.dict_pos_of(dict_limit); + let curr = src.index_of(ip); + let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); + let row_log = ROW_ENTRIES.trailing_zeros(); + let row_mask = ROW_ENTRIES as u32 - 1; + // There are no more searches than entries in a row. + let capped_search_log = ms.c_params.search_log.min(row_log); + let hash_salt = ms.hash_salt; + let mut nb_attempts = 1u32 << capped_search_log; + let mut ml = 4 - 1; + let mut match_buffer = [0u32; ROW_ENTRIES]; + + // Update the tables up to, and not including, `ip`. + let hash = if ms.lazy_skipping { + // While skipping, not every position is inserted. + ms.next_to_update = curr; + hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 + } else { + let prefix = src.prefix(); + row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; + row_next_cached_hash(&mut ms.hash_cache, prefix, curr, hash_log, MLS, hash_salt)? + }; + ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); + + { + let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; + let row = ms.hash_table.array_at_mut::(rel_row)?; + let tag_row = ms.tag_table.array_at_mut::(rel_row)?; + let num_matches = row_collect_matches( + row, + tag_row, + tag, + low_limit, + &mut nb_attempts, + &mut match_buffer, + )?; + + // Insert `ip`. + let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; + tag_row.put(pos, tag)?; + row.put(pos, ms.next_to_update)?; + ms.next_to_update += 1; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + if DICT_MODE != EXT_DICT || match_index >= dict_limit { + let match_pos = src.pos_of(match_index); + // Whether it can be longer. + if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { + current_ml = count(buf, ip, buf, match_pos, i_limit)?; + } + } else { + let match_pos = src.dict_pos_of(match_index); + if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + src.dict_buf, + match_pos + 4, + i_limit, + dict_end, + prefix_start, + )? + 4; + } + } + + if current_ml > ml { + ml = current_ml; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + if ip + current_ml == i_limit { + // The best there can be. The next attempt would read too far. + break; + } + } + } + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = src.dms.ok_or(BUG)?; + let dms_lowest_index = dms.ms.window.dict_limit; + let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); + + let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + let dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; + let dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + let num_matches = row_collect_matches( + dms.ms.hash_table.array_at::(dms_rel_row)?, + dms.ms.tag_table.array_at::(dms_rel_row)?, + (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, + dms_lowest_index, + &mut nb_attempts, + &mut match_buffer, + )?; + + for &match_index in match_buffer.sub(..num_matches)? { + let mut current_ml = 0; + let match_pos = dms.pos_of(match_index); + if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { + current_ml = count_2segments( + buf, + ip + 4, + dms.buf, + match_pos + 4, + i_limit, + dms.end, + prefix_start, + )? + 4; + } + + if current_ml > ml { + ml = current_ml; + let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + *offset_ptr = offset_to_offbase(offset); + if ip + current_ml == i_limit { + break; + } + } + } + } + + Ok(ml) +} + +// ── the parsers ─────────────────────────────────────────────────────────── + +// `searchMethod_e`, as an argument of a const generic. +const SEARCH_HASH_CHAIN: u8 = 0; +const SEARCH_BINARY_TREE: u8 = 1; +const SEARCH_ROW_HASH: u8 = 2; + +/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. +#[inline(always)] +fn search_max( + ms: &mut MatchState, + src: &BlockSrc<'_>, + ip: usize, + iend: usize, + offset_ptr: &mut u32, + mls: u32, + row_log: u32, +) -> Result { + match SEARCH_METHOD { + SEARCH_HASH_CHAIN => match mls { + 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + SEARCH_BINARY_TREE => match mls { + 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), + 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), + _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + _ => match (mls, row_log) { + (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), + (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), + }, + } +} + +/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match +/// state is searched. +#[derive(Clone, Copy)] +struct DictSegment<'a> { + buf: &'a [u8], + /// Its sum with an index of the window is a position in `buf`. It wraps around. + base: usize, + /// Where the segment starts in `buf`. + start: usize, + /// Where it ends. + end: usize, +} + +/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. +#[inline(always)] +fn rep_length( + ms: &MatchState, + src: &BlockSrc<'_>, + dict: &DictSegment<'_>, + ip: usize, + offset: u32, +) -> Result { + let buf = src.buf; + let iend = src.end; + if DICT_MODE == NO_DICT { + let rep_match = ip.wrapping_sub(offset as usize); + if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { + return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); + } + return Ok(0); + } + + let dict_limit = ms.window.dict_limit; + let curr = src.index_of(ip); + let rep_index = curr.wrapping_sub(offset); + if !index_overlap_check(dict_limit, rep_index) { + return Ok(0); + } + if DICT_MODE == EXT_DICT { + // The offset may lead to what is no longer in the window. + let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); + if offset > curr.wrapping_sub(window_low) { + return Ok(0); + } + } + let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { + let rep_match = dict.base.wrapping_add(rep_index as usize); + (dict.buf, rep_match, dict.end) + } else { + (buf, src.pos_of(rep_index), iend) + }; + if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { + return Ok(0); + } + let prefix_start = src.pos_of(dict_limit); + Ok(count_2segments( + buf, + ip + 4, + rep_buf, + rep_match + 4, + iend, + rep_end, + prefix_start, + )? + 4) +} + +/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions +/// further a better match is looked for, once there is one. +#[inline(never)] +fn compress_block_lazy_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let prefix_lowest_index = ms.window.dict_limit; + let prefix_lowest = src.pos_of(prefix_lowest_index); + let mls = ms.c_params.min_match.clamp(4, 6); + let row_log = ms.c_params.search_log.clamp(4, 6); + + let [rep_0, rep_1, _] = rep; + let mut offset_1 = *rep_0; + let mut offset_2 = *rep_1; + let mut offset_saved_1 = 0u32; + let mut offset_saved_2 = 0u32; + + ms.lazy_skipping = false; + + let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { + 8 + ZSTD_ROW_HASH_CACHE_SIZE + } else { + 8 + }; + let Some(ilimit) = iend.checked_sub(margin) else { + // There is no position to search, nor to hash. + return Ok(iend - istart); + }; + + let dict = match DICT_MODE { + NO_DICT => DictSegment { + buf: &[], + base: 0, + start: 0, + end: 0, + }, + EXT_DICT => DictSegment { + buf: src.dict_buf, + base: src.dict_base, + start: src.dict_pos_of(ms.window.low_limit), + end: src.dict_pos_of(prefix_lowest_index), + }, + _ => { + let dms = src.dms.ok_or(BUG)?; + let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); + DictSegment { + buf: dms.buf, + base: dms.base.wrapping_sub(dict_index_delta as usize), + start: dms.pos_of(dms.ms.window.dict_limit), + end: dms.end, + } + } + }; + + if DICT_MODE == EXT_DICT { + ip += usize::from(ip == prefix_lowest); + } else { + let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); + ip += usize::from(dict_and_prefix_length == 0); + } + if DICT_MODE == NO_DICT { + let curr = src.index_of(ip); + let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); + let max_rep = curr - window_low; + if offset_2 > max_rep { + offset_saved_2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved_1 = offset_1; + offset_1 = 0; + } + } + + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + + 'parse: while ip < ilimit { + let mut match_length; + let mut off_base = REPCODE1_TO_OFFBASE; + let mut start = ip + 1; + + 'found: { + // Check the repeat offset. + match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; + if DEPTH == 0 && match_length != 0 { + break 'found; + } + + // First search. + { + let mut off_base_found = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut off_base_found, + mls, + row_log, + )?; + if ml2 > match_length { + match_length = ml2; + start = ip; + off_base = off_base_found; + } + } + + if match_length < 4 { + // Jump faster over incompressible sections. + let step = (ip - anchor) >> K_SEARCH_STRENGTH; + ip += step + 1; + // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of + // no use out of the tables. As in the C code, it does not start at the same step in all modes. + ms.lazy_skipping = if DICT_MODE == EXT_DICT { + step > K_LAZY_SKIPPING_STEP + } else { + step + 1 > K_LAZY_SKIPPING_STEP + }; + continue 'parse; + } + + // Look for something better at the next positions. + if DEPTH >= 1 { + while ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 3) as i32; + let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + + // Nothing better: try one position further. + if DEPTH == 2 && ip < ilimit { + ip += 1; + if off_base != 0 { + let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; + let gain2 = (ml_rep * 4) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; + if ml_rep >= 4 && gain2 > gain1 { + match_length = ml_rep; + off_base = REPCODE1_TO_OFFBASE; + start = ip; + } + } + { + let mut ofb_candidate = 999_999_999u32; + let ml2 = search_max::( + ms, + src, + ip, + iend, + &mut ofb_candidate, + mls, + row_log, + )?; + let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; + let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; + if ml2 >= 4 && gain2 > gain1 { + match_length = ml2; + off_base = ofb_candidate; + start = ip; + continue; + } + } + } + break; + } + } + + // Catch up: the match may start earlier. + if offbase_is_offset(off_base) { + let offset = offbase_to_offset(off_base); + let match_index = src.index_of(start).wrapping_sub(offset); + let (match_buf, mut match_pos, m_start) = + if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { + let match_pos = dict.base.wrapping_add(match_index as usize); + (dict.buf, match_pos, dict.start) + } else { + (buf, src.pos_of(match_index), prefix_lowest) + }; + while start > anchor + && match_pos > m_start + && buf.at(start - 1)? == match_buf.at(match_pos - 1)? + { + start -= 1; + match_pos -= 1; + match_length += 1; + } + offset_2 = offset_1; + offset_1 = offset; + } + } + + // Store the sequence. + seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; + ip = start + match_length; + anchor = ip; + + if ms.lazy_skipping { + // There was a match: every position is inserted again. + if SEARCH_METHOD == SEARCH_ROW_HASH { + row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; + } + ms.lazy_skipping = false; + } + + // Check the second repeat offset right away. + while ip <= ilimit { + match_length = rep_length::(ms, src, &dict, ip, offset_2)?; + if match_length == 0 { + break; + } + (offset_1, offset_2) = (offset_2, offset_1); + seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; + ip += match_length; + anchor = ip; + } + } + + // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. + if offset_saved_1 != 0 && offset_1 != 0 { + offset_saved_2 = offset_saved_1; + } + + // Save the repeat offsets for the next block. + *rep_0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved_1 + }; + *rep_1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved_2 + }; + + // The size of the last literals. + Ok(iend - anchor) +} + +/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. +#[inline(always)] +fn compress_block_lazy_any_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + match dict_mode { + DictMode::NoDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::ExtDict => { + compress_block_lazy_generic::(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + compress_block_lazy_generic::( + ms, seq_store, rep, src, + ) + } + } +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row: bool, + dict_mode: DictMode, +) -> Result { + match (strategy, use_row) { + (Strategy::Greedy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Greedy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, false) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::Lazy2, true) => { + compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + } + (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( + ms, seq_store, rep, src, dict_mode, + ), + _ => Err(BUG), + } +} diff --git a/src/zstd/wip/work/ldm/zstd/compress/ldm.rs b/src/zstd/wip/work/ldm/zstd/compress/ldm.rs new file mode 100644 index 000000000000..7f060fcda748 --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/compress/ldm.rs @@ -0,0 +1,779 @@ +//! `zstd_ldm.c`: long distance matching. + +use super::internal::{ + BlockSrc, Buffers, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, LdmEntry, + LdmParams, LdmState, MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, + TableFillPurpose, count, count_2segments, offset_to_offbase, +}; +use super::{compress_block, double_fast, fast}; +use crate::common::ZSTD_REP_NUM; +use crate::common::xxhash::Xxh64; +use crate::error::{ErrorCode, Result}; +use crate::util::{BUG, SliceExt}; + +const ZSTD_HASHLOG_MIN: u32 = 6; +const ZSTD_HASHLOG_MAX: u32 = 30; +const ZSTD_LDM_BUCKETSIZELOG_MAX: u32 = 8; + +const LDM_BUCKET_SIZE_LOG: u32 = 4; +const LDM_MIN_MATCH_LENGTH: u32 = 64; +const LDM_BATCH_SIZE: usize = 64; + +/// `ZSTD_ldm_gearTab` +#[rustfmt::skip] +static GEAR_TAB: [u64; 256] = [ + 0xf5b8_f72c_5f77_775c, 0x8493_5f26_6b7a_c412, 0xb647_ada9_ca73_0ccc, 0xb065_bb4b_114f_b1de, + 0x3458_4e7e_8c3a_9fd0, 0x4e97_e17c_6ae2_6b05, 0x3a03_d743_bc99_a604, 0xcecd_0424_22c4_044f, + 0x76de_76c5_8524_259e, 0x9c85_28f6_5bad_eaca, 0x8656_3706_e209_7529, 0x2902_475f_a375_d889, + 0xafb3_2a97_39a5_ebe6, 0xce27_14da_3883_e639, 0x021e_af82_1722_e69e, 0x0003_7b62_8620_b628, + 0x049a_8d45_5d88_caf5, 0x8556_d711_e695_8140, 0x04f7_ae74_fc60_5c1f, 0x829f_0c34_68bd_3a20, + 0x4ffd_c885_c625_179e, 0x8473_de04_8a3d_af1b, 0x5100_8822_b056_46b2, 0x69d7_5d12_b2d1_cc5f, + 0x8c9d_4a19_1591_54bc, 0xc3cc_10f4_abbd_4003, 0xd06d_dc1c_ecb9_7391, 0xbe48_e6e7_ed80_302e, + 0x3481_db31_cee0_3547, 0xacc3_f67c_daa1_d210, 0x65cb_771d_8c7f_96cc, 0x8eb2_7177_0557_23dd, + 0xc789_950d_44cd_94be, 0x934f_eadc_3700_b12b, 0x5e48_5f11_edbd_f182, 0x1e2e_2a46_fd64_767a, + 0x2969_ca71_d82e_fa7c, 0x9d46_e993_5ebb_ba2e, 0xe056_b67e_05e6_822b, 0x94d7_3f55_739d_03a0, + 0xcd70_10bd_b69b_5a03, 0x455e_f9fc_d79b_82f4, 0x869c_b54a_8749_c161, 0x38d1_a4fa_6185_d225, + 0xb475_166f_94bb_e9bb, 0xa414_3548_7209_59f1, 0x7aed_4780_ba6b_26ba, 0xd0ce_2644_39e0_2312, + 0x8436_6d74_6078_d508, 0xa8ce_973c_72ed_17be, 0x21c3_23a2_9a43_0b01, 0x9962_d617_e3af_80ee, + 0xab0c_e91d_9c8c_f75b, 0x530e_8ee6_d19a_4dbc, 0x2ef6_8c0c_f53f_5d72, 0xc03a_6816_40a8_5506, + 0x496e_4e9f_9c31_0967, 0x7858_0472_b59b_14a0, 0x2738_24c2_3b38_8577, 0x66bf_923a_d45c_b553, + 0x47ae_1a5a_2492_ba86, 0x35e3_0456_9e22_9659, 0x4765_182a_4687_0b6f, 0x6cba_b625_e909_9412, + 0xddac_9a2e_5985_22c1, 0x7172_086e_6666_24f2, 0xdf50_03ca_503b_7837, 0x88c0_c1db_7856_3d09, + 0x58d5_1865_acfc_289d, 0x1776_71ae_c652_24f1, 0xfb79_d8a2_41e9_67d7, 0x2be1_e101_cad9_a49a, + 0x6625_682f_6e29_186b, 0x3995_5345_7ac0_6e50, 0x035d_ffb4_c23a_bb74, 0x429d_b259_1f54_aade, + 0xc528_02a8_037d_1009, 0x6acb_2738_1f0b_25f3, 0xf45e_2551_ee4f_823b, 0x8b0e_a2d9_9580_c2f7, + 0x3bed_519c_bcb4_e1e1, 0x0ff4_5282_3dbb_010a, 0x9d42_ed61_4f3d_d267, 0x5b93_13c0_6257_c57b, + 0xa114_b800_8b5e_1442, 0xc1fe_311c_11c1_3d4b, 0x66e8_763e_a34c_5568, 0x8b98_2af1_c262_f05d, + 0xee88_76fa_aa75_fbb7, 0x8a62_a4d0_d172_bb2a, 0xc13d_94a3_b744_9a97, 0x6dbb_ba9d_c15d_037c, + 0xc786_101f_1d92_e0f1, 0xd786_81a9_07a0_b79b, 0xf61a_af29_62c9_abb9, 0x2cfd_16fc_d3cb_7ad9, + 0x868c_5b67_4462_4d21, 0x25e6_5089_9c74_ddd7, 0xba04_2af4_a7c3_7463, 0x4eb1_a539_465a_3eca, + 0xbe09_dbf0_3b05_d5ca, 0x774e_5a36_2b54_72ba, 0x47a1_2212_29d1_83cd, 0x504b_0ca1_8ef5_a2df, + 0xdffb_dfbd_e245_6eb9, 0x46cd_2b2f_bee3_4634, 0xf2ae_f8fe_819d_98c3, 0x357f_5276_d459_9d61, + 0x24a5_4838_79c4_53e3, 0x0880_2688_9192_b4b9, 0x28da_9667_1782_dbec, 0x4ef3_7c40_588e_9aaa, + 0x8837_b906_51bc_9fb3, 0xc164_f741_d3f0_e5d6, 0xbc13_5a0a_704b_70ba, 0x069c_d868_f762_2ada, + 0xbc37_ba89_e0b9_c0ab, 0x47c1_4a01_3235_52f6, 0x4f00_794b_acee_98bb, 0x7107_de7d_637a_69d5, + 0x88af_793b_b6f2_255e, 0xf3c6_466b_8799_b598, 0xc288_c616_aa7f_3b59, 0x81ca_63cf_42fc_a3fd, + 0x88d8_5ace_36a2_674b, 0x0d05_6bd3_7923_89e7, 0xe55c_396c_4e9d_d32d, 0xbefb_5045_71e6_c0a6, + 0x96ab_3211_5e91_e8cc, 0xbf8a_cb18_de8f_38d1, 0x66da_e588_0167_2606, 0x833b_6017_8723_17fb, + 0xb87c_16f2_d1c9_2864, 0xdb76_6a74_e58b_669c, 0x8965_9f85_c614_17be, 0xc8da_ad85_6011_ea0c, + 0x76a4_b565_b6fe_7eae, 0xa469_d085_f623_7312, 0xaaf0_3656_83a3_e96c, 0x4dbb_746f_8424_f7b8, + 0x0638_755a_f4e4_acc1, 0x3d78_07f5_bde6_4486, 0x17be_6d8f_5bbb_7639, 0x0903_f0cd_44dc_35dc, + 0x67b6_72ea_fdf1_196c, 0xa676_ff93_ed4c_82f1, 0x521d_1004_c505_3d9d, 0x37ba_9ad0_9ccc_9202, + 0x84e5_4d29_7aac_fb51, 0x0a0b_4b77_6a14_3445, 0x0820_d471_e20b_348e, 0x1874_383c_b83d_46dc, + 0x97ed_eec7_a1ef_e11c, 0xb330_e50b_1bdc_42aa, 0x1dd9_1955_ce70_e032, 0xa514_cdb8_8f29_39d5, + 0x2791_233f_d90d_b9d3, 0x7b67_0a4c_c50f_7a9b, 0x77c0_7d2a_05c6_dfa5, 0xe377_8b66_46d0_a6fa, + 0xb39c_8eda_47b5_6749, 0x933e_d448_addb_ef28, 0xaf84_6af6_ab7d_0bf4, 0x0e5a_f208_eb66_6e49, + 0x5e66_22f7_3534_cd6a, 0x297d_aeca_42ef_5b6e, 0x862d_aef3_d355_39a6, 0xe687_2249_8f8e_1ea9, + 0x981c_5309_3dc0_d572, 0xfa09_b0bf_bf86_fbf5, 0x30b1_e961_6621_9f15, 0x70e7_d466_bdc4_fb83, + 0x5a66_736e_35f2_a8e9, 0xcddb_59d2_b7c1_baef, 0xd6c7_d247_d26d_8996, 0xea4e_39ea_c8de_1ba3, + 0x539c_8bb1_9fa3_aff2, 0x09f9_0e4c_5fd5_08d8, 0xa34e_5956_fbaf_3385, 0x2e2f_8e15_1d3e_f375, + 0x1736_91e9_b83f_aec1, 0xb85a_8d56_bf01_6379, 0x8382_3812_6740_8ae3, 0xb90f_901b_bdc0_096d, + 0x7c6a_d329_33bc_ec65, 0x76bb_5e2f_2c8a_d595, 0x390f_851a_6cf4_6d28, 0xc3e6_064d_a1c2_da72, + 0xc52a_0c10_1cfa_5389, 0xd78e_af84_a3fb_c530, 0x3781_b9e2_288b_997e, 0x73c2_f6de_a83d_05c4, + 0x0422_8e36_4c5b_5ed7, 0x9d7a_3edf_0da4_3911, 0x8edc_feda_2468_6756, 0x5e76_67a7_b7a9_b3a1, + 0x4c4f_389f_a143_791d, 0xb08b_c102_3da7_cddc, 0x7ab4_be3a_e529_b1cc, 0x754e_6132_dbe7_4ff9, + 0x7163_5442_a839_df45, 0x2f6f_b164_3fbe_52de, 0x961e_0a42_cf7a_8177, 0xf3b4_5d83_d89e_f2ea, + 0xee3d_e4cf_4a6e_3e9b, 0xcd68_4854_2c32_95e7, 0xe4ce_e166_4c78_662f, 0x9947_548b_474c_68c4, + 0x25d7_3777_a5ed_8b0b, 0x00c9_15b1_d636_b7fc, 0x21c2_ba75_d9b0_d2da, 0x5f6b_5dcf_608a_64a1, + 0xdcf3_3325_5ff9_570c, 0x633b_9224_18ce_d4ee, 0xc136_dde0_b004_b34a, 0x58cc_83b0_5d4b_2f5a, + 0x5eb4_24dd_a28e_42d2, 0x62df_4736_9739_cd98, 0xb4e0_b424_85e4_ce17, 0x16e1_f0c1_f9a8_d1e7, + 0x8ec3_9167_0756_0ebf, 0x62ba_6e2d_f2cc_9db3, 0xcbf9_f4ff_77d8_3a16, 0x78d9_d7d0_7d2b_bcc4, + 0xef55_4ce1_e02c_41f4, 0x8d75_8112_7ecc_f94d, 0xa9b5_3336_cb3c_8a05, 0x38c4_2c0b_f45c_4f91, + 0x6408_93cd_f448_8863, 0x80ec_34bc_575e_a568, 0x39f3_24f5_b48e_aa40, 0xe9d9_ed1f_8eff_527f, + 0x9224_fc05_8cc5_a214, 0xbaba_00b0_4cfe_7741, 0x309a_9f12_0fcf_52af, 0xa558_f3ec_6562_6212, + 0x424b_ec8b_7ada_be2f, 0x4162_2513_a6ae_a433, 0xb88d_a2d5_324c_a798, 0xd287_733b_2455_28a4, + 0x9a44_697e_6d68_aec3, 0x7b10_93be_2f49_bb28, 0x50bb_ec63_2e3d_8aad, 0x6cd9_0723_e1ea_8283, + 0x897b_9e74_31b0_2bf3, 0x219e_fdcb_338a_7047, 0x3b03_11f0_a27c_0656, 0xdb17_bf91_c0db_96e7, + 0x8cd4_fd6b_4e85_a5b2, 0xfab0_7105_4ba6_409d, 0x40d6_fe83_1fa9_dfd9, 0xaf35_8deb_ad7d_791e, + 0xeb8d_0e25_a65e_3e58, 0xbbcb_d3df_14e0_8580, 0x0cf7_51f2_7ecd_ab2b, 0x2b4d_a14f_2613_d8f4, +]; + +/// `ldmRollingHashState_t` +struct RollingHashState { + rolling: u64, + stop_mask: u64, +} + +impl RollingHashState { + /// `ZSTD_ldm_gear_init` + fn new(params: &LdmParams) -> Self { + let max_bits_in_mask = params.min_match_length.min(64); + let hash_rate_log = params.hash_rate_log; + let rate_mask = (1u64 << (hash_rate_log & 63)) - 1; + // A split is to happen every `1 << hash_rate_log` bytes on average, and bit n of the hash depends on the last + // n bytes: the mask has the highest bits that depend on no more than `min_match_length` bytes. + let stop_mask = if hash_rate_log > 0 && hash_rate_log <= max_bits_in_mask { + rate_mask << ((max_bits_in_mask - hash_rate_log) & 63) + } else { + rate_mask + }; + Self { + rolling: u64::from(u32::MAX), + stop_mask, + } + } + + /// `ZSTD_ldm_gear_feed`: writes to `splits` the split points of `data`, as positions in it. Stops at the end of + /// `data` or once `splits` is full. Returns how many bytes it went through, and how many splits it found. + #[inline] + fn feed(&mut self, data: &[u8], splits: &mut [usize; LDM_BATCH_SIZE]) -> (usize, usize) { + let mut hash = self.rolling; + let mask = self.stop_mask; + let mut n = 0usize; + let mut num_splits = 0usize; + // Returns whether `splits` is full. + let mut iter_once = |byte: u8| { + let gear = GEAR_TAB.get(usize::from(byte)).copied().unwrap_or(0); + hash = (hash << 1).wrapping_add(gear); + n += 1; + if hash & mask == 0 { + if let Some(split) = splits.get_mut(num_splits) { + *split = n; + } + num_splits += 1; + return num_splits == LDM_BATCH_SIZE; + } + false + }; + let (quads, rest) = data.as_chunks::<4>(); + let full = quads + .iter() + .any(|&[a, b, c, d]| iter_once(a) || iter_once(b) || iter_once(c) || iter_once(d)); + if !full { + for &byte in rest { + if iter_once(byte) { + break; + } + } + } + self.rolling = hash; + (n, num_splits) + } +} + +/// `XXH64`, with a seed of 0. +#[inline] +fn xxh64(data: &[u8]) -> u64 { + let mut state = Xxh64::new(); + state.update(data); + state.digest() +} + +/// `ZSTD_ldm_adjustParameters`: gives a value to the parameters that are 0. `c_params` are those that are applied. +pub(crate) fn adjust_parameters(params: &mut LdmParams, c_params: &CompressionParameters) { + let strategy = c_params.strat(); + params.window_log = c_params.window_log; + if params.hash_rate_log == 0 { + if params.hash_log > 0 { + if params.window_log > params.hash_log { + params.hash_rate_log = params.window_log - params.hash_log; + } + } else { + // From 7 for `fast` to 4 for `btultra2`. + params.hash_rate_log = 7 - (strategy as u32 / 3); + } + } + if params.hash_log == 0 { + params.hash_log = params + .window_log + .wrapping_sub(params.hash_rate_log) + .clamp(ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + } + if params.min_match_length == 0 { + params.min_match_length = LDM_MIN_MATCH_LENGTH; + if strategy >= Strategy::BtUltra { + params.min_match_length /= 2; + } + } + if params.bucket_size_log == 0 { + params.bucket_size_log = + (strategy as u32).clamp(LDM_BUCKET_SIZE_LOG, ZSTD_LDM_BUCKETSIZELOG_MAX); + } + params.bucket_size_log = params.bucket_size_log.min(params.hash_log); +} + +/// `ZSTD_cwksp_alloc_size`, which adds something only under ASAN. +const fn cwksp_alloc_size(size: usize) -> usize { + size +} + +/// `ZSTD_ldm_getTableSize`: how many bytes of workspace the C code takes for the tables. +pub(crate) fn get_table_size(params: LdmParams) -> usize { + let ldm_h_size = 1usize << (params.hash_log & 63); + let ldm_bucket_size_log = params.bucket_size_log.min(params.hash_log); + let ldm_bucket_size = 1usize << ((params.hash_log - ldm_bucket_size_log) & 63); + let total_size = cwksp_alloc_size(ldm_bucket_size) + + cwksp_alloc_size(ldm_h_size.wrapping_mul(size_of::())); + if params.enable_ldm == ParamSwitch::Enable { + total_size + } else { + 0 + } +} + +/// `ZSTD_ldm_getMaxNbSeq`: the most sequences that `max_chunk_size` bytes can give. +pub(crate) fn get_max_nb_seq(params: LdmParams, max_chunk_size: usize) -> usize { + if params.enable_ldm == ParamSwitch::Enable { + max_chunk_size + .checked_div(params.min_match_length as usize) + .unwrap_or(0) + } else { + 0 + } +} + +/// The mask that keeps, of a hash, the number of a bucket. +#[inline] +const fn bucket_mask(params: &LdmParams) -> u32 { + (1u32 << (params.hash_log.wrapping_sub(params.bucket_size_log) & 31)) - 1 +} + +/// `ZSTD_ldm_getBucket`: where the bucket `hash` starts in the hash table. +#[inline(always)] +const fn get_bucket(hash: usize, bucket_size_log: u32) -> usize { + hash << (bucket_size_log & 31) +} + +/// `ZSTD_ldm_insertEntry`: in place of the oldest entry of the bucket `hash`. +#[inline] +fn insert_entry( + ldm_state: &mut LdmState, + hash: usize, + entry: LdmEntry, + bucket_size_log: u32, +) -> Result<()> { + let p_offset = ldm_state.bucket_offsets.at_mut(hash)?; + let offset = u32::from(*p_offset); + *p_offset = ((offset + 1) & ((1u32 << (bucket_size_log & 31)) - 1)) as u8; + ldm_state + .hash_table + .put(get_bucket(hash, bucket_size_log) + offset as usize, entry) +} + +/// `ZSTD_ldm_countBackwardsMatch`: how many bytes before `p_in` in `buf`, down to `p_anchor`, are the same as those +/// before `p_match` in `match_buf`, down to `p_match_base`. +#[inline] +fn count_backwards_match( + buf: &[u8], + p_in: usize, + p_anchor: usize, + match_buf: &[u8], + p_match: usize, + p_match_base: usize, +) -> Result { + Ok(buf + .sub(p_anchor..p_in)? + .iter() + .rev() + .zip(match_buf.sub(p_match_base..p_match)?.iter().rev()) + .take_while(|(in_byte, match_byte)| in_byte == match_byte) + .count()) +} + +/// `ZSTD_ldm_countBackwardsMatch_2segments`, for a match in the current segment, which starts at `p_match_base`: from +/// there, the match goes on at the end of the older segment. A match in the older segment has nowhere to go on. +#[inline] +fn count_backwards_match_2segments( + buf: &[u8], + p_in: usize, + p_anchor: usize, + p_match: usize, + p_match_base: usize, + dict_buf: &[u8], + p_ext_dict_start: usize, + p_ext_dict_end: usize, +) -> Result { + let match_length = count_backwards_match(buf, p_in, p_anchor, buf, p_match, p_match_base)?; + if p_match - match_length != p_match_base { + return Ok(match_length); + } + Ok(match_length + + count_backwards_match( + buf, + p_in - match_length, + p_anchor, + dict_buf, + p_ext_dict_end, + p_ext_dict_start, + )?) +} + +/// `ZSTD_ldm_fillFastTables`: the strategies that index only what they search have to be told about what they skip. +fn fill_fast_tables(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) -> Result<()> { + match ms.c_params.strat() { + Strategy::Fast => fast::fill_hash_table( + ms, + prefix, + end, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + ), + Strategy::DFast => double_fast::fill_double_hash_table( + ms, + prefix, + end, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + ), + Strategy::Greedy + | Strategy::Lazy + | Strategy::Lazy2 + | Strategy::BtLazy2 + | Strategy::BtOpt + | Strategy::BtUltra + | Strategy::BtUltra2 => Ok(()), + } +} + +/// `ZSTD_ldm_fillHashTable`: indexes the content of a dictionary, which is `prefix.buf` from `ip` to `iend`. +pub(crate) fn fill_hash_table( + ldm_state: &mut LdmState, + prefix: Prefix<'_>, + mut ip: usize, + iend: usize, + params: &LdmParams, +) -> Result<()> { + let min_match_length = params.min_match_length as usize; + let bucket_mask = bucket_mask(params); + let istart = ip; + let mut hash_state = RollingHashState::new(params); + let mut splits = [0usize; LDM_BATCH_SIZE]; + + while ip < iend { + let (hashed, num_splits) = hash_state.feed(prefix.buf.sub(ip..iend)?, &mut splits); + for &split in splits.iter().take(num_splits) { + if ip + split >= istart + min_match_length { + let split = ip + split - min_match_length; + let xxhash = xxh64(prefix.buf.span(split, min_match_length)?); + let hash = xxhash as u32 & bucket_mask; + let entry = LdmEntry { + offset: prefix.index_of(split), + checksum: (xxhash >> 32) as u32, + }; + insert_entry(ldm_state, hash as usize, entry, params.bucket_size_log)?; + } + } + ip += hashed; + } + Ok(()) +} + +/// `ZSTD_ldm_limitTableUpdate`: if the tables of `ms` are far behind `anchor`, skips most of the way. +fn limit_table_update(ms: &mut MatchState, src: &BlockSrc<'_>, anchor: usize) { + let curr = src.index_of(anchor); + if curr > ms.next_to_update.wrapping_add(1024) { + let behind = curr.wrapping_sub(ms.next_to_update).wrapping_sub(1024); + ms.next_to_update = curr.wrapping_sub(behind.min(512)); + } +} + +/// `ldmMatchCandidate_t` +#[derive(Clone, Copy, Default)] +struct MatchCandidate { + split: usize, + hash: u32, + checksum: u32, + /// Where the bucket starts in the hash table. + bucket: usize, +} + +/// `ZSTD_ldm_generateSequences_internal`: the input is from `istart` to `iend` in the buffer of the current segment. +/// Returns how many bytes at its end are in no sequence. +fn generate_sequences_internal( + ldm_state: &mut LdmState, + raw_seq_store: &mut RawSeqStore, + params: &LdmParams, + bufs: &Buffers<'_>, + istart: usize, + iend: usize, +) -> Result { + let window = ldm_state.window; + let ext_dict = window.has_ext_dict(); + let min_match_length = params.min_match_length as usize; + let ents_per_bucket = 1usize << (params.bucket_size_log & 31); + let bucket_mask = bucket_mask(params); + let dict_limit = window.dict_limit; + let lowest_index = if ext_dict { + window.low_limit + } else { + dict_limit + }; + let prefix = Prefix { + buf: bufs.get(window.base_buf), + base: window.base.wrapping_sub(window.base_buf.start()), + }; + let buf = prefix.buf; + let dict_buf = bufs.get(window.dict_buf); + let dict_base = window.dict_base.wrapping_sub(window.dict_buf.start()); + let dict_start = dict_base.wrapping_add(lowest_index as usize); + let dict_end = dict_base.wrapping_add(dict_limit as usize); + let low_prefix = prefix.pos_of(dict_limit); + let ilimit = iend.saturating_sub(HASH_READ_SIZE); + let mut anchor = istart; + let mut ip = istart; + let mut splits = [0usize; LDM_BATCH_SIZE]; + let mut candidates = [MatchCandidate::default(); LDM_BATCH_SIZE]; + + if iend.checked_sub(istart).ok_or(BUG)? < min_match_length { + return Ok(iend - anchor); + } + + // Here and where it skips over a match, the C code calls `ZSTD_ldm_gear_reset`, which computes the hash of the + // `min_match_length` bytes before `ip` and drops it: the rolling hash goes on from what it was. + let mut hash_state = RollingHashState::new(params); + ip += min_match_length; + + while ip < ilimit { + let (hashed, num_splits) = hash_state.feed(buf.sub(ip..ilimit)?, &mut splits); + + for (candidate, &split) in candidates.iter_mut().zip(&splits).take(num_splits) { + let split = ip + split - min_match_length; + let xxhash = xxh64(buf.span(split, min_match_length)?); + let hash = xxhash as u32 & bucket_mask; + *candidate = MatchCandidate { + split, + hash, + checksum: (xxhash >> 32) as u32, + bucket: get_bucket(hash as usize, params.bucket_size_log), + }; + } + + for candidate in candidates.iter().take(num_splits) { + let MatchCandidate { + split, + hash, + checksum, + bucket, + } = *candidate; + let hash = hash as usize; + let mut forward_match_length = 0usize; + let mut backward_match_length = 0usize; + let mut best_match_length = 0usize; + let mut best_entry = None; + let new_entry = LdmEntry { + offset: prefix.index_of(split), + checksum, + }; + + // A split inside the previous match is indexed, but not searched. + if split < anchor { + insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; + continue; + } + + for cur in ldm_state.hash_table.span(bucket, ents_per_bucket)? { + if cur.checksum != checksum || cur.offset <= lowest_index { + continue; + } + let (cur_forward_match_length, cur_backward_match_length) = if ext_dict + && cur.offset < dict_limit + { + let p_match = dict_base.wrapping_add(cur.offset as usize); + let forward = + count_2segments(buf, split, dict_buf, p_match, iend, dict_end, low_prefix)?; + if forward < min_match_length { + continue; + } + let backward = + count_backwards_match(buf, split, anchor, dict_buf, p_match, dict_start)?; + (forward, backward) + } else { + let p_match = prefix.pos_of(cur.offset); + let forward = count(buf, split, buf, p_match, iend)?; + if forward < min_match_length { + continue; + } + let backward = if ext_dict { + count_backwards_match_2segments( + buf, split, anchor, p_match, low_prefix, dict_buf, dict_start, dict_end, + )? + } else { + count_backwards_match(buf, split, anchor, buf, p_match, low_prefix)? + }; + (forward, backward) + }; + let cur_total_match_length = cur_forward_match_length + cur_backward_match_length; + + if cur_total_match_length > best_match_length { + best_match_length = cur_total_match_length; + forward_match_length = cur_forward_match_length; + backward_match_length = cur_backward_match_length; + best_entry = Some(*cur); + } + } + + let Some(best_entry) = best_entry else { + insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; + continue; + }; + + let offset = new_entry.offset.wrapping_sub(best_entry.offset); + let m_length = forward_match_length + backward_match_length; + if raw_seq_store.size == raw_seq_store.capacity { + return Err(ErrorCode::DstSizeTooSmall); + } + raw_seq_store.seq.put( + raw_seq_store.size, + RawSeq { + offset, + lit_length: (split - backward_match_length - anchor) as u32, + match_length: m_length as u32, + }, + )?; + raw_seq_store.size += 1; + + // Only now, so that the position does not match itself. + insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; + + anchor = split + forward_match_length; + + // The match goes beyond what was hashed: what is left of it is skipped, and the other candidates dropped. + if anchor > ip + hashed { + ip = anchor - hashed; + break; + } + } + + ip += hashed; + } + + Ok(iend - anchor) +} + +/// `ZSTD_ldm_reduceTable` +fn reduce_table(table: &mut [LdmEntry], reducer_value: u32) { + for entry in table { + entry.offset = entry.offset.saturating_sub(reducer_value); + } +} + +/// `ZSTD_ldm_generateSequences`: adds to `sequences` the long distance matches of the input, which is the bytes from +/// `start` to `end` of the buffer of the current segment of the window, and which the window was updated with. +/// Stops early, without an error, at a chunk boundary where `sequences` is full. +pub(crate) fn generate_sequences( + ldm_state: &mut LdmState, + sequences: &mut RawSeqStore, + params: &LdmParams, + bufs: &Buffers<'_>, + start: usize, + end: usize, +) -> Result<()> { + const K_MAX_CHUNK_SIZE: usize = 1 << 20; + let max_dist = 1u32 << (params.window_log & 31); + let buf_start = ldm_state.window.base_buf.start(); + let mut leftover_size = 0usize; + let mut chunk_start = start; + + while chunk_start < end && sequences.size < sequences.capacity { + let chunk_end = end.min(chunk_start + K_MAX_CHUNK_SIZE); + let chunk_size = chunk_end - chunk_start; + let prev_size = sequences.size; + + if ldm_state + .window + .need_overflow_correction(buf_start + chunk_end) + { + let ldm_h_size = 1usize << (params.hash_log & 31); + let correction = + ldm_state + .window + .correct_overflow(0, max_dist, buf_start + chunk_start); + reduce_table(ldm_state.hash_table.sub_mut(..ldm_h_size)?, correction); + // An overflow correction invalidates the dictionary. + ldm_state.loaded_dict_end = 0; + } + // The whole chunk is searched with the window that its end has, so that no match is too far for any position + // of it, at the cost of some matches at its start. + if ldm_state.window.enforce_max_dist( + buf_start + chunk_end, + max_dist, + ldm_state.loaded_dict_end, + ) { + ldm_state.loaded_dict_end = 0; + } + let new_leftover_size = generate_sequences_internal( + ldm_state, + sequences, + params, + bufs, + chunk_start, + chunk_end, + )?; + // What the previous chunks left goes to the literals of the first sequence of this one. + if prev_size < sequences.size { + let first = sequences.seq.at_mut(prev_size)?; + first.lit_length = first.lit_length.wrapping_add(leftover_size as u32); + leftover_size = new_leftover_size; + } else { + leftover_size += chunk_size; + } + chunk_start = chunk_end; + } + Ok(()) +} + +/// `ZSTD_ldm_skipSequences`: moves `src_size` bytes forward in the sequences. A match of which less than `min_match` +/// is left becomes literals of the next sequence. +pub(crate) fn skip_sequences( + raw_seq_store: &mut RawSeqStore, + mut src_size: usize, + min_match: u32, +) -> Result<()> { + while src_size > 0 && raw_seq_store.pos < raw_seq_store.size { + let seq = raw_seq_store.seq.at_mut(raw_seq_store.pos)?; + if src_size <= seq.lit_length as usize { + seq.lit_length -= src_size as u32; + return Ok(()); + } + src_size -= seq.lit_length as usize; + seq.lit_length = 0; + if src_size < seq.match_length as usize { + seq.match_length -= src_size as u32; + if seq.match_length < min_match { + let match_length = seq.match_length; + if raw_seq_store.pos + 1 < raw_seq_store.size { + let next = raw_seq_store.seq.at_mut(raw_seq_store.pos + 1)?; + next.lit_length = next.lit_length.wrapping_add(match_length); + } + raw_seq_store.pos += 1; + } + return Ok(()); + } + src_size -= seq.match_length as usize; + seq.match_length = 0; + raw_seq_store.pos += 1; + } + Ok(()) +} + +/// `maybeSplitSequence`: the next sequence, cut to the `remaining` bytes of the block. What is cut off stays in the +/// store. If there is no match of `min_match` left in the block, the offset is 0. +fn maybe_split_sequence( + raw_seq_store: &mut RawSeqStore, + remaining: u32, + min_match: u32, +) -> Result { + let mut sequence = raw_seq_store.seq.at(raw_seq_store.pos)?; + let length = sequence.lit_length.wrapping_add(sequence.match_length); + if remaining >= length { + raw_seq_store.pos += 1; + return Ok(sequence); + } + if remaining <= sequence.lit_length { + sequence.offset = 0; + } else { + sequence.match_length = remaining - sequence.lit_length; + if sequence.match_length < min_match { + sequence.offset = 0; + } + } + skip_sequences(raw_seq_store, remaining as usize, min_match)?; + Ok(sequence) +} + +/// `ZSTD_ldm_skipRawSeqStoreBytes`: moves `nb_bytes` forward in the sequences, leaving them as they are. +pub(crate) fn skip_raw_seq_store_bytes( + raw_seq_store: &mut RawSeqStore, + nb_bytes: usize, +) -> Result<()> { + let mut curr_pos = raw_seq_store.pos_in_sequence.wrapping_add(nb_bytes) as u32; + while curr_pos != 0 && raw_seq_store.pos < raw_seq_store.size { + let curr_seq = raw_seq_store.seq.at(raw_seq_store.pos)?; + let length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); + if curr_pos >= length { + curr_pos -= length; + raw_seq_store.pos += 1; + } else { + raw_seq_store.pos_in_sequence = curr_pos as usize; + break; + } + } + if curr_pos == 0 || raw_seq_store.pos == raw_seq_store.size { + raw_seq_store.pos_in_sequence = 0; + } + Ok(()) +} + +/// `ZSTD_ldm_blockCompress`: finds the sequences of the block `src`, given the long distance matches, which it moves +/// forward in by the size of the block. Returns how many bytes at the end of the block are in no sequence. +pub(crate) fn block_compress( + raw_seq_store: &mut RawSeqStore, + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + use_row_match_finder: bool, + src: &BlockSrc<'_>, +) -> Result { + let strategy = ms.c_params.strat(); + let min_match = ms.c_params.min_match; + let dict_mode = ms.dict_mode(src.dms.as_ref()); + let iend = src.end; + let mut ip = src.start; + + // The optimal parser weighs the long distance matches against its own. + if strategy >= Strategy::BtOpt { + let block = BlockSrc { + ldm_seq_store: Some(&*raw_seq_store), + ..*src + }; + let last_ll_size = compress_block( + ms, + seq_store, + rep, + &block, + strategy, + use_row_match_finder, + dict_mode, + )?; + skip_raw_seq_store_bytes(raw_seq_store, iend.wrapping_sub(ip))?; + return Ok(last_ll_size); + } + + while raw_seq_store.pos < raw_seq_store.size && ip < iend { + let sequence = maybe_split_sequence(raw_seq_store, (iend - ip) as u32, min_match)?; + if sequence.offset == 0 { + break; + } + + limit_table_update(ms, src, ip); + fill_fast_tables(ms, src.prefix(), ip)?; + // The block compressor takes care of the literals, and what it leaves goes with the long distance match. + let literals = BlockSrc { + start: ip, + end: ip + sequence.lit_length as usize, + ..*src + }; + let new_lit_length = compress_block( + ms, + seq_store, + rep, + &literals, + strategy, + use_row_match_finder, + dict_mode, + )?; + ip += sequence.lit_length as usize; + let [rep0, rep1, _] = *rep; + *rep = [sequence.offset, rep0, rep1]; + seq_store.store_seq( + new_lit_length, + src.buf, + ip.checked_sub(new_lit_length).ok_or(BUG)?, + offset_to_offbase(sequence.offset), + sequence.match_length as usize, + )?; + ip += sequence.match_length as usize; + } + limit_table_update(ms, src, ip); + fill_fast_tables(ms, src.prefix(), ip)?; + let last = BlockSrc { start: ip, ..*src }; + compress_block( + ms, + seq_store, + rep, + &last, + strategy, + use_row_match_finder, + dict_mode, + ) +} diff --git a/src/zstd/wip/work/ldm/zstd/compress/mod.rs b/src/zstd/wip/work/ldm/zstd/compress/mod.rs new file mode 100644 index 000000000000..13ddecdf82e4 --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/compress/mod.rs @@ -0,0 +1,68 @@ +//! `lib/compress` + +pub(crate) mod double_fast; +pub(crate) mod fast; +pub(crate) mod fse; +pub(crate) mod hist; +pub(crate) mod huf; +pub(crate) mod internal; +pub(crate) mod lazy; +pub(crate) mod ldm; +pub(crate) mod opt; +pub(crate) mod presplit; + +use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; + +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; + +/// `ZSTD_rowMatchFinderSupported` +pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { + strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 +} + +/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to +/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies +/// that support it look at. Returns how many bytes at the end of the block are in no sequence. +pub(crate) fn compress_block( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row_match_finder: bool, + dict_mode: DictMode, +) -> Result { + match strategy { + Strategy::Fast => match dict_mode { + DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), + DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), + DictMode::DictMatchState => { + fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::DFast => match dict_mode { + DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), + DictMode::ExtDict => { + double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { + lazy::compress_block_lazy( + ms, + seq_store, + rep, + src, + strategy, + use_row_match_finder && row_match_finder_supported(strategy), + dict_mode, + ) + } + Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { + opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) + } + } +} diff --git a/src/zstd/wip/work/ldm/zstd/compress/opt.rs b/src/zstd/wip/work/ldm/zstd/compress/opt.rs new file mode 100644 index 000000000000..a7a6022a3a45 --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/compress/opt.rs @@ -0,0 +1,1721 @@ +//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. + +use core::mem; + +use super::hist; +use super::huf::HufRepeat; +use super::internal::{ + BlockSrc, DictMode, EntropyCTables, Match, MatchState, OptPrice, OptState, Optimal, + ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, ZSTD_OPT_NUM, count, + count_2segments, hash_ptr, hash3_ptr, index_overlap_check, ll_code, ml_code, new_rep, + offset_to_offbase, +}; +use crate::common::{ + LL_BITS, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, +}; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +/// How much a literal adds to its frequency, so that frequencies adapt fast to new statistics. +const ZSTD_LITFREQ_ADD: u32 = 2; +const ZSTD_MAX_PRICE: i32 = 1 << 30; +/// A first block of at most this size is priced by fixed rules, not by statistics. +const ZSTD_PREDEF_THRESHOLD: usize = 8; + +const MAX_LIT: u32 = 255; + +// ── prices ──────────────────────────────────────────────────────────────── + +/// Prices are in units of `1 / BITCOST_MULTIPLIER` of a bit. +const BITCOST_ACCURACY: u32 = 8; +const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY; + +/// `WEIGHT`: approximate for `btopt`, accurate for `btultra`. +#[inline(always)] +const fn weight(stat: u32, opt_level: i32) -> u32 { + if opt_level != 0 { + frac_weight(stat) + } else { + bit_weight(stat) + } +} + +/// `ZSTD_bitWeight`: the cost of a statistic, in whole bits. +#[inline(always)] +const fn bit_weight(stat: u32) -> u32 { + highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER +} + +/// `ZSTD_fracWeight`: the cost of a statistic, with a fraction of a bit that is interpolated linearly. +#[inline(always)] +const fn frac_weight(raw_stat: u32) -> u32 { + let stat = raw_stat.wrapping_add(1); + let hb = highbit32(stat); + let b_weight = hb * BITCOST_MULTIPLIER; + // From 1 to 2, in fixed point. + let f_weight = (stat << BITCOST_ACCURACY) >> hb; + b_weight.wrapping_add(f_weight) +} + +/// `ZSTD_compressedLiterals` +#[inline(always)] +fn compressed_literals(opt_ptr: &OptState) -> bool { + opt_ptr.literal_compression_mode != ParamSwitch::Disable +} + +/// `ZSTD_setBasePrices` +#[inline(always)] +fn set_base_prices(opt_ptr: &mut OptState, opt_level: i32) { + if compressed_literals(opt_ptr) { + opt_ptr.lit_sum_base_price = weight(opt_ptr.lit_sum, opt_level); + } + opt_ptr.lit_length_sum_base_price = weight(opt_ptr.lit_length_sum, opt_level); + opt_ptr.match_length_sum_base_price = weight(opt_ptr.match_length_sum, opt_level); + opt_ptr.off_code_sum_base_price = weight(opt_ptr.off_code_sum, opt_level); +} + +/// `sum_u32` +fn sum_u32(table: &[u32]) -> u32 { + table.iter().fold(0, |total, &elt| total.wrapping_add(elt)) +} + +/// `base_directive_e`: the least a scaled down statistic can be. +#[derive(Clone, Copy, PartialEq, Eq)] +enum BaseDirective { + /// 0, if it is 0 already. + Base0Possible, + Base1Guaranteed, +} + +/// `ZSTD_downscaleStats`: returns the new sum of the elements. +fn downscale_stats( + table: &mut [u32], + last_elt_index: u32, + shift: u32, + base1: BaseDirective, +) -> Result { + let mut sum = 0u32; + for elt in table.sub_mut(..=last_elt_index as usize)? { + let base = u32::from(base1 == BaseDirective::Base1Guaranteed || *elt > 0); + let new_stat = base.wrapping_add(*elt >> (shift & 31)); + sum = sum.wrapping_add(new_stat); + *elt = new_stat; + } + Ok(sum) +} + +/// `ZSTD_scaleStats`: scales all the elements down if their sum is too large. Returns the sum that results. +fn scale_stats(table: &mut [u32], last_elt_index: u32, log_target: u32) -> Result { + let prevsum = sum_u32(table.sub(..=last_elt_index as usize)?); + let factor = prevsum >> log_target; + if factor <= 1 { + return Ok(prevsum); + } + downscale_stats( + table, + last_elt_index, + highbit32(factor), + BaseDirective::Base1Guaranteed, + ) +} + +/// Frequencies, out of about `1 << scale_log`, that make each symbol cost the bits `bit_cost` tells. Returns their sum. +fn freqs_from_bit_costs( + freqs: &mut [u32], + last_elt_index: u32, + scale_log: u32, + bit_cost: impl Fn(u32) -> u32, +) -> Result { + let mut sum = 0u32; + for (symbol, freq) in (0u32..).zip(freqs.sub_mut(..=last_elt_index as usize)?) { + let bit_cost = bit_cost(symbol); + // At least 1, to have a cost. + *freq = if bit_cost != 0 { + 1 << (scale_log.wrapping_sub(bit_cost) & 31) + } else { + 1 + }; + sum = sum.wrapping_add(*freq); + } + Ok(sum) +} + +/// `ZSTD_rescaleFreqs`: for the first block, which no statistics tell, starts them from the tables of the dictionary +/// if there is one, else from the literals of `src` and from fixed values for the sequences. For the next ones, scales +/// down what the blocks before have accumulated. +fn rescale_freqs( + opt_ptr: &mut OptState, + symbol_costs: &EntropyCTables, + src: &[u8], + opt_level: i32, +) -> Result<()> { + #[rustfmt::skip] + const BASE_LL_FREQS: [u32; MAX_LL as usize + 1] = [ + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, + ]; + #[rustfmt::skip] + const BASE_OFC_FREQS: [u32; MAX_OFF as usize + 1] = [ + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + ]; + + let compressed_literals = compressed_literals(opt_ptr); + opt_ptr.price_type = OptPrice::Dynamic; + + if opt_ptr.lit_length_sum == 0 { + if src.len() <= ZSTD_PREDEF_THRESHOLD { + opt_ptr.price_type = OptPrice::Predef; + } + + if symbol_costs.huf.repeat_mode == HufRepeat::Valid { + // A Huffman table that has all the values is presumed to come from a dictionary. + opt_ptr.price_type = OptPrice::Dynamic; + + if compressed_literals { + let huf = &symbol_costs.huf.c_table; + opt_ptr.lit_sum = + freqs_from_bit_costs(&mut opt_ptr.lit_freq, MAX_LIT, 11, |lit| { + huf.get_nb_bits(lit) + })?; + } + let fse = &symbol_costs.fse; + opt_ptr.lit_length_sum = + freqs_from_bit_costs(&mut opt_ptr.lit_length_freq, MAX_LL, 10, |ll| { + fse.litlength_c_table.get_max_nb_bits(ll) + })?; + opt_ptr.match_length_sum = + freqs_from_bit_costs(&mut opt_ptr.match_length_freq, MAX_ML, 10, |ml| { + fse.matchlength_c_table.get_max_nb_bits(ml) + })?; + opt_ptr.off_code_sum = + freqs_from_bit_costs(&mut opt_ptr.off_code_freq, MAX_OFF, 10, |of| { + fse.offcode_c_table.get_max_nb_bits(of) + })?; + } else { + if compressed_literals { + let mut lit = MAX_LIT; + hist::count_simple(&mut opt_ptr.lit_freq, &mut lit, src)?; + opt_ptr.lit_sum = downscale_stats( + &mut opt_ptr.lit_freq, + MAX_LIT, + 8, + BaseDirective::Base0Possible, + )?; + } + + opt_ptr.lit_length_freq.write_at(0, &BASE_LL_FREQS)?; + opt_ptr.lit_length_sum = sum_u32(&BASE_LL_FREQS); + + opt_ptr + .match_length_freq + .sub_mut(..=MAX_ML as usize)? + .fill(1); + opt_ptr.match_length_sum = MAX_ML + 1; + + opt_ptr.off_code_freq.write_at(0, &BASE_OFC_FREQS)?; + opt_ptr.off_code_sum = sum_u32(&BASE_OFC_FREQS); + } + } else { + if compressed_literals { + opt_ptr.lit_sum = scale_stats(&mut opt_ptr.lit_freq, MAX_LIT, 12)?; + } + opt_ptr.lit_length_sum = scale_stats(&mut opt_ptr.lit_length_freq, MAX_LL, 11)?; + opt_ptr.match_length_sum = scale_stats(&mut opt_ptr.match_length_freq, MAX_ML, 11)?; + opt_ptr.off_code_sum = scale_stats(&mut opt_ptr.off_code_freq, MAX_OFF, 11)?; + } + + set_base_prices(opt_ptr, opt_level); + Ok(()) +} + +/// `ZSTD_rawLiteralsCost`: the price of `literals`, without that of their length. +#[inline(always)] +fn raw_literals_cost(literals: &[u8], opt_ptr: &OptState, opt_level: i32) -> Result { + let lit_length = literals.len() as u32; + if lit_length == 0 { + return Ok(0); + } + + if !compressed_literals(opt_ptr) { + // 8 bits per literal. + return Ok((lit_length << 3).wrapping_mul(BITCOST_MULTIPLIER)); + } + + if opt_ptr.price_type == OptPrice::Predef { + // 6 bits per literal. + return Ok(lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER)); + } + + let mut price = opt_ptr.lit_sum_base_price.wrapping_mul(lit_length); + let lit_price_max = opt_ptr.lit_sum_base_price.wrapping_sub(BITCOST_MULTIPLIER); + for &literal in literals { + let lit_price = weight(opt_ptr.lit_freq.at(usize::from(literal))?, opt_level); + price = price.wrapping_sub(lit_price.min(lit_price_max)); + } + Ok(price) +} + +/// `ZSTD_litLengthPrice`: the price of the symbol for a length of literals. +#[inline(always)] +fn lit_length_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + if opt_ptr.price_type == OptPrice::Predef { + return Ok(weight(lit_length, opt_level)); + } + + // The format has no code for the length of a whole block of the largest size, which is then all literals. It is + // made to cost 1 bit more than the length before it. + let (lit_length, extra) = if lit_length == ZSTD_BLOCKSIZE_MAX as u32 { + (ZSTD_BLOCKSIZE_MAX as u32 - 1, BITCOST_MULTIPLIER) + } else { + (lit_length, 0) + }; + + let ll_code = ll_code(lit_length) as usize; + Ok(extra + .wrapping_add(u32::from(LL_BITS.at(ll_code)?) * BITCOST_MULTIPLIER) + .wrapping_add(opt_ptr.lit_length_sum_base_price) + .wrapping_sub(weight(opt_ptr.lit_length_freq.at(ll_code)?, opt_level))) +} + +/// `ZSTD_getMatchPrice`: the price of the offset and of the match length of a sequence. `off_base` is as in +/// [`SeqStore::store_seq`]. Below level 2, it favours small offsets, which are faster to decompress. +#[inline(always)] +fn get_match_price( + off_base: u32, + match_length: u32, + opt_ptr: &OptState, + opt_level: i32, +) -> Result { + let off_code = highbit32(off_base); + let ml_base = match_length.wrapping_sub(MINMATCH); + + if opt_ptr.price_type == OptPrice::Predef { + return Ok(weight(ml_base, opt_level).wrapping_add((16 + off_code) * BITCOST_MULTIPLIER)); + } + + let off_code_weight = weight(opt_ptr.off_code_freq.at(off_code as usize)?, opt_level); + let mut price = (off_code * BITCOST_MULTIPLIER).wrapping_add( + opt_ptr + .off_code_sum_base_price + .wrapping_sub(off_code_weight), + ); + if opt_level < 2 && off_code >= 20 { + price = price.wrapping_add((off_code - 19) * 2 * BITCOST_MULTIPLIER); + } + + let ml_code = ml_code(ml_base) as usize; + let ml_code_weight = weight(opt_ptr.match_length_freq.at(ml_code)?, opt_level); + price = price + .wrapping_add(u32::from(ML_BITS.at(ml_code)?) * BITCOST_MULTIPLIER) + .wrapping_add( + opt_ptr + .match_length_sum_base_price + .wrapping_sub(ml_code_weight), + ); + + // Matches cost a bit more, to have fewer sequences, which is faster to decompress. + Ok(price.wrapping_add(BITCOST_MULTIPLIER / 5)) +} + +/// `ZSTD_updateStats` +#[inline(always)] +fn update_stats( + opt_ptr: &mut OptState, + literals: &[u8], + off_base: u32, + match_length: u32, +) -> Result<()> { + #[inline(always)] + fn increment(freqs: &mut [u32], code: u32, sum: &mut u32) -> Result<()> { + let freq = freqs.at_mut(code as usize)?; + *freq = freq.wrapping_add(1); + *sum = sum.wrapping_add(1); + Ok(()) + } + + let lit_length = literals.len() as u32; + if compressed_literals(opt_ptr) { + for &literal in literals { + let freq = opt_ptr.lit_freq.at_mut(usize::from(literal))?; + *freq = freq.wrapping_add(ZSTD_LITFREQ_ADD); + } + opt_ptr.lit_sum = opt_ptr + .lit_sum + .wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD)); + } + + increment( + &mut opt_ptr.lit_length_freq, + ll_code(lit_length), + &mut opt_ptr.lit_length_sum, + )?; + increment( + &mut opt_ptr.off_code_freq, + highbit32(off_base), + &mut opt_ptr.off_code_sum, + )?; + increment( + &mut opt_ptr.match_length_freq, + ml_code(match_length.wrapping_sub(MINMATCH)), + &mut opt_ptr.match_length_sum, + ) +} + +/// `ZSTD_readMINMATCH`: `length` bytes, 3 or 4, to compare with others. Reads 4. +#[inline(always)] +fn read_minmatch(buf: &[u8], pos: usize, length: u32) -> Result { + let value = read_le32(buf, pos)?; + Ok(if length == 3 { value << 8 } else { value }) +} + +/// `ZSTD_insertAndFindFirstIndexHash3`: indexes the 3-byte hashes up to `ip`, which is not included, and looks it up. +/// All of it is in the current segment. +fn insert_and_find_first_index_hash3( + ms: &mut MatchState, + next_to_update3: &mut u32, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + let hash_table3 = ms.hash_table3.as_mut_slice(); + let hash_log3 = ms.hash_log3; + let target = prefix.index_of(ip); + let hash3 = hash3_ptr(prefix.buf, ip, hash_log3)?; + + for idx in *next_to_update3..target { + hash_table3.put(hash3_ptr(prefix.buf, prefix.pos_of(idx), hash_log3)?, idx)?; + } + + *next_to_update3 = target; + hash_table3.at(hash3) +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `*ptr = value`, where `ptr` is an element of `bt`, or `None` for a place that nothing reads. +#[inline(always)] +fn store(bt: &mut [u32], ptr: Option, value: u32) -> Result<()> { + match ptr { + Some(ptr) => bt.put(ptr, value), + None => Ok(()), + } +} + +/// `ZSTD_insertBt1`: adds the position `ip`, which is at most `iend - 8`, to the tree. `target` is up to where the +/// caller is filling it, and `dict` the older segment, which is looked at if `ext_dict`. Returns how many positions +/// to move forward. +#[inline(always)] +fn insert_bt1( + ms: &mut MatchState, + prefix: Prefix<'_>, + dict: Prefix<'_>, + ip: usize, + iend: usize, + target: u32, + mls: u32, + ext_dict: bool, +) -> Result { + let c_params = ms.c_params; + let h = hash_ptr(prefix.buf, ip, c_params.hash_log, mls)?; + let bt_log = c_params.chain_log.wrapping_sub(1); + let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = dict.pos_of(dict_limit); + let prefix_start = prefix.pos_of(dict_limit); + let curr = prefix.index_of(ip); + let bt_low = curr.saturating_sub(bt_mask); + let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); + let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); + // Only the positions that are still in the window once the tree is filled are of use. + let window_low = ms.get_lowest_match_index(target, c_params.window_log); + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut best_length = 8usize; + let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); + + let bt = ms.chain_table.as_mut_slice(); + let hash_entry = ms.hash_table.at_mut(h)?; + let mut match_index = *hash_entry; + *hash_entry = curr; + + while nb_compares != 0 && match_index >= window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + // What is sure to be common. + let mut match_length = common_length_smaller.min(common_length_larger); + + // A position of the older segment, as one of the current segment, only makes sense with a length that + // brings it there. + let mut match_buf = prefix.buf; + let mut match_pos = prefix.pos_of(match_index); + if !ext_dict || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + prefix.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + prefix.buf, + ip + match_length, + dict.buf, + dict.pos_of(match_index) + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + match_buf = dict.buf; + match_pos = dict.pos_of(match_index); + } + } + + if match_length > best_length { + best_length = match_length; + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + } + + if ip + match_length == iend { + // They are equal: there is no telling which is smaller. Dropping it keeps the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < prefix.buf.at(ip + match_length)? { + // All the smaller ones will have at least this much in common. + store(bt, smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. + smaller_ptr = None; + break; + } + smaller_ptr = Some(next_ptr + 1); + match_index = bt.at(next_ptr + 1)?; + } else { + store(bt, larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = None; + break; + } + larger_ptr = Some(next_ptr); + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + store(bt, smaller_ptr, 0)?; + store(bt, larger_ptr, 0)?; + + let positions = if best_length > 384 { + 192.min((best_length - 384) as u32) + } else { + 0 + }; + Ok(positions.max(match_end_idx.wrapping_sub(curr.wrapping_add(8)))) +} + +/// `ZSTD_updateTree_internal` +#[inline(always)] +fn update_tree_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + dict: Prefix<'_>, + ip: usize, + iend: usize, + mls: u32, + dict_mode: DictMode, +) -> Result<()> { + let target = prefix.index_of(ip); + let mut idx = ms.next_to_update; + + while idx < target { + let forward = insert_bt1( + ms, + prefix, + dict, + prefix.pos_of(idx), + iend, + target, + mls, + dict_mode == DictMode::ExtDict, + )?; + idx = idx.wrapping_add(forward); + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. +/// Matches are not compared past the position `iend`. +pub(crate) fn update_tree( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + iend: usize, +) -> Result<()> { + let no_dict = Prefix { buf: &[], base: 0 }; + let mls = ms.c_params.min_match; + update_tree_internal(ms, prefix, no_dict, ip, iend, mls, DictMode::NoDict) +} + +/// `ZSTD_insertBtAndGetAllMatches`: adds the position `ip` to the tree, and stores in `matches` the matches found on +/// the way, each longer than the one before and all at least `length_to_beat` long. `ll0` is whether no literals +/// come before `ip`. Returns how many there are. +#[inline(always)] +fn insert_bt_and_get_all_matches( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + dict_mode: DictMode, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, + mls: u32, +) -> Result { + let buf = src.buf; + let c_params = ms.c_params; + let sufficient_len = c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); + let curr = src.index_of(ip); + let min_match = if mls == 3 { 3 } else { 4 }; + let h = hash_ptr(buf, ip, c_params.hash_log, mls)?; + let bt_log = c_params.chain_log.wrapping_sub(1); + let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt_low = curr.saturating_sub(bt_mask); + let window_low = ms.get_lowest_match_index(curr, c_params.window_log); + let match_low = window_low.max(1); + let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); + let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); + // The farthest position any match refers to, which detects repetitive patterns. + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut mnum = 0u32; + let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); + + let dms = if dict_mode == DictMode::DictMatchState { + Some(src.dms.ok_or(BUG)?) + } else { + None + }; + let dms_high_limit = dms.map_or(0, |dms| dms.end.wrapping_sub(dms.base) as u32); + let dms_low_limit = dms.map_or(0, |dms| dms.ms.window.low_limit); + let dms_index_delta = dms.map_or(0, |_| window_low.wrapping_sub(dms_high_limit)); + + let mut best_length = length_to_beat.wrapping_sub(1) as usize; + + // The repeat offsets. + let ll0 = u32::from(ll0); + let [rep0, _, _] = *rep; + for rep_code in ll0..ZSTD_REP_NUM as u32 + ll0 { + let rep_offset = match rep.get(rep_code as usize) { + Some(&offset) => offset, + None => rep0.wrapping_sub(1), + }; + let rep_index = curr.wrapping_sub(rep_offset); + let mut rep_len = 0u32; + // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. + if rep_offset.wrapping_sub(1) < curr.wrapping_sub(dict_limit) { + // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. + let rep_match = ip.wrapping_sub(rep_offset as usize); + if rep_index >= window_low + && read_minmatch(buf, ip, min_match)? == read_minmatch(buf, rep_match, min_match)? + { + rep_len = count( + buf, + ip + min_match as usize, + buf, + rep_match + min_match as usize, + i_limit, + )? as u32 + + min_match; + } + } else if dict_mode == DictMode::ExtDict { + // `curr > rep_index >= window_low` + if rep_offset.wrapping_sub(1) < curr.wrapping_sub(window_low) + && index_overlap_check(dict_limit, rep_index) + { + let rep_match = src.dict_pos_of(rep_index); + if read_minmatch(buf, ip, min_match)? + == read_minmatch(src.dict_buf, rep_match, min_match)? + { + rep_len = count_2segments( + buf, + ip + min_match as usize, + src.dict_buf, + rep_match + min_match as usize, + i_limit, + dict_end, + prefix_start, + )? as u32 + + min_match; + } + } + } else if let Some(dms) = dms { + // `curr > rep_index >= dms_low_limit` + if rep_offset.wrapping_sub(1) + < curr.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta)) + && index_overlap_check(dict_limit, rep_index) + { + let rep_match = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); + if read_minmatch(buf, ip, min_match)? + == read_minmatch(dms.buf, rep_match, min_match)? + { + rep_len = count_2segments( + buf, + ip + min_match as usize, + dms.buf, + rep_match + min_match as usize, + i_limit, + dms.end, + prefix_start, + )? as u32 + + min_match; + } + } + } + if rep_len as usize > best_length { + best_length = rep_len as usize; + matches.put( + mnum as usize, + Match { + off: rep_code - ll0 + 1, + len: rep_len, + }, + )?; + mnum += 1; + if rep_len > sufficient_len || ip + rep_len as usize == i_limit { + // The best there can be. + return Ok(mnum); + } + } + } + + // The table of 3-byte hashes. That of a dictionary is not filled. + if mls == 3 && best_length < mls as usize { + let match_index3 = + insert_and_find_first_index_hash3(ms, next_to_update3, src.prefix(), ip)?; + // Farther than that is likely to cost too much. + if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) { + let mlen = if dict_mode != DictMode::ExtDict || match_index3 >= dict_limit { + count(buf, ip, buf, src.pos_of(match_index3), i_limit)? + } else { + count_2segments( + buf, + ip, + src.dict_buf, + src.dict_pos_of(match_index3), + i_limit, + dict_end, + prefix_start, + )? + }; + + if mlen >= mls as usize { + best_length = mlen; + matches.put( + 0, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index3)), + len: mlen as u32, + }, + )?; + mnum = 1; + if mlen > sufficient_len as usize || ip + mlen == i_limit { + // The best there can be: `ip` is not inserted. + ms.next_to_update = curr.wrapping_add(1); + return Ok(1); + } + } + } + } + + let bt = ms.chain_table.as_mut_slice(); + let hash_entry = ms.hash_table.at_mut(h)?; + let mut match_index = *hash_entry; + *hash_entry = curr; + + while nb_compares != 0 && match_index >= match_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + // What is sure to be common. + let mut match_length = common_length_smaller.min(common_length_larger); + + // A position of the older segment, as one of the current segment, only makes sense with a length that + // brings it there. + let mut match_buf = buf; + let mut match_pos = src.pos_of(match_index); + if dict_mode != DictMode::ExtDict + || match_index as usize + match_length >= dict_limit as usize + { + match_length += count( + buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + i_limit, + )?; + } else { + match_length += count_2segments( + buf, + ip + match_length, + src.dict_buf, + src.dict_pos_of(match_index) + match_length, + i_limit, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + match_buf = src.dict_buf; + match_pos = src.dict_pos_of(match_index); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + best_length = match_length; + matches.put( + mnum as usize, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index)), + len: match_length as u32, + }, + )?; + mnum += 1; + if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { + // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. + if dict_mode == DictMode::DictMatchState { + // The dictionary is not to be searched either. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < buf.at(ip + match_length)? { + // All the smaller ones will have at least this much in common. + store(bt, smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. + smaller_ptr = None; + break; + } + smaller_ptr = Some(next_ptr + 1); + match_index = bt.at(next_ptr + 1)?; + } else { + store(bt, larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = None; + break; + } + larger_ptr = Some(next_ptr); + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + store(bt, smaller_ptr, 0)?; + store(bt, larger_ptr, 0)?; + + if let Some(dms) = dms + && nb_compares != 0 + { + let dms_c_params = &dms.ms.c_params; + let dms_bt_log = dms_c_params.chain_log.wrapping_sub(1); + let dms_bt_mask = 1u32.wrapping_shl(dms_bt_log).wrapping_sub(1); + let dms_bt_low = if dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit) { + dms_high_limit.wrapping_sub(dms_bt_mask) + } else { + dms_low_limit + }; + let dms_h = hash_ptr(buf, ip, dms_c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(dms_h)?; + let dms_bt = dms.ms.chain_table.as_slice(); + common_length_smaller = 0; + common_length_larger = 0; + while nb_compares != 0 && dict_match_index > dms_low_limit { + let next_ptr = 2 * (dict_match_index & dms_bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + match_length += count_2segments( + buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + i_limit, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + match_index = dict_match_index.wrapping_add(dms_index_delta); + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + best_length = match_length; + matches.put( + mnum as usize, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index)), + len: match_length as u32, + }, + )?; + mnum += 1; + if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { + break; + } + } + + if dict_match_index <= dms_bt_low { + // Beyond the tree. + break; + } + // A match that reaches the end of the dictionary goes on at the start of the current segment. + let match_byte = if dict_match_index as usize + match_length >= dms_high_limit as usize + { + let match_pos = src.pos_of(dict_match_index.wrapping_add(dms_index_delta)); + buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dms_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dms_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + } + + // Repetitive patterns are skipped. + ms.next_to_update = match_end_idx.wrapping_sub(8); + Ok(mnum) +} + +/// `ZSTD_getAllMatchesFn`: the arguments are `matches`, `ms`, `next_to_update3`, `src`, `ip`, `i_high_limit`, `rep`, +/// `ll0` and `length_to_beat`. +type GetAllMatchesFn = fn( + &mut [Match], + &mut MatchState, + &mut u32, + &BlockSrc<'_>, + usize, + usize, + &[u32; ZSTD_REP_NUM], + bool, + u32, +) -> Result; + +/// `ZSTD_btGetAllMatches_internal` +#[inline(always)] +fn bt_get_all_matches_internal( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, + dict_mode: DictMode, + mls: u32, +) -> Result { + if src.index_of(ip) < ms.next_to_update { + // An area that is skipped. + return Ok(0); + } + let dict = Prefix { + buf: src.dict_buf, + base: src.dict_base, + }; + update_tree_internal(ms, src.prefix(), dict, ip, i_high_limit, mls, dict_mode)?; + insert_bt_and_get_all_matches( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + dict_mode, + rep, + ll0, + length_to_beat, + mls, + ) +} + +/// `ZSTD_btGetAllMatches_noDict_3` to `ZSTD_btGetAllMatches_noDict_6` +fn bt_get_all_matches_no_dict( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::NoDict, + MLS, + ) +} + +/// `ZSTD_btGetAllMatches_extDict_3` to `ZSTD_btGetAllMatches_extDict_6` +fn bt_get_all_matches_ext_dict( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::ExtDict, + MLS, + ) +} + +/// `ZSTD_btGetAllMatches_dictMatchState_3` to `ZSTD_btGetAllMatches_dictMatchState_6` +fn bt_get_all_matches_dict_match_state( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::DictMatchState, + MLS, + ) +} + +/// `ZSTD_selectBtGetAllMatches` +fn select_bt_get_all_matches(ms: &MatchState, dict_mode: DictMode) -> GetAllMatchesFn { + match (dict_mode, ms.c_params.min_match) { + (DictMode::NoDict, ..=3) => bt_get_all_matches_no_dict::<3>, + (DictMode::NoDict, 4) => bt_get_all_matches_no_dict::<4>, + (DictMode::NoDict, 5) => bt_get_all_matches_no_dict::<5>, + (DictMode::NoDict, 6..) => bt_get_all_matches_no_dict::<6>, + (DictMode::ExtDict, ..=3) => bt_get_all_matches_ext_dict::<3>, + (DictMode::ExtDict, 4) => bt_get_all_matches_ext_dict::<4>, + (DictMode::ExtDict, 5) => bt_get_all_matches_ext_dict::<5>, + (DictMode::ExtDict, 6..) => bt_get_all_matches_ext_dict::<6>, + (DictMode::DictMatchState, ..=3) => bt_get_all_matches_dict_match_state::<3>, + (DictMode::DictMatchState, 4) => bt_get_all_matches_dict_match_state::<4>, + (DictMode::DictMatchState, 5) => bt_get_all_matches_dict_match_state::<5>, + (DictMode::DictMatchState, 6..) => bt_get_all_matches_dict_match_state::<6>, + } +} + +// ── candidates from long distance matching ──────────────────────────────── + +/// `ZSTD_optLdm_t`: what it takes to decide whether to use a long distance match. +struct OptLdm<'a> { + /// `seqStore.seq`: the `seqStore.size` candidates for this block and the next ones. + seq: &'a [RawSeq], + /// `seqStore.pos` + pos: usize, + /// `seqStore.posInSequence` + pos_in_sequence: usize, + /// Where the current candidate starts. + start_pos_in_block: u32, + /// Where it ends. + end_pos_in_block: u32, + offset: u32, +} + +impl<'a> OptLdm<'a> { + fn new(seq_store: Option<&'a RawSeqStore>) -> Self { + Self { + seq: seq_store + .and_then(|store| store.seq.get(..store.size)) + .unwrap_or_default(), + pos: seq_store.map_or(0, |store| store.pos), + pos_in_sequence: seq_store.map_or(0, |store| store.pos_in_sequence), + start_pos_in_block: 0, + end_pos_in_block: 0, + offset: 0, + } + } + + /// `ZSTD_optLdm_skipRawSeqStoreBytes`: moves forward by `nb_bytes`. + fn skip_raw_seq_store_bytes(&mut self, nb_bytes: usize) { + let mut curr_pos = self.pos_in_sequence.wrapping_add(nb_bytes) as u32; + while curr_pos != 0 { + let Some(curr_seq) = self.seq.get(self.pos) else { + break; + }; + let seq_length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); + if curr_pos >= seq_length { + curr_pos -= seq_length; + self.pos += 1; + } else { + self.pos_in_sequence = curr_pos as usize; + break; + } + } + if curr_pos == 0 || self.pos == self.seq.len() { + self.pos_in_sequence = 0; + } + } + + /// `ZSTD_opt_getNextMatchAndUpdateSeqStore`: finds where the next candidate starts and ends in the block, and + /// moves past it. + fn get_next_match_and_update_seq_store( + &mut self, + curr_pos_in_block: u32, + block_bytes_remaining: u32, + ) { + let Some(&curr_seq) = self.seq.get(self.pos) else { + // No position gets there, so that there is no candidate in this block. + self.start_pos_in_block = u32::MAX; + self.end_pos_in_block = u32::MAX; + return; + }; + // What is left of the literals and of the match. + let pos_in_sequence = self.pos_in_sequence as u32; + let curr_block_end_pos = curr_pos_in_block.wrapping_add(block_bytes_remaining); + let literals_bytes_remaining = curr_seq.lit_length.saturating_sub(pos_in_sequence); + let match_bytes_remaining = if literals_bytes_remaining == 0 { + curr_seq + .match_length + .wrapping_sub(pos_in_sequence.wrapping_sub(curr_seq.lit_length)) + } else { + curr_seq.match_length + }; + + if literals_bytes_remaining >= block_bytes_remaining { + // The match starts after the block. + self.start_pos_in_block = u32::MAX; + self.end_pos_in_block = u32::MAX; + self.skip_raw_seq_store_bytes(block_bytes_remaining as usize); + return; + } + + // What is left of a match can be too short to use, which is checked later. + self.start_pos_in_block = curr_pos_in_block.wrapping_add(literals_bytes_remaining); + self.end_pos_in_block = self.start_pos_in_block.wrapping_add(match_bytes_remaining); + self.offset = curr_seq.offset; + + if self.end_pos_in_block > curr_block_end_pos { + // The match ends after the block, which can only use its start. + self.end_pos_in_block = curr_block_end_pos; + self.skip_raw_seq_store_bytes( + curr_block_end_pos.wrapping_sub(curr_pos_in_block) as usize + ); + } else { + self.skip_raw_seq_store_bytes( + literals_bytes_remaining.wrapping_add(match_bytes_remaining) as usize, + ); + } + } + + /// `ZSTD_optLdm_maybeAddMatch`: adds what is left of the candidate at `curr_pos_in_block` to `matches`, if it is + /// long enough and longer than all of them, since they are in order of length. + fn maybe_add_match( + &self, + matches: &mut [Match], + nb_matches: &mut u32, + curr_pos_in_block: u32, + min_match: u32, + ) -> Result<()> { + let pos_diff = curr_pos_in_block.wrapping_sub(self.start_pos_in_block); + let candidate_match_length = self + .end_pos_in_block + .wrapping_sub(self.start_pos_in_block) + .wrapping_sub(pos_diff); + + if curr_pos_in_block < self.start_pos_in_block + || curr_pos_in_block >= self.end_pos_in_block + || candidate_match_length < min_match + { + return Ok(()); + } + + if *nb_matches == 0 + || (candidate_match_length > matches.at(*nb_matches as usize - 1)?.len + && (*nb_matches as usize) < ZSTD_OPT_NUM) + { + matches.put( + *nb_matches as usize, + Match { + off: offset_to_offbase(self.offset), + len: candidate_match_length, + }, + )?; + *nb_matches += 1; + } + Ok(()) + } + + /// `ZSTD_optLdm_processMatchCandidate` + #[inline(always)] + fn process_match_candidate( + &mut self, + matches: &mut [Match], + nb_matches: &mut u32, + curr_pos_in_block: u32, + remaining_bytes: u32, + min_match: u32, + ) -> Result<()> { + if self.pos >= self.seq.len() { + return Ok(()); + } + + if curr_pos_in_block >= self.end_pos_in_block { + if curr_pos_in_block > self.end_pos_in_block { + // The parser does not stop at the end of each candidate: it is often some bytes past it. + let pos_overshoot = curr_pos_in_block - self.end_pos_in_block; + self.skip_raw_seq_store_bytes(pos_overshoot as usize); + } + self.get_next_match_and_update_seq_store(curr_pos_in_block, remaining_bytes); + } + self.maybe_add_match(matches, nb_matches, curr_pos_in_block, min_match) + } +} + +// ── the optimal parser ──────────────────────────────────────────────────── + +/// `LIT_PRICE`: the price of the literal at `pos`. +#[inline(always)] +fn lit_price(buf: &[u8], pos: usize, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok(raw_literals_cost(buf.span(pos, 1)?, opt_ptr, opt_level)? as i32) +} + +/// `LL_PRICE` +#[inline(always)] +fn ll_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok(lit_length_price(lit_length, opt_ptr, opt_level)? as i32) +} + +/// `LL_INCPRICE`: what one more literal, which makes `lit_length` of them, adds to the price of their length. +#[inline(always)] +fn ll_inc_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok( + ll_price(lit_length, opt_ptr, opt_level)?.wrapping_sub(ll_price( + lit_length.wrapping_sub(1), + opt_ptr, + opt_level, + )?), + ) +} + +/// `ZSTD_compressBlock_opt_generic` +#[inline(always)] +fn compress_block_opt_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + opt_level: i32, + dict_mode: DictMode, +) -> Result { + // The match finders borrow the whole of `ms`. + let mut price_table = mem::take(&mut ms.opt.price_table); + let mut match_table = mem::take(&mut ms.opt.match_table); + let result = parse_block( + ms, + seq_store, + rep, + src, + &mut price_table, + &mut match_table, + opt_level, + dict_mode, + ); + ms.opt.price_table = price_table; + ms.opt.match_table = match_table; + result +} + +/// The body of `ZSTD_compressBlock_opt_generic`. `opt` is `ms.opt.price_table` and `matches` is `ms.opt.match_table`. +#[inline(always)] +fn parse_block( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + opt: &mut [Optimal], + matches: &mut [Match], + opt_level: i32, + dict_mode: DictMode, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let ilimit = iend.saturating_sub(8); + let prefix_start = src.pos_of(ms.window.dict_limit); + + let get_all_matches = select_bt_get_all_matches(ms, dict_mode); + + let sufficient_len = ms.c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); + let min_match = if ms.c_params.min_match == 3 { 3 } else { 4 }; + let mut next_to_update3 = ms.next_to_update; + + let mut last_stretch = Optimal::default(); + let mut opt_ldm = OptLdm::new(src.ldm_seq_store); + opt_ldm.get_next_match_and_update_seq_store(0, (iend - ip) as u32); + + rescale_freqs( + &mut ms.opt, + src.symbol_costs, + buf.sub(istart..iend)?, + opt_level, + )?; + ip += usize::from(ip == prefix_start); + + 'series: while ip < ilimit { + let mut last_pos: u32; + + // The forward pass. `opt` holds stretches, each a match followed by literals, unlike a sequence, which is + // literals followed by a match: that way each position of a run of literals can have another match before it. + // It ends with the last stretch in `last_stretch`, which starts at `cur` and ends at `last_pos`. + let mut cur = 'forward: { + // Find a first match. + { + let litlen = (ip - anchor) as u32; + let mut nb_matches = get_all_matches( + matches, + ms, + &mut next_to_update3, + src, + ip, + iend, + rep, + litlen == 0, + min_match, + )?; + opt_ldm.process_match_candidate( + matches, + &mut nb_matches, + (ip - istart) as u32, + (iend - ip) as u32, + min_match, + )?; + if nb_matches == 0 { + ip += 1; + continue 'series; + } + let matches = matches.sub(..nb_matches as usize)?; + + // The price of the literals before the first match is the same whatever comes next, so it is left + // out. But how the price of their length changes depends on their length. + let first_price = ll_price(litlen, &ms.opt, opt_level)?; + { + let first = opt.at_mut(0)?; + first.mlen = 0; + first.litlen = litlen; + first.price = first_price; + first.rep = *rep; + } + + // A large match is encoded at once. + let longest = matches.at(matches.len() - 1)?; + if longest.len > sufficient_len { + last_stretch.litlen = 0; + last_stretch.mlen = longest.len; + last_stretch.off = longest.off; + last_pos = longest.len; + break 'forward 0; + } + + // Set the prices of the first matches. + let ll0_price = ll_price(0, &ms.opt, opt_level)?; + let mut pos = 1u32; + while pos < min_match { + let stretch = opt.at_mut(pos as usize)?; + stretch.price = ZSTD_MAX_PRICE; + stretch.mlen = 0; + stretch.litlen = litlen.wrapping_add(pos); + pos += 1; + } + for candidate in matches { + while pos <= candidate.len { + let match_price = + get_match_price(candidate.off, pos, &ms.opt, opt_level)? as i32; + let sequence_price = first_price.wrapping_add(match_price); + let stretch = opt.at_mut(pos as usize)?; + stretch.mlen = pos; + stretch.off = candidate.off; + stretch.litlen = 0; + stretch.price = sequence_price.wrapping_add(ll0_price); + pos += 1; + } + } + last_pos = pos - 1; + opt.at_mut(pos as usize)?.price = ZSTD_MAX_PRICE; + } + + // Check the positions that follow. + let mut cur = 0u32; + while cur < last_pos { + cur += 1; + let inr = ip + cur as usize; + + // Get to this position by a literal, if it is cheaper. + { + let before = opt.at(cur as usize - 1)?; + let litlen = before.litlen.wrapping_add(1); + let price = before + .price + .wrapping_add(lit_price(buf, inr - 1, &ms.opt, opt_level)?) + .wrapping_add(ll_inc_price(litlen, &ms.opt, opt_level)?); + let prev_match = opt.at(cur as usize)?; + if price <= prev_match.price { + opt.put( + cur as usize, + Optimal { + litlen, + price, + ..before + }, + )?; + // If this replaces a match, and 1 literal is cheaper than none, the match followed by a + // literal may be a cheaper way to the next position. + if opt_level >= 1 + && prev_match.litlen == 0 + && ll_inc_price(1, &ms.opt, opt_level)? < 0 + && inr < iend + { + let next_lit_price = lit_price(buf, inr, &ms.opt, opt_level)?; + let with_1_literal = prev_match + .price + .wrapping_add(next_lit_price) + .wrapping_add(ll_inc_price(1, &ms.opt, opt_level)?); + let with_more_literals = price + .wrapping_add(next_lit_price) + .wrapping_add(ll_inc_price( + litlen.wrapping_add(1), + &ms.opt, + opt_level, + )?); + if with_1_literal < with_more_literals + && with_1_literal < opt.at(cur as usize + 1)?.price + { + let prev = opt.at(cur.wrapping_sub(prev_match.mlen) as usize)?; + opt.put( + cur as usize + 1, + Optimal { + rep: new_rep(&prev.rep, prev_match.off, prev.litlen == 0), + litlen: 1, + price: with_1_literal, + ..prev_match + }, + )?; + last_pos = last_pos.max(cur + 1); + } + } + } + } + + // The repeat offsets are not updated while matches are compared, but here, now that one is selected. + let mut here = opt.at(cur as usize)?; + if here.litlen == 0 { + let prev = opt.at(cur.wrapping_sub(here.mlen) as usize)?; + here.rep = new_rep(&prev.rep, here.off, prev.litlen == 0); + opt.at_mut(cur as usize)?.rep = here.rep; + } + + // The last match starts at least 8 bytes before the end. + if inr > ilimit { + continue; + } + + if cur == last_pos { + break; + } + + if opt_level == 0 + && opt.at(cur as usize + 1)?.price + <= here.price.wrapping_add(BITCOST_MULTIPLIER as i32 / 2) + { + // The next position is cheaper to get to: this one is not promising. + continue; + } + + { + let base_price = here.price.wrapping_add(ll_price(0, &ms.opt, opt_level)?); + let mut nb_matches = get_all_matches( + matches, + ms, + &mut next_to_update3, + src, + inr, + iend, + &here.rep, + here.litlen == 0, + min_match, + )?; + opt_ldm.process_match_candidate( + matches, + &mut nb_matches, + (inr - istart) as u32, + (iend - inr) as u32, + min_match, + )?; + if nb_matches == 0 { + continue; + } + let matches = matches.sub(..nb_matches as usize)?; + + let longest = matches.at(matches.len() - 1)?; + if longest.len > sufficient_len + || cur.wrapping_add(longest.len) as usize >= ZSTD_OPT_NUM + || inr + longest.len as usize >= iend + { + last_stretch.mlen = longest.len; + last_stretch.off = longest.off; + last_stretch.litlen = 0; + last_pos = cur.wrapping_add(longest.len); + break 'forward cur; + } + + // Set the prices that the matches found here give. + let mut start_ml = min_match; + for candidate in matches { + let mut mlen = candidate.len; + while mlen >= start_ml { + let pos = cur + mlen; + let price = base_price.wrapping_add(get_match_price( + candidate.off, + mlen, + &ms.opt, + opt_level, + )? + as i32); + + if pos > last_pos || price < opt.at(pos as usize)?.price { + while last_pos < pos { + // Positions that nothing leads to yet. + last_pos += 1; + let stretch = opt.at_mut(last_pos as usize)?; + stretch.price = ZSTD_MAX_PRICE; + // Not 0, which would be the end of a match. + stretch.litlen = 1; + } + let stretch = opt.at_mut(pos as usize)?; + stretch.mlen = mlen; + stretch.off = candidate.off; + stretch.litlen = 0; + stretch.price = price; + } else if opt_level == 0 { + // Shorter lengths are not tried. + break; + } + mlen -= 1; + } + start_ml = candidate.len + 1; + } + } + opt.at_mut(last_pos as usize + 1)?.price = ZSTD_MAX_PRICE; + } + + last_stretch = opt.at(last_pos as usize)?; + last_pos.wrapping_sub(last_stretch.mlen) + }; + + if last_stretch.mlen == 0 { + // All the matches have been turned into literals. + ip += last_pos as usize; + continue; + } + + if last_stretch.litlen == 0 { + // It ends with a match. + let prev = opt.at(cur as usize)?; + *rep = new_rep(&prev.rep, last_stretch.off, prev.litlen == 0); + } else { + *rep = last_stretch.rep; + cur = cur.wrapping_sub(last_stretch.litlen); + } + + // Write the cheapest path in `opt`, backwards from `store_end`, over the stretches that are not on it, and as + // sequences. The literals that end the last stretch are in no sequence: they are parsed again. (The C code + // starts by making them a last sequence without a match, and overwrites it at once.) + let store_end = cur.wrapping_add(2) as usize; + let mut store_start = store_end; + let mut stretch_pos = cur; + opt.put(store_end, last_stretch)?; + loop { + let next_stretch = opt.at(stretch_pos as usize)?; + opt.at_mut(store_start)?.litlen = next_stretch.litlen; + if next_stretch.mlen == 0 { + // The start of the series. + break; + } + store_start = store_start.wrapping_sub(1); + opt.put(store_start, next_stretch)?; + stretch_pos = + stretch_pos.wrapping_sub(next_stretch.litlen.wrapping_add(next_stretch.mlen)); + } + + for sequence in opt.sub(store_start..=store_end)? { + let llen = sequence.litlen as usize; + let mlen = sequence.mlen as usize; + update_stats( + &mut ms.opt, + buf.span(anchor, llen)?, + sequence.off, + sequence.mlen, + )?; + seq_store.store_seq(llen, buf, anchor, sequence.off, mlen)?; + anchor += llen + mlen; + ip = anchor; + } + + set_base_prices(&mut ms.opt, opt_level); + } + + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_opt0` +fn compress_block_opt0( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode) +} + +/// `ZSTD_compressBlock_opt2` +fn compress_block_opt2( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode) +} + +/// `ZSTD_initStats_ultra`: a first pass over the block, only to have better statistics to start from. It has to be the +/// first block, with no dictionary and no long distance matching. Returns the block as the window then sees it, since +/// its indices change. +fn init_stats_ultra<'a>( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &[u32; ZSTD_REP_NUM], + src: &BlockSrc<'a>, +) -> Result> { + // The repeat offsets it leaves are of no use. + let mut tmp_rep = *rep; + compress_block_opt2(ms, seq_store, &mut tmp_rep, src, DictMode::NoDict)?; + + // Of the first pass, only the statistics are kept: what the tables have is made out of reach. + let src_size = src.end - src.start; + seq_store.reset(); + ms.window.base = ms.window.base.wrapping_sub(src_size); + ms.window.dict_limit = ms.window.dict_limit.wrapping_add(src_size as u32); + ms.window.low_limit = ms.window.dict_limit; + ms.next_to_update = ms.window.dict_limit; + Ok(BlockSrc { + base: src.base.wrapping_sub(src_size), + ..*src + }) +} + +/// `ZSTD_compressBlock_btultra2` +fn compress_block_btultra2( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let curr = src.index_of(src.start); + + // The first block is parsed twice, the first time to collect statistics for the second. Since the first pass is + // then forgotten, nothing must be in the tables before it: no dictionary, no prefix, no long distance matching. + if ms.opt.lit_length_sum == 0 + && seq_store.sequences.is_empty() + && ms.window.dict_limit == ms.window.low_limit + && curr == ms.window.dict_limit + && src.end - src.start > ZSTD_PREDEF_THRESHOLD + { + let src = init_stats_ultra(ms, seq_store, rep, src)?; + return compress_block_opt2(ms, seq_store, rep, &src, DictMode::NoDict); + } + + compress_block_opt2(ms, seq_store, rep, src, DictMode::NoDict) +} + +/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` +/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. +/// +/// `btultra2` can move `ms.window.base`, which `src.base` comes from. +pub(crate) fn compress_block_opt( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + dict_mode: DictMode, +) -> Result { + match (strategy, dict_mode) { + (Strategy::BtUltra2, DictMode::NoDict) => compress_block_btultra2(ms, seq_store, rep, src), + (Strategy::BtUltra | Strategy::BtUltra2, _) => { + compress_block_opt2(ms, seq_store, rep, src, dict_mode) + } + _ => compress_block_opt0(ms, seq_store, rep, src, dict_mode), + } +} diff --git a/src/zstd/wip/work/ldm/zstd/compress/presplit.rs b/src/zstd/wip/work/ldm/zstd/compress/presplit.rs new file mode 100644 index 000000000000..6676175f0c1f --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/compress/presplit.rs @@ -0,0 +1,184 @@ +//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. + +use super::hist; +use crate::error::Result; +use crate::util::SliceExt; + +/// What the C code needs of workspace, which counts in the size of a context. +pub(crate) const ZSTD_SLIPBLOCK_WORKSPACESIZE: usize = 8208; + +const THRESHOLD_PENALTY_RATE: u64 = 16; +const THRESHOLD_BASE: i32 = THRESHOLD_PENALTY_RATE as i32 - 2; +const THRESHOLD_PENALTY: i32 = 3; + +const HASHLENGTH: usize = 2; +const KNUTH: u32 = 0x9e37_79b9; + +const CHUNKSIZE: usize = 8 << 10; +const SEGMENT_SIZE: usize = 512; + +/// `Fingerprint`, with a table of `SIZE` events, which is `1 << hashLog`: the C code leaves the rest of its table at 0. +struct Fingerprint { + events: [u32; SIZE], + nb_events: usize, +} + +impl Fingerprint { + const fn new() -> Self { + Self { + events: [0; SIZE], + nb_events: 0, + } + } + + /// `hash2`: a table of 256 takes the first byte as it is, a larger one a hash of the 2 bytes. + #[inline(always)] + const fn hash2(pair: [u8; HASHLENGTH]) -> usize { + if SIZE == 256 { + let [first, _] = pair; + return first as usize; + } + ((u16::from_le_bytes(pair) as u32).wrapping_mul(KNUTH) >> (32 - SIZE.trailing_zeros())) + as usize + } + + /// `addEvents_generic`: `src` has at least 2 bytes. + #[inline(always)] + fn add_events(&mut self, src: &[u8]) { + let limit = src.len().saturating_sub(HASHLENGTH - 1); + let mut n = 0; + while n < limit { + if let Some(pair) = src.get(n..).and_then(<[u8]>::first_chunk) + && let Some(event) = self.events.get_mut(Self::hash2(*pair) & (SIZE - 1)) + { + *event += 1; + } + n += SAMPLING_RATE; + } + self.nb_events += limit / SAMPLING_RATE; + } + + /// `recordFingerprint_generic` + #[inline(always)] + fn record(&mut self, src: &[u8]) { + self.events.fill(0); + self.nb_events = 0; + self.add_events::(src); + } + + /// `fpDistance` + fn distance(&self, other: &Self) -> u64 { + let nb_events1 = self.nb_events as i64; + let nb_events2 = other.nb_events as i64; + self.events + .iter() + .zip(&other.events) + .map(|(&event1, &event2)| { + (i64::from(event1) * nb_events2 - i64::from(event2) * nb_events1).unsigned_abs() + }) + .sum() + } + + /// `compareFingerprints`: whether `newfp` is too different from this one. + fn differs_from(&self, newfp: &Self, penalty: i32) -> bool { + let p50 = self.nb_events as u64 * newfp.nb_events as u64; + let deviation = self.distance(newfp); + let threshold = p50 * (THRESHOLD_BASE + penalty) as u64 / THRESHOLD_PENALTY_RATE; + deviation >= threshold + } + + /// `mergeEvents` + fn merge(&mut self, newfp: &Self) { + for (acc, &event) in self.events.iter_mut().zip(&newfp.events) { + *acc += event; + } + self.nb_events += newfp.nb_events; + } +} + +/// `ZSTD_splitBlock_byChunks`: cuts before the first chunk that is too different from all there is before it. +#[inline(always)] +fn split_block_by_chunks(block: &[u8]) -> usize { + let mut past_events = Fingerprint::::new(); + let mut new_events = Fingerprint::::new(); + let mut penalty = THRESHOLD_PENALTY; + let (chunks, _) = block.as_chunks::(); + let Some((first, chunks)) = chunks.split_first() else { + return block.len(); + }; + + past_events.record::(first); + let mut pos = CHUNKSIZE; + for chunk in chunks { + new_events.record::(chunk); + if past_events.differs_from(&new_events, penalty) { + return pos; + } + past_events.merge(&new_events); + if penalty > 0 { + penalty -= 1; + } + pos += CHUNKSIZE; + } + block.len() +} + +fn split_block_by_chunks_43(block: &[u8]) -> usize { + split_block_by_chunks::<43, { 1 << 8 }>(block) +} + +fn split_block_by_chunks_11(block: &[u8]) -> usize { + split_block_by_chunks::<11, { 1 << 9 }>(block) +} + +fn split_block_by_chunks_5(block: &[u8]) -> usize { + split_block_by_chunks::<5, { 1 << 10 }>(block) +} + +fn split_block_by_chunks_1(block: &[u8]) -> usize { + split_block_by_chunks::<1, { 1 << 10 }>(block) +} + +/// `ZSTD_splitBlock_fromBorders`: cuts if the start and the end of the block are too different. In the middle, or at a +/// quarter from the border that the middle is clearly the most different from. +fn split_block_from_borders(block: &[u8]) -> Result { + fn segment(block: &[u8], start: usize) -> Result> { + let mut fingerprint = Fingerprint::new(); + hist::add(&mut fingerprint.events, block.span(start, SEGMENT_SIZE)?)?; + fingerprint.nb_events = SEGMENT_SIZE; + Ok(fingerprint) + } + + let block_size = block.len(); + let past_events = segment(block, 0)?; + let new_events = segment(block, block_size.wrapping_sub(SEGMENT_SIZE))?; + if !past_events.differs_from(&new_events, 0) { + return Ok(block_size); + } + + let middle_events = segment(block, (block_size / 2).wrapping_sub(SEGMENT_SIZE / 2))?; + let dist_from_begin = past_events.distance(&middle_events); + let dist_from_end = new_events.distance(&middle_events); + let min_distance = (SEGMENT_SIZE * SEGMENT_SIZE / 3) as u64; + if dist_from_begin.abs_diff(dist_from_end) < min_distance { + return Ok(64 << 10); + } + Ok(if dist_from_begin > dist_from_end { + 32 << 10 + } else { + 96 << 10 + }) +} + +/// `ZSTD_splitBlock`: `block` is 128 KB, and `level` is from 0 to 4: the higher, the more work is put in. Returns the +/// size of the first part, which is the whole if the block is better left in one piece. +pub(crate) fn split_block(block: &[u8], level: i32) -> Result { + debug_assert!(block.len() == 128 << 10); + Ok(match level { + 0 => split_block_from_borders(block)?, + 1 => split_block_by_chunks_43(block), + 2 => split_block_by_chunks_11(block), + 3 => split_block_by_chunks_5(block), + _ => split_block_by_chunks_1(block), + }) +} diff --git a/src/zstd/wip/work/ldm/zstd/lib.rs b/src/zstd/wip/work/ldm/zstd/lib.rs new file mode 100644 index 000000000000..31c98c72e5e4 --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/lib.rs @@ -0,0 +1,385 @@ +//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the +//! same bytes in, in both directions, and the same errors. +#![forbid(unsafe_code)] +// Frames come from anywhere and a panic aborts the process. +#![deny( + clippy::indexing_slicing, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unreachable +)] +#![warn(unused_must_use)] + +mod common; +mod compress; +mod decompress; +mod error; +mod util; + +pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; +pub use decompress::{ + DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, + find_frame_compressed_size, get_frame_content_size, +}; +pub use error::ErrorCode; + +/// `ZSTD_inBuffer` +pub struct InBuffer<'a> { + pub src: &'a [u8], + /// How much of `src` is read. + pub pos: usize, +} + +/// `ZSTD_outBuffer` +pub struct OutBuffer<'a> { + pub dst: &'a mut [u8], + /// How much of `dst` is written. + pub pos: usize, +} + +/// `ZSTD_ResetDirective` +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ResetDirective { + SessionOnly = 1, + Parameters = 2, + SessionAndParameters = 3, +} + +// ----------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] +pub enum ZstdError { + /// The output, or the decoder state the frame's window size dictates, could not be allocated. + OutOfMemory, + InvalidZstdData, + DecompressionFailed, + ZstdDecompressionError, + ShortRead, +} + +impl core::fmt::Display for ZstdError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(<&'static str>::from(self)) + } +} +impl core::error::Error for ZstdError {} + +impl ZstdError { + /// The error for a failed decompression call; `other` is the non-allocation failure. + fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { + if code == ErrorCode::MemoryAllocation { + ZstdError::OutOfMemory + } else { + other + } + } +} + +/// `len` zeros for a decoder to write to. +fn output_buffer(len: usize) -> Result, ZstdError> { + util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) +} + +/// Decompress data, automatically allocating the output buffer. +/// Handles both frames with known and unknown content sizes. +/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. +/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. +pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { + let size = find_decompressed_size(src); + + const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit + + if size == ZSTD_CONTENTSIZE_ERROR { + return Err(ZstdError::InvalidZstdData); + } + + // Use streaming decompression if: + // 1. Content size is unknown, OR + // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) + if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { + let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { + // A frame's output is rarely smaller than its input. + src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) + } else { + // The header size is untrusted: reserve no more than the fast path below would. + MAX_PREALLOCATE_SIZE + }; + let mut list: Vec = Vec::new(); + list.try_reserve_exact(initial_capacity) + .map_err(|_| ZstdError::OutOfMemory)?; + StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; + return Ok(list); + } + + // Fast path: size is known and within reasonable limits + let mut output = output_buffer(size as usize)?; + let written = decompress(&mut output, src) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is +/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. +pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { + const MAX_SIZE: u64 = 16 * 1024 * 1024; + let size = find_decompressed_size(src); + // Also if it is unknown or invalid, which are the two largest values. + if size > MAX_SIZE { + return Err(ZstdError::InvalidZstdData); + } + let mut output = output_buffer(size as usize)?; + let written = DDict::prefix(prefix) + .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + Uninitialized, + Inflating, + End, + Error, +} + +/// Minimum spare output capacity offered to the decoder per call. +const STREAMING_OUTPUT_STEP: usize = 4096; + +// ────────────────────────────────────────────────────────────────────────── +// StreamingDecoder +// ────────────────────────────────────────────────────────────────────────── + +/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) +/// per call, so callers can hold the decoder across multiple body chunks. +pub struct StreamingDecoder { + stream: DCtx, + state: State, + /// The decoder filled its last window and may hold more. `max_output` can end a call there. + output_full: bool, +} + +impl Default for StreamingDecoder { + fn default() -> Self { + Self::new() + } +} + +impl StreamingDecoder { + pub fn new() -> Self { + Self { + stream: DCtx::new(), + state: State::Uninitialized, + output_full: false, + } + } + + #[inline] + pub fn is_inflating(&self) -> bool { + matches!(self.state, State::Inflating) + } + + /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is + /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. + /// Returns `ShortRead` when more input is required and `is_done` is false. + pub fn decompress( + &mut self, + input: &[u8], + out: &mut Vec, + max_output: usize, + is_done: bool, + ) -> Result { + if matches!(self.state, State::End | State::Error) { + return Ok(input.len()); + } + + let mut in_buf = InBuffer { src: input, pos: 0 }; + while matches!(self.state, State::Uninitialized | State::Inflating) { + let next_in_len = input.len() - in_buf.pos; + + // Call the decoder again with no input until it leaves the window short. + if next_in_len == 0 && !self.output_full { + if is_done { + if self.state == State::Inflating { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + self.state = State::End; + } + return Ok(in_buf.pos); + } + + if out.len() >= max_output { + return Ok(in_buf.pos); + } + + if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { + self.state = State::Error; + return Err(ZstdError::OutOfMemory); + } + // The decoder writes to initialized memory only. + let written_before = out.len(); + let window = (out.capacity() - written_before).min(max_output - written_before); + out.resize(written_before + window, 0); + let mut out_buf = OutBuffer { + dst: out.get_mut(written_before..).unwrap_or_default(), + pos: 0, + }; + let read_before = in_buf.pos; + + let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); + let bytes_written = out_buf.pos; + out.truncate(written_before + bytes_written); + let remaining = match result { + Ok(remaining) => remaining, + Err(code) => { + self.state = State::Error; + return Err(ZstdError::for_decompression( + code, + ZstdError::ZstdDecompressionError, + )); + } + }; + + self.output_full = bytes_written == window; + + if remaining == 0 { + // Frame complete, and fully flushed. + self.state = State::Uninitialized; + self.output_full = false; + if in_buf.pos >= input.len() { + if is_done { + self.state = State::End; + } + return Ok(in_buf.pos); + } + // More input available — reinitialize for the next frame. + self.stream.init_stream(); + continue; + } + + self.state = State::Inflating; + + if in_buf.pos - read_before == next_in_len { + if self.output_full { + continue; + } + if is_done { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + return Err(ZstdError::ShortRead); + } + } + Ok(in_buf.pos) + } +} + +// ── compressed embedded assets ──────────────────────────────────────────── + +/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes +/// Bun never executes or parses itself: the shell completion scripts and the +/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, +/// error overlay/page). Anything that runs inside Bun stays uncompressed. +/// +/// The exception is text that only `bun lint`, `bun fmt` and `bun check` +/// read. It may be compressed because it isn't on the path of running a program. +/// +/// Release (`bun_codegen_embed`) builds include the `.zst` twin that +/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to +/// `/compressed/.zst`; other builds evaluate `$fallback`. +#[macro_export] +macro_rules! embed_compressed { + // A codegen output (`/$sub`); debug builds read it at runtime. + (codegen $sub:literal) => { + $crate::embed_compressed!( + ("codegen/", $sub), + ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() + ) + }; + // Same, with a trailing NUL included in the slice (for C-string consumers). + (codegen_nul $sub:literal) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/codegen/", + $sub, + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); + &__COPY.get_or_init(|| { + let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); + let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); + copy.extend_from_slice(text); + copy.push(0); + copy.into_boxed_slice() + })[..] + } + }; + __bytes + }}; + // A file under `src/`; debug builds read it from the source tree at runtime. + (src $sub:literal) => { + $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) + }; + ($name:literal, $fallback:expr) => { + $crate::embed_compressed!(($name), $fallback) + }; + (($($name:literal),+), $fallback:expr) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/", + $($name,)+ + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + $fallback + } + }; + __bytes + }}; +} + +/// Cold, shared body of [`embed_compressed!`]'s release arm. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { + decompress_alloc(compressed) + .expect("embedded asset: invalid zstd frame") + .into_boxed_slice() +} + +/// [`inflate_embedded`] plus a trailing NUL byte. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { + let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); + inflated.reserve_exact(1); + inflated.push(0); + inflated.into_boxed_slice() +} +pub mod testing; diff --git a/src/zstd/wip/work/ldm/zstd/testing.rs b/src/zstd/wip/work/ldm/zstd/testing.rs new file mode 100644 index 000000000000..6c3ab8e86702 --- /dev/null +++ b/src/zstd/wip/work/ldm/zstd/testing.rs @@ -0,0 +1,656 @@ +//! Hooks for the differential tests. Only in scratch copies of the crate. +#![allow(clippy::all, dead_code, unreachable_pub)] + +use crate::ErrorCode; +use crate::compress::internal::*; +use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; + +fn bitmix(mut val: u64, len: u64) -> u64 { + val ^= val.rotate_right(49) ^ val.rotate_right(24); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^= (val >> 35).wrapping_add(len); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^ (val >> 28) +} + +/// The sequences of the first block of a frame, as (literal length, match length, offset), and then the literals +/// that are left as (length, 0, 0). +/// +/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, already adjusted to the sizes. +/// `dict` is a prefix (raw content, as `ZSTD_CCtx_refPrefix`), or empty. `src` is from 7 to 131071 bytes. +pub fn first_block( + cp: [u32; 7], + use_row: bool, + dict: &[u8], + src: &[u8], +) -> Result, ErrorCode> { + let strategy = Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?; + let c_params = CompressionParameters { + window_log: cp[0], + chain_log: cp[1], + hash_log: cp[2], + search_log: cp[3], + min_match: cp[4], + target_length: cp[5], + strategy: Some(strategy), + }; + let row_used = row_match_finder_supported(strategy) && use_row; + + // ZSTD_reset_matchState + let mut ms = MatchState::new(); + ms.hash_log3 = if c_params.min_match == 3 { + c_params.window_log.min(17) + } else { + 0 + }; + ms.window.clear(); + ms.next_to_update = ms.window.dict_limit; + ms.hash_table = vec![0; 1 << c_params.hash_log]; + ms.chain_table = vec![ + 0; + if strategy != Strategy::Fast && !row_used { + 1 << c_params.chain_log + } else { + 0 + } + ]; + ms.hash_table3 = vec![ + 0; + if ms.hash_log3 != 0 { + 1 << ms.hash_log3 + } else { + 0 + } + ]; + if row_used { + ms.tag_table = vec![0; 1 << c_params.hash_log]; + ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4); + ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); + } + if strategy >= Strategy::BtOpt { + ms.opt.lit_freq = vec![0; 256]; + ms.opt.lit_length_freq = vec![0; 36]; + ms.opt.match_length_freq = vec![0; 53]; + ms.opt.off_code_freq = vec![0; 32]; + ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; + ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; + } + ms.c_params = c_params; + + let block_size = (1usize << c_params.window_log).min(128 << 10); + let mut seq_store = SeqStore::new(); + seq_store.reserve( + block_size / if c_params.min_match == 3 { 3 } else { 4 }, + block_size, + )?; + + let bufs = Buffers { + in_buff: &[], + user: src, + dict, + }; + + // ZSTD_loadDictionaryContent(dtlm_fast, tfp_forCCtx) + if !dict.is_empty() { + let iend = dict.len(); + ms.window + .update(BufId::Dict.start(), BufId::Dict, dict.len(), false); + let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); + let ip = iend - dict.len().min(max_dict_size); + let prefix = Prefix { + buf: dict, + base: ms.window.base.wrapping_sub(BufId::Dict.start()), + }; + ms.next_to_update = prefix.index_of(ip); + ms.loaded_dict_end = prefix.index_of(iend); + if iend - ip > HASH_READ_SIZE { + match strategy { + Strategy::Fast => fast::fill_hash_table( + &mut ms, + prefix, + iend, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + )?, + Strategy::DFast => double_fast::fill_double_hash_table( + &mut ms, + prefix, + iend, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + )?, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { + if use_row { + ms.tag_table.fill(0); + lazy::row_update(&mut ms, prefix, iend - HASH_READ_SIZE)?; + } else { + lazy::insert_and_find_first_index(&mut ms, prefix, iend - HASH_READ_SIZE)?; + } + } + _ => opt::update_tree(&mut ms, prefix, iend - HASH_READ_SIZE, iend)?, + } + ms.next_to_update = prefix.index_of(iend); + } + } + + // ZSTD_compressContinue_internal + let src_addr = BufId::User.start(); + if !ms + .window + .update(src_addr, BufId::User, src.len(), ms.force_non_contiguous) + { + ms.force_non_contiguous = false; + ms.next_to_update = ms.window.dict_limit; + } + // ZSTD_compress_frameChunk + let max_dist = 1u32 << c_params.window_log; + if ms + .window + .dict_is_invalid(src_addr + src.len(), max_dist, ms.loaded_dict_end) + { + ms.loaded_dict_end = 0; + } + if ms + .window + .enforce_max_dist(src_addr, max_dist, ms.loaded_dict_end) + { + ms.loaded_dict_end = 0; + } + if ms.next_to_update < ms.window.low_limit { + ms.next_to_update = ms.window.low_limit; + } + // ZSTD_buildSeqStore + let prev = CompressedBlockState::new(); + let block = BlockSrc { + buf: bufs.get(ms.window.base_buf), + base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), + dict_buf: bufs.get(ms.window.dict_buf), + dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), + start: 0, + end: src.len(), + dms: None, + symbol_costs: &prev.entropy, + ldm_seq_store: None, + }; + let curr = block.index_of(0); + if curr > ms.next_to_update + 384 { + ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); + } + let dict_mode = ms.dict_mode(None); + let mut rep = prev.rep; + let last_ll = compress::compress_block( + &mut ms, + &mut seq_store, + &mut rep, + &block, + strategy, + use_row, + dict_mode, + )?; + + let mut out = Vec::new(); + let mut reps = prev.rep; + for (i, seq) in seq_store.sequences.iter().enumerate() { + let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); + if seq_store.long_length_pos as usize == i { + match seq_store.long_length_type { + LongLengthType::LiteralLength => ll += 0x10000, + LongLengthType::MatchLength => ml += 0x10000, + LongLengthType::None => {} + } + } + update_rep(&mut reps, seq.off_base, seq.lit_length == 0); + out.push((ll, ml, reps[0])); + } + out.push((last_ll as u32, 0, 0)); + assert_eq!( + reps, rep, + "the repeat offsets the block compressor leaves are those of its sequences" + ); + Ok(out) +} + +// ── long distance matching and the pre-splitter ─────────────────────────── + +use crate::compress::{ldm, presplit}; + +pub fn split_block(block: &[u8], level: i32) -> Result { + presplit::split_block(block, level) +} + +/// enableLdm, hashLog, bucketSizeLog, minMatchLength, hashRateLog, windowLog +fn ldm_params(p: [u32; 6]) -> LdmParams { + LdmParams { + enable_ldm: match p[0] { + 0 => ParamSwitch::Auto, + 1 => ParamSwitch::Enable, + _ => ParamSwitch::Disable, + }, + hash_log: p[1], + bucket_size_log: p[2], + min_match_length: p[3], + hash_rate_log: p[4], + window_log: p[5], + } +} + +pub fn ldm_adjust_parameters(p: [u32; 6], cp: [u32; 7]) -> [u32; 6] { + let mut params = ldm_params(p); + let c_params = CompressionParameters { + window_log: cp[0], + chain_log: cp[1], + hash_log: cp[2], + search_log: cp[3], + min_match: cp[4], + target_length: cp[5], + strategy: Strategy::from_int(cp[6] as i32), + }; + ldm::adjust_parameters(&mut params, &c_params); + [ + params.enable_ldm as u32, + params.hash_log, + params.bucket_size_log, + params.min_match_length, + params.hash_rate_log, + params.window_log, + ] +} +pub fn ldm_get_table_size(p: [u32; 6]) -> usize { + ldm::get_table_size(ldm_params(p)) +} +pub fn ldm_get_max_nb_seq(p: [u32; 6], max_chunk_size: usize) -> usize { + ldm::get_max_nb_seq(ldm_params(p), max_chunk_size) +} + +/// (offset, litLength, matchLength) +pub type Raw = (u32, u32, u32); +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct RawStore { + pub seq: Vec, + pub pos: usize, + pub pos_in_sequence: usize, + pub size: usize, +} +impl RawStore { + fn to(&self) -> RawSeqStore { + RawSeqStore { + seq: self + .seq + .iter() + .map(|s| RawSeq { + offset: s.0, + lit_length: s.1, + match_length: s.2, + }) + .collect(), + pos: self.pos, + pos_in_sequence: self.pos_in_sequence, + size: self.size, + capacity: self.seq.len(), + } + } + fn from(s: &RawSeqStore) -> Self { + Self { + seq: s + .seq + .iter() + .map(|s| (s.offset, s.lit_length, s.match_length)) + .collect(), + pos: s.pos, + pos_in_sequence: s.pos_in_sequence, + size: s.size, + } + } +} +pub fn ldm_skip_sequences( + store: &mut RawStore, + src_size: usize, + min_match: u32, +) -> Result<(), ErrorCode> { + let mut s = store.to(); + let r = ldm::skip_sequences(&mut s, src_size, min_match); + *store = RawStore::from(&s); + r +} +pub fn ldm_skip_raw_seq_store_bytes( + store: &mut RawStore, + nb_bytes: usize, +) -> Result<(), ErrorCode> { + let mut s = store.to(); + let r = ldm::skip_raw_seq_store_bytes(&mut s, nb_bytes); + *store = RawStore::from(&s); + r +} + +const BUF_IDS: [BufId; 3] = [BufId::InBuff, BufId::User, BufId::Dict]; + +pub struct Ldm { + state: LdmState, + params: LdmParams, + bufs: [Vec; 3], + store: RawSeqStore, +} +impl Ldm { + /// `start_index`: if not 0, the window is as if that many bytes had gone through it and it had been cleared. + pub fn new(p: [u32; 6], buf_sizes: [usize; 3], capacity: usize, start_index: u32) -> Self { + let params = ldm_params(p); + let mut state = LdmState::new(); + state.hash_table = vec![LdmEntry::default(); 1 << params.hash_log]; + state.bucket_offsets = vec![0; 1 << (params.hash_log - params.bucket_size_log)]; + if start_index != 0 { + state.window.next_src = state.window.base + start_index as usize; + state.window.clear(); + } + let store = RawSeqStore { + seq: vec![RawSeq::default(); capacity], + pos: 0, + pos_in_sequence: 0, + size: 0, + capacity, + }; + Self { + state, + params, + bufs: buf_sizes.map(|n| vec![0; n]), + store, + } + } + pub fn update(&mut self, buf: usize, pos: usize, data: &[u8]) { + self.bufs[buf][pos..pos + data.len()].copy_from_slice(data); + self.state + .window + .update(BUF_IDS[buf].start() + pos, BUF_IDS[buf], data.len(), false); + } + /// Changes a byte without telling the window. + pub fn poke(&mut self, buf: usize, pos: usize, byte: u8) { + self.bufs[buf][pos] = byte; + } + pub fn load_dict(&mut self, buf: usize, pos: usize, data: &[u8]) -> Result<(), ErrorCode> { + self.update(buf, pos, data); + let prefix = Prefix { + buf: &self.bufs[buf], + base: self.state.window.base.wrapping_sub(BUF_IDS[buf].start()), + }; + self.state.loaded_dict_end = prefix.index_of(pos + data.len()); + ldm::fill_hash_table(&mut self.state, prefix, pos, pos + data.len(), &self.params) + } + pub fn generate(&mut self, pos: usize, len: usize, reset: bool) -> Result<(), ErrorCode> { + if reset { + self.store.size = 0; + self.store.pos = 0; + } + let bufs = Buffers { + in_buff: &self.bufs[0], + user: &self.bufs[1], + dict: &self.bufs[2], + }; + ldm::generate_sequences( + &mut self.state, + &mut self.store, + &self.params, + &bufs, + pos, + pos + len, + ) + } + pub fn seqs(&self) -> Vec { + self.store.seq[..self.store.size] + .iter() + .map(|s| (s.offset, s.lit_length, s.match_length)) + .collect() + } + /// dictLimit, lowLimit, loadedDictEnd, nextSrc - base, nbOverflowCorrections + pub fn window(&self) -> [u32; 5] { + let w = &self.state.window; + [ + w.dict_limit, + w.low_limit, + self.state.loaded_dict_end, + w.index_of(w.next_src), + w.nb_overflow_corrections, + ] + } + pub fn table(&self) -> impl Iterator + '_ { + self.state.hash_table.iter().map(|e| (e.offset, e.checksum)) + } + pub fn bucket_offsets(&self) -> &[u8] { + &self.state.bucket_offsets + } +} + +/// `ZSTD_ldm_blockCompress` over the blocks of one input, as the first input of a frame. +pub struct LdmBlocks { + ms: MatchState, + seq_store: SeqStore, + store: RawSeqStore, + rep: [u32; 3], + prev: CompressedBlockState, + dict: Vec, + src: Vec, + use_row: bool, +} +#[derive(PartialEq, Eq, Debug)] +pub struct BlockOut { + /// offBase, litLength, mlBase + pub seqs: Vec<(u32, u16, u16)>, + pub long_length: Option<(u32, u32)>, + pub literals: Vec, + pub last_ll: usize, + pub rep: [u32; 3], + pub next_to_update: u32, + pub store: RawStore, +} +impl LdmBlocks { + pub fn new( + cp: [u32; 7], + use_row: bool, + dict: &[u8], + src: &[u8], + seqs: &[Raw], + ) -> Result { + let strategy = Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?; + let c_params = CompressionParameters { + window_log: cp[0], + chain_log: cp[1], + hash_log: cp[2], + search_log: cp[3], + min_match: cp[4], + target_length: cp[5], + strategy: Some(strategy), + }; + let row_used = row_match_finder_supported(strategy) && use_row; + let mut ms = MatchState::new(); + ms.hash_log3 = if c_params.min_match == 3 { + c_params.window_log.min(17) + } else { + 0 + }; + ms.window.clear(); + ms.next_to_update = ms.window.dict_limit; + ms.hash_table = vec![0; 1 << c_params.hash_log]; + ms.chain_table = vec![ + 0; + if strategy != Strategy::Fast && !row_used { + 1 << c_params.chain_log + } else { + 0 + } + ]; + ms.hash_table3 = vec![ + 0; + if ms.hash_log3 != 0 { + 1 << ms.hash_log3 + } else { + 0 + } + ]; + if row_used { + ms.tag_table = vec![0; 1 << c_params.hash_log]; + ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4); + ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); + } + if strategy >= Strategy::BtOpt { + ms.opt.lit_freq = vec![0; 256]; + ms.opt.lit_length_freq = vec![0; 36]; + ms.opt.match_length_freq = vec![0; 53]; + ms.opt.off_code_freq = vec![0; 32]; + ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; + ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; + } + ms.c_params = c_params; + let mut seq_store = SeqStore::new(); + seq_store.reserve((128 << 10) / 3, 128 << 10)?; + if !dict.is_empty() { + let iend = dict.len(); + ms.window + .update(BufId::Dict.start(), BufId::Dict, dict.len(), false); + let max_dict_size = + 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); + let ip = iend - dict.len().min(max_dict_size); + let prefix = Prefix { + buf: dict, + base: ms.window.base.wrapping_sub(BufId::Dict.start()), + }; + ms.next_to_update = prefix.index_of(ip); + ms.loaded_dict_end = prefix.index_of(iend); + if iend - ip > HASH_READ_SIZE { + match strategy { + Strategy::Fast => fast::fill_hash_table( + &mut ms, + prefix, + iend, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + )?, + Strategy::DFast => double_fast::fill_double_hash_table( + &mut ms, + prefix, + iend, + DictTableLoadMethod::Fast, + TableFillPurpose::ForCCtx, + )?, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { + if use_row { + ms.tag_table.fill(0); + lazy::row_update(&mut ms, prefix, iend - HASH_READ_SIZE)?; + } else { + lazy::insert_and_find_first_index( + &mut ms, + prefix, + iend - HASH_READ_SIZE, + )?; + } + } + _ => opt::update_tree(&mut ms, prefix, iend - HASH_READ_SIZE, iend)?, + } + ms.next_to_update = prefix.index_of(iend); + } + } + if !ms.window.update( + BufId::User.start(), + BufId::User, + src.len(), + ms.force_non_contiguous, + ) { + ms.force_non_contiguous = false; + ms.next_to_update = ms.window.dict_limit; + } + let prev = CompressedBlockState::new(); + let store = RawStore { + seq: seqs.to_vec(), + pos: 0, + pos_in_sequence: 0, + size: seqs.len(), + } + .to(); + Ok(Self { + ms, + seq_store, + store, + rep: prev.rep, + prev, + dict: dict.to_vec(), + src: src.to_vec(), + use_row, + }) + } + + pub fn run(&mut self, pos: usize, block_size: usize) -> Result { + let ms = &mut self.ms; + let src_addr = BufId::User.start() + pos; + let max_dist = 1u32 << ms.c_params.window_log; + if ms + .window + .dict_is_invalid(src_addr + block_size, max_dist, ms.loaded_dict_end) + { + ms.loaded_dict_end = 0; + } + if ms + .window + .enforce_max_dist(src_addr, max_dist, ms.loaded_dict_end) + { + ms.loaded_dict_end = 0; + } + if ms.next_to_update < ms.window.low_limit { + ms.next_to_update = ms.window.low_limit; + } + self.seq_store.reset(); + let mut last_ll = 0; + if block_size < 7 { + if ms.c_params.strat() >= Strategy::BtOpt { + ldm::skip_raw_seq_store_bytes(&mut self.store, block_size)?; + } else { + ldm::skip_sequences(&mut self.store, block_size, ms.c_params.min_match)?; + } + } else { + let bufs = Buffers { + in_buff: &[], + user: &self.src, + dict: &self.dict, + }; + let block = BlockSrc { + buf: bufs.get(ms.window.base_buf), + base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), + dict_buf: bufs.get(ms.window.dict_buf), + dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), + start: pos, + end: pos + block_size, + dms: None, + symbol_costs: &self.prev.entropy, + ldm_seq_store: None, + }; + let curr = block.index_of(pos); + if curr > ms.next_to_update + 384 { + ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); + } + last_ll = ldm::block_compress( + &mut self.store, + ms, + &mut self.seq_store, + &mut self.rep, + self.use_row, + &block, + )?; + self.seq_store + .store_last_literals(&self.src[pos + block_size - last_ll..pos + block_size])?; + } + Ok(BlockOut { + seqs: self + .seq_store + .sequences + .iter() + .map(|s| (s.off_base, s.lit_length, s.ml_base)) + .collect(), + long_length: match self.seq_store.long_length_type { + LongLengthType::None => None, + LongLengthType::LiteralLength => Some((1, self.seq_store.long_length_pos)), + LongLengthType::MatchLength => Some((2, self.seq_store.long_length_pos)), + }, + literals: self.seq_store.literals().to_vec(), + last_ll, + rep: self.rep, + next_to_update: self.ms.next_to_update, + store: RawStore::from(&self.store), + }) + } +} diff --git a/src/zstd/wip/work/opt/harness/Cargo.toml b/src/zstd/wip/work/opt/harness/Cargo.toml new file mode 100644 index 000000000000..131797af3edb --- /dev/null +++ b/src/zstd/wip/work/opt/harness/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "zstd_harness" +version = "0.0.0" +edition = "2024" + +[dependencies] +bun_zstd = { path = "/root/zstd-ref/work/opt/zstd" } + +[profile.release] +debug = "line-tables-only" +overflow-checks = true +debug-assertions = true +panic = "abort" + +[profile.bench] +inherits = "release" +overflow-checks = false +debug-assertions = false + +[workspace] diff --git a/src/zstd/wip/work/opt/harness/build.rs b/src/zstd/wip/work/opt/harness/build.rs new file mode 100644 index 000000000000..91da5928b882 --- /dev/null +++ b/src/zstd/wip/work/opt/harness/build.rs @@ -0,0 +1,20 @@ +use std::process::Command; +fn main() { + let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); + println!("cargo:rerun-if-env-changed=ZSTD_REF"); + println!("cargo:rerun-if-changed=shim.c"); + let out = std::env::var("OUT_DIR").unwrap(); + let ok = Command::new("clang-23") + .args(["-c", "-O2", "-fPIC", "-march=nehalem", "-DZSTD_MULTITHREAD", "-DZSTD_LEGACY_SUPPORT=0", "-DXXH_NAMESPACE=ZSTD_", + "-I/root/zstd-ref/zstd/lib", "-I/root/zstd-ref/zstd/lib/common", "shim.c", "-o"]) + .arg(format!("{out}/shim.o")).status().unwrap().success(); + assert!(ok); + let _ = std::fs::remove_file(format!("{out}/libshim.a")); + assert!(Command::new("llvm-ar-23").args(["rcs", &format!("{out}/libshim.a"), &format!("{out}/shim.o")]).status().unwrap().success()); + println!("cargo:rustc-link-search=native={out}"); + println!("cargo:rustc-link-lib=static=shim"); + let dir = if variant == "ref" || variant == "noasm" { format!("/root/zstd-ref/build/{variant}") } else { format!("/root/zstd-ref/work/opt/cbuild/{variant}") }; + println!("cargo:rerun-if-changed={dir}/libzstd.a"); + println!("cargo:rustc-link-search=native={dir}"); + println!("cargo:rustc-link-lib=static=zstd"); +} diff --git a/src/zstd/wip/work/opt/harness/rust-toolchain.toml b/src/zstd/wip/work/opt/harness/rust-toolchain.toml new file mode 100644 index 000000000000..c71778e8ccd3 --- /dev/null +++ b/src/zstd/wip/work/opt/harness/rust-toolchain.toml @@ -0,0 +1,35 @@ +[toolchain] +channel = "nightly-2026-09-15" +# rust-src is needed for -Zbuild-std (Tier 3 targets like +# aarch64-unknown-freebsd have no prebuilt std). miri is for +# `bun run rust:miri`. targets ensures the +# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain +# (CI's images install this same channel with these components and +# targets, from scripts/build/ci-images/spec.ts; a source lint keeps +# the two files in step). Note: rustup ignores this whole file when +# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree +# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` +# self-heals that path. Keep this list as a fallback for direct +# `cargo build`/`cargo check`/rust-analyzer use. +# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd +# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). +# darwin/windows triples build natively on their own agents, so the host std +# comes with the toolchain install regardless; they're listed here so a dev +# running `bun run rust:check-all` or `cargo check --target` from any host +# still gets prebuilt std for every Tier 1/2 triple. +# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at +# what it emits. The build does not use them. +components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] +targets = [ + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-musl", + "aarch64-linux-android", + "x86_64-linux-android", + "x86_64-unknown-freebsd", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-pc-windows-msvc", + "x86_64-pc-windows-msvc", +] diff --git a/src/zstd/wip/work/opt/harness/shim.c b/src/zstd/wip/work/opt/harness/shim.c new file mode 100644 index 000000000000..95dacbec60fc --- /dev/null +++ b/src/zstd/wip/work/opt/harness/shim.c @@ -0,0 +1,98 @@ +// Exposes the internal state of the C library to the differential tests. +#define ZSTD_STATIC_LINKING_ONLY +#include "compress/zstd_compress_internal.h" +#include +#include + +typedef struct { + unsigned cp[7]; + unsigned useRow; + unsigned hasDms; + unsigned nextToUpdate, loadedDictEnd, lowLimit, dictLimit, rowHashLog, hashSaltEntropy, lazySkipping; + unsigned long long hashSalt; + unsigned hashCache[8]; + unsigned rep[3]; + const unsigned* hashTable; + const unsigned* chainTable; + const unsigned char* tagTable; + const unsigned* hashTable3; + unsigned hashLog3; + unsigned sums[8]; + const unsigned* freqs[4]; +} ShimState; + +void shim_state(const ZSTD_CCtx* c, ShimState* s) { + const ZSTD_MatchState_t* ms = &c->blockState.matchState; + const ZSTD_compressionParameters* p = &c->appliedParams.cParams; + int i; + s->cp[0] = p->windowLog; s->cp[1] = p->chainLog; s->cp[2] = p->hashLog; s->cp[3] = p->searchLog; + s->cp[4] = p->minMatch; s->cp[5] = p->targetLength; s->cp[6] = (unsigned)p->strategy; + s->useRow = c->appliedParams.useRowMatchFinder == ZSTD_ps_enable; + s->hasDms = ms->dictMatchState != NULL; + s->nextToUpdate = ms->nextToUpdate; s->loadedDictEnd = ms->loadedDictEnd; + s->lowLimit = ms->window.lowLimit; s->dictLimit = ms->window.dictLimit; + s->rowHashLog = ms->rowHashLog; s->hashSaltEntropy = ms->hashSaltEntropy; s->lazySkipping = (unsigned)ms->lazySkipping; + s->hashSalt = ms->hashSalt; + for (i = 0; i < 8; i++) s->hashCache[i] = ms->hashCache[i]; + for (i = 0; i < 3; i++) s->rep[i] = c->blockState.prevCBlock->rep[i]; + s->hashTable = ms->hashTable; s->chainTable = ms->chainTable; s->tagTable = ms->tagTable; + s->hashTable3 = ms->hashTable3; s->hashLog3 = ms->hashLog3; + s->sums[0] = ms->opt.litSum; s->sums[1] = ms->opt.litLengthSum; s->sums[2] = ms->opt.matchLengthSum; s->sums[3] = ms->opt.offCodeSum; + s->sums[4] = ms->opt.litSumBasePrice; s->sums[5] = ms->opt.litLengthSumBasePrice; s->sums[6] = ms->opt.matchLengthSumBasePrice; s->sums[7] = ms->opt.offCodeSumBasePrice; + s->freqs[0] = ms->opt.litFreq; s->freqs[1] = ms->opt.litLengthFreq; s->freqs[2] = ms->opt.matchLengthFreq; s->freqs[3] = ms->opt.offCodeFreq; +} + +typedef struct { + unsigned long long huf[257]; + unsigned ll[36], ml[53], of[32]; /* deltaNbBits */ +} ShimEntropy; + +static void shim_poke_fse(FSE_CTable* ct, unsigned tableLog, unsigned maxSymbol, const unsigned* deltaNbBits) { + FSE_symbolCompressionTransform* tt = (FSE_symbolCompressionTransform*)(ct + 1 + (1 << (tableLog - 1))); + unsigned s; + ((U16*)ct)[0] = (U16)tableLog; ((U16*)ct)[1] = (U16)maxSymbol; + for (s = 0; s <= maxSymbol; s++) { tt[s].deltaNbBits = deltaNbBits[s]; tt[s].deltaFindState = 0; } +} + +/* A frame by the block-level API, with what the public parameters cannot set: exact parameters, sequences that stand + * for those of long distance matching, the entropy tables of a dictionary. `cdict` or `dict` or none. */ +size_t shim_frame2(ZSTD_CCtx* c, const unsigned cp[7], const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + int litMode, rawSeq* seqs, size_t nbSeqs, const ShimEntropy* e, + ZSTD_Sequence* out, size_t outCap, const void* first, size_t firstSize, const void* src, size_t srcSize) +{ + size_t const cap = ZSTD_compressBound(srcSize + firstSize); + void* const dst = malloc(cap); + size_t r; + if (cdict) { + ZSTD_frameParameters fp = { 0, 0, 0 }; + r = ZSTD_compressBegin_usingCDict_advanced(c, cdict, fp, ZSTD_CONTENTSIZE_UNKNOWN); + } else { + ZSTD_parameters p; + memset(&p, 0, sizeof(p)); + p.cParams.windowLog = cp[0]; p.cParams.chainLog = cp[1]; p.cParams.hashLog = cp[2]; p.cParams.searchLog = cp[3]; + p.cParams.minMatch = cp[4]; p.cParams.targetLength = cp[5]; p.cParams.strategy = (ZSTD_strategy)cp[6]; + r = ZSTD_compressBegin_advanced(c, dict, dictSize, p, srcSize + firstSize); + } + if (ZSTD_isError(r)) { free(dst); return r; } + c->appliedParams.postBlockSplitter = ZSTD_ps_disable; + c->appliedParams.literalCompressionMode = (ZSTD_ParamSwitch_e)litMode; + if (nbSeqs) ZSTD_referenceExternalSequences(c, seqs, nbSeqs); + if (e) { + ZSTD_entropyCTables_t* t = &c->blockState.prevCBlock->entropy; + memcpy(t->huf.CTable, e->huf, sizeof(e->huf)); + t->huf.repeatMode = HUF_repeat_valid; + shim_poke_fse(t->fse.litlengthCTable, LLFSELog, MaxLL, e->ll); + shim_poke_fse(t->fse.matchlengthCTable, MLFSELog, MaxML, e->ml); + shim_poke_fse(t->fse.offcodeCTable, OffFSELog, MaxOff, e->of); + } + c->seqCollector.collectSequences = 1; + c->seqCollector.seqStart = out; + c->seqCollector.seqIndex = 0; + c->seqCollector.maxSequences = outCap; + r = firstSize ? ZSTD_compressContinue_public(c, dst, cap, first, firstSize) : 0; + if (!ZSTD_isError(r)) r = ZSTD_compressEnd_public(c, dst, cap, src, srcSize); + free(dst); + c->seqCollector.collectSequences = 0; + if (ZSTD_isError(r)) return r; + return c->seqCollector.seqIndex; +} diff --git a/src/zstd/wip/work/opt/harness/src/c.rs b/src/zstd/wip/work/opt/harness/src/c.rs new file mode 100644 index 000000000000..d5a82e1f6b78 --- /dev/null +++ b/src/zstd/wip/work/opt/harness/src/c.rs @@ -0,0 +1,104 @@ +//! The reference C library. +#![allow(non_camel_case_types, dead_code)] +use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; + +#[repr(C)] +pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } +#[repr(C)] +pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } +pub enum ZSTD_CCtx {} +pub enum ZSTD_DCtx {} +pub enum ZSTD_DDict {} + +unsafe extern "C" { + pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; + pub fn ZSTD_compressBound(size: usize) -> usize; + pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_isError(code: usize) -> c_uint; + pub fn ZSTD_getErrorCode(code: usize) -> c_uint; + pub fn ZSTD_getErrorName(code: usize) -> *const c_char; + pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; + pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; + pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; + pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; + pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; + pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; + pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; + pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; + pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; + pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; + pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; + pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; + pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; + pub fn ZSTD_defaultCLevel() -> c_int; + pub fn ZSTD_minCLevel() -> c_int; + pub fn ZSTD_maxCLevel() -> c_int; +} + +/// `Ok(value)` or `Err(error code)`. +pub fn res(code: usize) -> Result { + unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } +} + +pub struct CCtx(pub *mut ZSTD_CCtx); +impl CCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } + pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + /// Returns (result, out pos, in pos). + pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); + (r, o.pos, i.pos) + } +} +impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } + +pub struct DCtx(pub *mut ZSTD_DCtx); +impl DCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } + pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } + pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); + (r, o.pos, i.pos) + } +} +impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } + +pub fn compress(src: &[u8], level: i32) -> Vec { + let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; + let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); + dst.truncate(n); + dst +} +pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) +} diff --git a/src/zstd/wip/work/opt/harness/src/data.rs b/src/zstd/wip/work/opt/harness/src/data.rs new file mode 100644 index 000000000000..c48a645e1b5d --- /dev/null +++ b/src/zstd/wip/work/opt/harness/src/data.rs @@ -0,0 +1,65 @@ +//! Deterministic inputs. +pub struct Rng(pub u64); +impl Rng { + pub fn next(&mut self) -> u64 { + // splitmix64 + self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + } + pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } + pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } + pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } + pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } + /// Skewed towards small values. + pub fn size(&mut self, max: usize) -> usize { + let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); + (self.next() as usize & ((1usize << bits) - 1)).min(max) + } +} + +static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; + +/// One buffer of about `len` bytes, of a kind picked by `rng`. +pub fn data(rng: &mut Rng, len: usize) -> Vec { + let mut v = Vec::with_capacity(len + 64); + match rng.below(9) { + 0 => v.resize(len, rng.next() as u8), + 1 => while v.len() < len { v.push(rng.next() as u8) }, + 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, + 3 => { // few symbols, skewed + let n = rng.range(2, 40); + while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } + } + 4 => { // LZ-like: copies of earlier parts, at all distances + while v.len() < len { + if v.len() > 8 && rng.chance(3, 4) { + let dist = 1 + rng.size(v.len() - 1); + let n = 3 + rng.size(300); + for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } + } else { + for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } + } + } + } + 5 => { // periodic with noise + let period = rng.range(1, 300); + let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); + while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } + } + 6 => { // sections of different kinds + while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } + } + 7 => { // numbers as text + let mut x = rng.next() % 1000; + while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } + } + _ => { // long runs + while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } + } + } + v.truncate(len); + v +} diff --git a/src/zstd/wip/work/opt/harness/src/main.rs b/src/zstd/wip/work/opt/harness/src/main.rs new file mode 100644 index 000000000000..adce8cad3d7a --- /dev/null +++ b/src/zstd/wip/work/opt/harness/src/main.rs @@ -0,0 +1,25 @@ +mod c; +mod data; +mod seqs; + +fn main() { + let args: Vec = std::env::args().collect(); + let cmd = args.get(1).map(String::as_str).unwrap_or(""); + let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); + let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); + match cmd { + // seqs + "seqs" => { + let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); + // 0: no dictionary, 1: a prefix or none, 2: an attached CDict or none + let modes: &[seqs::Mode] = match args.get(5).map(String::as_str) { + Some("1") => &[seqs::Mode::NoDict, seqs::Mode::Prefix], + Some("3") => &[seqs::Mode::CDict, seqs::Mode::Prefix, seqs::Mode::NoDict], + Some("2") => &[seqs::Mode::CDict, seqs::Mode::CDict, seqs::Mode::CDict, seqs::Mode::NoDict], + _ => &[seqs::Mode::NoDict], + }; + seqs::run(seed, iters, &strategies, modes, args.get(6).is_some_and(|s| s == "x")); + } + _ => eprintln!("unknown command"), + } +} diff --git a/src/zstd/wip/work/opt/harness/src/seqs.rs b/src/zstd/wip/work/opt/harness/src/seqs.rs new file mode 100644 index 000000000000..57a0a4752b18 --- /dev/null +++ b/src/zstd/wip/work/opt/harness/src/seqs.rs @@ -0,0 +1,235 @@ +//! A frame through the Rust block compressors and through the C library (`ZSTD_generateSequences`). +use crate::c; +use crate::data::{Rng, data}; +use bun_zstd::testing::{Dict, Extra, State}; +use std::ffi::{c_uint, c_void}; + +#[repr(C)] +#[derive(Clone, Copy, Default, Debug)] +struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } +#[repr(C)] +#[derive(Clone, Copy, Debug)] +struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } +#[repr(C)] +#[derive(Clone, Copy)] +struct CustomMem { a: *const c_void, b: *const c_void, c: *const c_void } +#[repr(C)] +struct ShimState { + cp: [c_uint; 7], use_row: c_uint, has_dms: c_uint, + next_to_update: c_uint, loaded_dict_end: c_uint, low_limit: c_uint, dict_limit: c_uint, row_hash_log: c_uint, hash_salt_entropy: c_uint, lazy_skipping: c_uint, + hash_salt: u64, hash_cache: [c_uint; 8], rep: [c_uint; 3], + hash_table: *const u32, chain_table: *const u32, tag_table: *const u8, + hash_table3: *const u32, hash_log3: c_uint, sums: [c_uint; 8], freqs: [*const u32; 4], +} +enum CDict {} +enum CCtxParams {} +unsafe extern "C" { + fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; + fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; + fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; + fn ZSTD_createCCtxParams() -> *mut CCtxParams; + fn ZSTD_freeCCtxParams(p: *mut CCtxParams) -> usize; + fn ZSTD_CCtxParams_setParameter(p: *mut CCtxParams, param: c_uint, v: i32) -> usize; + fn ZSTD_createCDict_advanced2(d: *const c_void, n: usize, load: c_uint, ty: c_uint, p: *const CCtxParams, mem: CustomMem) -> *mut CDict; + fn ZSTD_freeCDict(d: *mut CDict) -> usize; + fn ZSTD_CCtx_refCDict(c: *mut c::ZSTD_CCtx, d: *const CDict) -> usize; + fn ZSTD_getCParamsFromCDict(d: *const CDict) -> CParams; + fn shim_state(c: *const c::ZSTD_CCtx, s: *mut ShimState); + fn shim_frame2(c: *mut c::ZSTD_CCtx, cp: *const c_uint, dict: *const c_void, dict_size: usize, cdict: *const CDict, lit_mode: i32, + seqs: *mut [u32; 3], nb_seqs: usize, e: *const ShimEntropy, out: *mut Sequence, cap: usize, first: *const c_void, first_size: usize, src: *const c_void, size: usize) -> usize; +} + +#[repr(C)] +pub struct ShimEntropy { huf: [u64; 257], ll: [u32; 36], ml: [u32; 53], of: [u32; 32] } + +/// What `shim_frame2` sets beside the parameters. +pub struct CExtra { lit_mode: u32, ldm: Vec<[u32; 3]>, entropy: Option>, first_chunk: Vec } + +fn arr(p: CParams) -> [u32; 7] { [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy] } + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum Mode { NoDict, Prefix, CDict } + +struct COut { seqs: Vec<(u32, u32, u32, u32)>, cp: [u32; 7], use_row: bool, cdict_cp: [u32; 7], state: State } + +/// `None` if the C library refuses (a last block that is too small). +fn c_frame(cp: [u32; 7], use_row: bool, mode: Mode, dict: &[u8], src: &[u8], extra: Option<&CExtra>) -> Option { + let mut cctx = c::CCtx::new(); + let mut cdict = std::ptr::null_mut(); + let mut cdict_cp = [0; 7]; + cctx.set(1010, 2).unwrap(); // no block splitter + if mode == Mode::CDict { + unsafe { + let p = ZSTD_createCCtxParams(); + for (i, v) in cp.iter().enumerate() { c::res(ZSTD_CCtxParams_setParameter(p, 101 + i as u32, *v as i32)).unwrap(); } + c::res(ZSTD_CCtxParams_setParameter(p, 1011, if use_row { 1 } else { 2 })).unwrap(); + let null = std::ptr::null(); + cdict = ZSTD_createCDict_advanced2(dict.as_ptr().cast(), dict.len(), 1, 1, p, CustomMem { a: null, b: null, c: null }); + assert!(!cdict.is_null()); + ZSTD_freeCCtxParams(p); + if extra.is_none() { c::res(ZSTD_CCtx_refCDict(cctx.0, cdict)).unwrap(); } + cdict_cp = arr(ZSTD_getCParamsFromCDict(cdict)); + } + cctx.set(1001, 1).unwrap(); // force attach + cctx.set(101, cp[0] as i32).unwrap(); + } else { + for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } + cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder + if mode == Mode::Prefix { cctx.ref_prefix(dict).unwrap(); } + } + let total = src.len() + extra.map_or(0, |x| x.first_chunk.len()); + let mut out = vec![Sequence::default(); total / 3 + total / 1024 + 8]; + let n = c::res(unsafe { + match extra { + None => ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()), + Some(x) => { + let mut ldm = x.ldm.clone(); + let d = if mode == Mode::Prefix { dict } else { &[] }; + shim_frame2(cctx.0, cp.as_ptr(), d.as_ptr().cast(), d.len(), cdict, x.lit_mode as i32, ldm.as_mut_ptr(), ldm.len(), + x.entropy.as_deref().map_or(std::ptr::null(), |e| e), out.as_mut_ptr(), out.len(), x.first_chunk.as_ptr().cast(), x.first_chunk.len(), src.as_ptr().cast(), src.len()) + } + } + }); + let r = n.ok().map(|n| { + let mut s: ShimState = unsafe { std::mem::zeroed() }; + unsafe { shim_state(cctx.0, &mut s) }; + let row_used = s.use_row != 0 && (3..=5).contains(&s.cp[6]); + let table = |p: *const u32, n: usize| if n == 0 { Vec::new() } else { unsafe { std::slice::from_raw_parts(p, n) }.to_vec() }; + let state = State { + has_dms: s.has_dms != 0, next_to_update: s.next_to_update, loaded_dict_end: s.loaded_dict_end, low_limit: s.low_limit, dict_limit: s.dict_limit, + hash_salt_entropy: s.hash_salt_entropy, lazy_skipping: s.lazy_skipping != 0, hash_cache: s.hash_cache, rep: s.rep, + hash_table: table(s.hash_table, 1 << s.cp[2]), + chain_table: table(s.chain_table, if s.cp[6] != 1 && !row_used { 1 << s.cp[1] } else { 0 }), + hash_table3: table(s.hash_table3, if s.hash_log3 != 0 { 1 << s.hash_log3 } else { 0 }), + sums: if s.cp[6] >= 7 { s.sums } else { [0; 8] }, + freqs: if s.cp[6] >= 7 { [table(s.freqs[0], 256), table(s.freqs[1], 36), table(s.freqs[2], 53), table(s.freqs[3], 32)] } else { Default::default() }, + tag_table: if row_used { unsafe { std::slice::from_raw_parts(s.tag_table, 1 << s.cp[2]) }.to_vec() } else { Vec::new() }, + }; + COut { + seqs: out[..n].iter().map(|s| (s.lit_length, s.match_length, s.rep, if s.rep == 0 { s.offset } else { 0 })).collect(), + cp: s.cp, use_row: s.use_row != 0, cdict_cp, state, + } + }); + if !cdict.is_null() { unsafe { ZSTD_freeCDict(cdict) }; } + r +} + +fn first_diff(a: &[T], b: &[T]) -> Option { + if a.len() != b.len() { return Some(a.len().min(b.len())); } + a.iter().zip(b).position(|(a, b)| a != b) +} + +/// `strategies`: which of 1..=9 to test. `modes`: which kinds of dictionary. +pub fn run(seed: u64, iters: u64, strategies: &[u32], modes: &[Mode], with_extra: bool) { + let (mut with_ldm, mut with_entropy) = (0u64, 0u64); + let mut skipped = 0u64; + let mut dms_blocks = 0u64; + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if i % 2000 == 0 { eprintln!("seqs iter {i} (seed {s})"); } + let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071, 400_000]); + let len = rng.range(7, max); + let mut src = data(&mut rng, len); + if rng.chance(1, 40) { + // Whole blocks of noise, of which only the end matches something. + src = (0..(128 << 10) * rng.range(1, 2)).map(|_| rng.next() as u8).collect(); + let n = rng.range(9, 40); + let (from, to) = (rng.below(src.len() - 2 * n), src.len() - n); + src.copy_within(from..from + n, to); + } + let mode = *rng.pick(modes); + let mut dict = if mode != Mode::NoDict { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; + if !dict.is_empty() && rng.chance(1, 4) { + // The input goes on where the dictionary ends. + let mut all = data(&mut rng, dict.len() + src.len()); + src = all.split_off(dict.len()); + dict = all; + } + let mode = if dict.is_empty() { Mode::NoDict } else { mode }; + if !dict.is_empty() && rng.chance(2, 3) { + for _ in 0..rng.range(1, 6) { + let n = rng.size(dict.len().min(src.len())); + let at = if rng.chance(1, 3) { dict.len() - n } else { rng.below(dict.len() - n + 1) }; + let to = if rng.chance(1, 4) { 0 } else { rng.below(src.len() - n + 1) }; + src[to..to + n].copy_from_slice(&dict[at..at + n]); + } + } + let strategy = *rng.pick(strategies); + let p = if rng.chance(1, 3) { + // the parameters of a level, with the strategy forced + let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; + p.strategy = strategy; + p + } else { + CParams { + window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, + search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, + target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, + } + }; + let p = if with_extra && rng.chance(1, 2) { p } else { unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) } }; + let use_row = rng.chance(1, 2); + let extra = with_extra.then(|| { + let mut ldm = Vec::new(); + if rng.chance(2, 3) { + let max_ll = *rng.pick(&[4usize, 100, 3000, 100_000]); + let max_ml = *rng.pick(&[8usize, 100, 3000, 5000, 200_000]); + let target = if rng.chance(1, 4) { rng.below(src.len()) } else { src.len() + rng.below(1000) }; + let mut total = 0; + while total < target { + let (ll, ml) = (rng.size(max_ll), rng.size(max_ml)); + // Most of them are matches, as far as the block goes. + let at = total + ll; + let offset = if rng.chance(1, 8) || at == 0 || at >= src.len() { 1 + rng.size(1 << p.window_log) } else { 1 + rng.size(at - 1) }; + ldm.push([offset as u32, ll as u32, ml as u32]); + total += ll + ml; + } + } + let entropy = rng.chance(1, 2).then(|| { + let mut e = Box::new(ShimEntropy { huf: [0; 257], ll: [0; 36], ml: [0; 53], of: [0; 32] }); + let max_symbol = if rng.chance(3, 4) { 255 } else { rng.below(256) as u64 }; + e.huf[0] = 11 | max_symbol << 8; + let lo = rng.below(2); + for v in &mut e.huf[1..] { *v = rng.range(lo, 11) as u64 | rng.next() << 8; } + let mut fill = |t: &mut [u32], max: usize| for v in t { let bits = rng.range(lo, max) as u32; *v = (bits << 16).wrapping_sub(rng.below(1 << 16) as u32 * (bits != 0) as u32); }; + fill(&mut e.ll, 10); fill(&mut e.ml, 10); fill(&mut e.of, 9); + e + }); + let first_chunk = if src.len() > 20 && rng.chance(1, 2) { + let rest = src.split_off(if rng.chance(1, 2) { rng.range(7, src.len() - 7) } else { src.len() - 7 - rng.size(src.len() - 14) }); + std::mem::replace(&mut src, rest) + } else { Vec::new() }; + CExtra { lit_mode: rng.below(3) as u32, ldm, entropy, first_chunk } + }); + let Some(mut c) = c_frame(arr(p), use_row, mode, &dict, &src, extra.as_ref()) else { skipped += 1; continue }; + let d = match mode { Mode::NoDict => Dict::None, Mode::Prefix => Dict::Prefix(&dict), Mode::CDict => Dict::CDict(&dict, c.cdict_cp) }; + let what = format!("seed {s}: {mode:?} cp {:?} cdict cp {:?} row {} dict {} src {}", c.cp, c.cdict_cp, c.use_row, dict.len(), src.len()); + let ldm: Vec<(u32, u32, u32)> = extra.iter().flat_map(|x| &x.ldm).map(|s| (s[0], s[1], s[2])).collect(); + let x = extra.as_ref().map_or(Extra::default(), |x| Extra { + lit_mode: x.lit_mode, ldm: &ldm, entropy: x.entropy.as_deref().map(|e| (&e.huf, &e.ll, &e.ml, &e.of)), + first_chunk: &x.first_chunk, + }); + with_ldm += !ldm.is_empty() as u64; + with_entropy += x.entropy.is_some() as u64; + let what = format!("{what} lit mode {} ldm {} entropy {} first chunk {}", x.lit_mode, ldm.len(), x.entropy.is_some(), x.first_chunk.len()); + let (got, mut state) = bun_zstd::testing::frame2(c.cp, c.use_row, d, &src, x).unwrap_or_else(|e| panic!("{what}: error {e:?}")); + if x.lit_mode == 2 { + // Nothing writes the statistics of literals. + state.freqs[0].clear(); c.state.freqs[0].clear(); + state.sums[0] = 0; c.state.sums[0] = 0; state.sums[4] = 0; c.state.sums[4] = 0; + } + if mode == Mode::CDict && dict.len() >= 8 { dms_blocks += 1; } + if let Some(at) = first_diff(&got, &c.seqs) { + panic!("{what}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", got.get(at), c.seqs.get(at), got.len(), c.seqs.len()); + } + if state != c.state { + let (a, b) = (&state, &c.state); + panic!("{what}: state differs: hash table 3 {:?} sums {:?} {:?} freqs {:?} hash table {:?} chain table {:?} tag table {:?}\n got {:?}\n exp {:?}", + first_diff(&a.hash_table3, &b.hash_table3), a.sums, b.sums, a.freqs == b.freqs, first_diff(&a.hash_table, &b.hash_table), first_diff(&a.chain_table, &b.chain_table), first_diff(&a.tag_table, &b.tag_table), + (a.has_dms, a.next_to_update, a.loaded_dict_end, a.low_limit, a.dict_limit, a.hash_salt_entropy, a.lazy_skipping, a.hash_cache, a.rep), + (b.has_dms, b.next_to_update, b.loaded_dict_end, b.low_limit, b.dict_limit, b.hash_salt_entropy, b.lazy_skipping, b.hash_cache, b.rep)); + } + } + println!("seqs ok: {iters} iterations from seed {seed} ({skipped} skipped, {dms_blocks} with an attached CDict, {with_ldm} with LDM sequences, {with_entropy} with entropy tables), strategies {strategies:?}, modes {modes:?}"); +} diff --git a/src/zstd/wip/work/opt/zstd/compress/double_fast.rs b/src/zstd/wip/work/opt/zstd/compress/double_fast.rs new file mode 100644 index 000000000000..65d401675d1b --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/compress/double_fast.rs @@ -0,0 +1,48 @@ +//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_double_hash_table( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _end: usize, + _dtlm: DictTableLoadMethod, + _tfp: TableFillPurpose, +) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast` +pub(crate) fn compress_block_double_fast( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState` +pub(crate) fn compress_block_double_fast_dict_match_state( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_doubleFast_extDict` +pub(crate) fn compress_block_double_fast_ext_dict( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/opt/zstd/compress/fast.rs b/src/zstd/wip/work/opt/zstd/compress/fast.rs new file mode 100644 index 000000000000..1f4788af28b4 --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/compress/fast.rs @@ -0,0 +1,48 @@ +//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. + +use super::internal::{ + BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_hash_table( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _end: usize, + _dtlm: DictTableLoadMethod, + _tfp: TableFillPurpose, +) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast` +pub(crate) fn compress_block_fast( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast_dictMatchState` +pub(crate) fn compress_block_fast_dict_match_state( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_fast_extDict` +pub(crate) fn compress_block_fast_ext_dict( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/opt/zstd/compress/fse.rs b/src/zstd/wip/work/opt/zstd/compress/fse.rs new file mode 100644 index 000000000000..1b8f7cbf3173 --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/compress/fse.rs @@ -0,0 +1,70 @@ +//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. + +use crate::common::{MAX_SEQ, ML_FSE_LOG}; +use crate::util::SliceExt; + +/// `FSE_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum FseRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `FSE_symbolCompressionTransform` +#[derive(Clone, Copy, Default)] +pub(crate) struct FseSymbolTransform { + pub(crate) delta_find_state: i32, + pub(crate) delta_nb_bits: u32, +} + +/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. +#[derive(Clone)] +pub(crate) struct FseCTable { + pub(crate) table_log: u32, + pub(crate) max_symbol_value: u32, + pub(crate) state_table: [u16; 1 << ML_FSE_LOG], + pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], +} + +impl FseCTable { + pub(crate) const fn new() -> Self { + Self { + table_log: 0, + max_symbol_value: 0, + state_table: [0; 1 << ML_FSE_LOG], + symbol_tt: [FseSymbolTransform { + delta_find_state: 0, + delta_nb_bits: 0, + }; MAX_SEQ as usize + 1], + } + } + + /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log + /// if the table has no code for it. + #[inline(always)] + pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 + } + + /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. + /// The table has a code for it. + #[inline(always)] + pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + let min_nb_bits = transform.delta_nb_bits >> 16; + let threshold = (min_nb_bits + 1) << 16; + let table_size = 1u32 << self.table_log; + let delta_from_threshold = + threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); + // A linear interpolation, which is very approximate. + let normalized_delta_from_threshold = + (delta_from_threshold << accuracy_log) >> self.table_log; + let bit_multiplier = 1u32 << accuracy_log; + ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) + } +} diff --git a/src/zstd/wip/work/opt/zstd/compress/hist.rs b/src/zstd/wip/work/opt/zstd/compress/hist.rs new file mode 100644 index 000000000000..becf42516867 --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/compress/hist.rs @@ -0,0 +1,33 @@ +//! `hist.c`: how many times each byte value occurs. + +use crate::error::Result; +use crate::util::SliceExt; + +/// `HIST_add`: `count` has room for every value in `src`. +pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { + for &byte in src { + *count.at_mut(usize::from(byte))? += 1; + } + Ok(()) +} + +/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. +/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. +pub(crate) fn count_simple( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + let mut max_symbol = *max_symbol_value as usize; + count.sub_mut(..=max_symbol)?.fill(0); + if src.is_empty() { + *max_symbol_value = 0; + return Ok(0); + } + add(count, src)?; + while count.at(max_symbol)? == 0 { + max_symbol -= 1; + } + *max_symbol_value = max_symbol as u32; + Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) +} diff --git a/src/zstd/wip/work/opt/zstd/compress/huf.rs b/src/zstd/wip/work/opt/zstd/compress/huf.rs new file mode 100644 index 000000000000..78373a89ebf0 --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/compress/huf.rs @@ -0,0 +1,38 @@ +//! `huf_compress.c`: Huffman coding of literals. + +use crate::common::HUF_SYMBOLVALUE_MAX; +use crate::util::SliceExt; + +/// `HUF_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum HufRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the +/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in +/// its highest bits. +#[derive(Clone)] +pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); + +impl HufCTable { + pub(crate) const fn new() -> Self { + Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) + } + + /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. + #[inline] + pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { + let [header, elements @ ..] = &self.0; + let max_symbol_value = (header >> 8) as u8; + if symbol_value > u32::from(max_symbol_value) { + return 0; + } + (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 + } +} diff --git a/src/zstd/wip/work/opt/zstd/compress/internal.rs b/src/zstd/wip/work/opt/zstd/compress/internal.rs new file mode 100644 index 000000000000..51157ea6c1ce --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/compress/internal.rs @@ -0,0 +1,1212 @@ +//! `zstd_compress_internal.h`: what the parts of the compressor share. + +use super::fse::{FseCTable, FseRepeat}; +use super::huf::{HufCTable, HufRepeat}; +use crate::common::{ + BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, + ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, +}; +use crate::error::{ErrorCode, Result}; +use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; + +pub(crate) const K_SEARCH_STRENGTH: u32 = 8; +pub(crate) const HASH_READ_SIZE: usize = 8; +/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. +pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; + +pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; +pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; + +pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; +pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; + +/// `ZSTD_strategy` +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub(crate) enum Strategy { + Fast = 1, + DFast = 2, + Greedy = 3, + Lazy = 4, + Lazy2 = 5, + BtLazy2 = 6, + BtOpt = 7, + BtUltra = 8, + BtUltra2 = 9, +} + +impl Strategy { + pub(crate) const fn from_int(value: i32) -> Option { + Some(match value { + 1 => Self::Fast, + 2 => Self::DFast, + 3 => Self::Greedy, + 4 => Self::Lazy, + 5 => Self::Lazy2, + 6 => Self::BtLazy2, + 7 => Self::BtOpt, + 8 => Self::BtUltra, + 9 => Self::BtUltra2, + _ => return None, + }) + } +} + +/// `ZSTD_ParamSwitch_e` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum ParamSwitch { + Auto = 0, + Enable = 1, + Disable = 2, +} + +/// `ZSTD_compressionParameters` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct CompressionParameters { + pub(crate) window_log: u32, + pub(crate) chain_log: u32, + pub(crate) hash_log: u32, + pub(crate) search_log: u32, + pub(crate) min_match: u32, + pub(crate) target_length: u32, + /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. + pub(crate) strategy: Option, +} + +impl CompressionParameters { + /// The strategy of parameters that are ready to be applied. + #[inline(always)] + pub(crate) fn strat(&self) -> Strategy { + self.strategy.unwrap_or(Strategy::Fast) + } +} + +/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum DictMode { + NoDict, + ExtDict, + DictMatchState, +} + +/// `ZSTD_dictTableLoadMethod_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DictTableLoadMethod { + Fast, + Full, +} + +/// `ZSTD_tableFillPurpose_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TableFillPurpose { + ForCCtx, + ForCDict, +} + +// ── entropy tables ──────────────────────────────────────────────────────── + +/// `ZSTD_hufCTables_t` +#[derive(Clone)] +pub(crate) struct HufCTables { + pub(crate) c_table: HufCTable, + pub(crate) repeat_mode: HufRepeat, +} + +/// `ZSTD_fseCTables_t` +#[derive(Clone)] +pub(crate) struct FseCTables { + pub(crate) offcode_c_table: FseCTable, + pub(crate) matchlength_c_table: FseCTable, + pub(crate) litlength_c_table: FseCTable, + pub(crate) offcode_repeat_mode: FseRepeat, + pub(crate) matchlength_repeat_mode: FseRepeat, + pub(crate) litlength_repeat_mode: FseRepeat, +} + +/// `ZSTD_entropyCTables_t` +#[derive(Clone)] +pub(crate) struct EntropyCTables { + pub(crate) huf: HufCTables, + pub(crate) fse: FseCTables, +} + +/// `ZSTD_compressedBlockState_t` +#[derive(Clone)] +pub(crate) struct CompressedBlockState { + pub(crate) entropy: EntropyCTables, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +impl CompressedBlockState { + /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. + pub(crate) fn new() -> Self { + Self { + entropy: EntropyCTables { + huf: HufCTables { + c_table: HufCTable::new(), + repeat_mode: HufRepeat::None, + }, + fse: FseCTables { + offcode_c_table: FseCTable::new(), + matchlength_c_table: FseCTable::new(), + litlength_c_table: FseCTable::new(), + offcode_repeat_mode: FseRepeat::None, + matchlength_repeat_mode: FseRepeat::None, + litlength_repeat_mode: FseRepeat::None, + }, + }, + rep: REP_START_VALUE, + } + } + + /// `ZSTD_reset_compressedBlockState` + pub(crate) fn reset(&mut self) { + self.rep = REP_START_VALUE; + self.entropy.huf.repeat_mode = HufRepeat::None; + self.entropy.fse.offcode_repeat_mode = FseRepeat::None; + self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; + self.entropy.fse.litlength_repeat_mode = FseRepeat::None; + } +} + +/// `ZSTD_hufCTablesMetadata_t` +pub(crate) struct HufCTablesMetadata { + pub(crate) h_type: SymbolEncodingType, + pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], + pub(crate) huf_des_size: usize, +} + +/// `ZSTD_fseCTablesMetadata_t` +pub(crate) struct FseCTablesMetadata { + pub(crate) ll_type: SymbolEncodingType, + pub(crate) of_type: SymbolEncodingType, + pub(crate) ml_type: SymbolEncodingType, + pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], + pub(crate) fse_tables_size: usize, + /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. + pub(crate) last_count_size: usize, +} + +/// `ZSTD_entropyCTablesMetadata_t` +pub(crate) struct EntropyCTablesMetadata { + pub(crate) huf_metadata: HufCTablesMetadata, + pub(crate) fse_metadata: FseCTablesMetadata, +} + +impl EntropyCTablesMetadata { + pub(crate) const fn new() -> Self { + Self { + huf_metadata: HufCTablesMetadata { + h_type: SymbolEncodingType::Basic, + huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], + huf_des_size: 0, + }, + fse_metadata: FseCTablesMetadata { + ll_type: SymbolEncodingType::Basic, + of_type: SymbolEncodingType::Basic, + ml_type: SymbolEncodingType::Basic, + fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], + fse_tables_size: 0, + last_count_size: 0, + }, + } + } +} + +// ── sequences ───────────────────────────────────────────────────────────── + +/// `SeqDef` +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct SeqDef { + /// A repeat code (1 to 3), or an offset plus 3. + pub(crate) off_base: u32, + pub(crate) lit_length: u16, + /// The match length minus [`MINMATCH`]. + pub(crate) ml_base: u16, +} + +/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum LongLengthType { + None, + LiteralLength, + MatchLength, +} + +/// `SeqStore_t`: the sequences and the literals of a block. +pub(crate) struct SeqStore { + /// Never grows beyond the capacity it is given by [`Self::reserve`]. + pub(crate) sequences: Vec, + /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. + pub(crate) lit_buffer: Vec, + /// How much of `lit_buffer` is literals. + pub(crate) lit_len: usize, + pub(crate) ll_code: Vec, + pub(crate) ml_code: Vec, + pub(crate) of_code: Vec, + pub(crate) max_nb_seq: usize, + pub(crate) max_nb_lit: usize, + pub(crate) long_length_type: LongLengthType, + /// Which sequence has the long length. + pub(crate) long_length_pos: u32, +} + +/// `ZSTD_SequenceLength` +#[derive(Clone, Copy)] +pub(crate) struct SequenceLength { + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +impl SeqStore { + pub(crate) const fn new() -> Self { + Self { + sequences: Vec::new(), + lit_buffer: Vec::new(), + lit_len: 0, + ll_code: Vec::new(), + ml_code: Vec::new(), + of_code: Vec::new(), + max_nb_seq: 0, + max_nb_lit: 0, + long_length_type: LongLengthType::None, + long_length_pos: 0, + } + } + + /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. + pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { + fn grow(vec: &mut Vec, len: usize) -> Result<()> { + vec.clear(); + vec.try_reserve_exact(len) + .map_err(|_| ErrorCode::MemoryAllocation)?; + vec.resize(len, T::default()); + Ok(()) + } + grow(&mut self.sequences, max_nb_seq)?; + grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; + grow(&mut self.ll_code, max_nb_seq)?; + grow(&mut self.ml_code, max_nb_seq)?; + grow(&mut self.of_code, max_nb_seq)?; + self.max_nb_seq = max_nb_seq; + self.max_nb_lit = max_nb_lit; + self.reset(); + Ok(()) + } + + /// `ZSTD_resetSeqStore` + #[inline] + pub(crate) fn reset(&mut self) { + self.lit_len = 0; + self.sequences.clear(); + self.long_length_type = LongLengthType::None; + } + + #[inline] + pub(crate) fn literals(&self) -> &[u8] { + self.lit_buffer.get(..self.lit_len).unwrap_or_default() + } + + /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. + #[inline(always)] + pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { + debug_assert!(self.sequences.len() < self.max_nb_seq); + if lit_length > 0xFFFF { + self.long_length_type = LongLengthType::LiteralLength; + self.long_length_pos = self.sequences.len() as u32; + } + let ml_base = match_length.wrapping_sub(MINMATCH as usize); + if ml_base > 0xFFFF { + self.long_length_type = LongLengthType::MatchLength; + self.long_length_pos = self.sequences.len() as u32; + } + self.sequences.push(SeqDef { + off_base, + lit_length: lit_length as u16, + ml_base: ml_base as u16, + }); + } + + /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. + #[inline(always)] + pub(crate) fn store_seq( + &mut self, + lit_length: usize, + buf: &[u8], + literals: usize, + off_base: u32, + match_length: usize, + ) -> Result<()> { + match ( + buf.array_at::<16>(literals), + self.lit_buffer.array_at_mut::<16>(self.lit_len), + ) { + (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, + _ => self + .lit_buffer + .write_at(self.lit_len, buf.span(literals, lit_length)?)?, + } + self.lit_len += lit_length; + self.store_seq_only(lit_length, off_base, match_length); + Ok(()) + } + + /// Appends the literals that follow the last sequence of a block. + #[inline] + pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { + self.lit_buffer.write_at(self.lit_len, literals)?; + self.lit_len += literals.len(); + Ok(()) + } +} + +pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; +pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; +pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; + +/// `OFFSET_TO_OFFBASE`: `offset` is not 0. +#[inline(always)] +pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { + offset.wrapping_add(ZSTD_REP_NUM as u32) +} + +/// `OFFBASE_IS_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { + off_base > ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_IS_REPCODE` +#[inline(always)] +pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { + 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_TO_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { + off_base.wrapping_sub(ZSTD_REP_NUM as u32) +} + +/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. +#[inline(always)] +pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { + let [rep0, rep1, rep2] = *rep; + if offbase_is_offset(off_base) { + *rep = [offbase_to_offset(off_base), rep0, rep1]; + } else { + match off_base - 1 + u32::from(ll0) { + 0 => {} + 1 => *rep = [rep1, rep0, rep2], + 2 => *rep = [rep2, rep0, rep1], + _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], + } + } +} + +/// `ZSTD_newRep` +#[inline(always)] +pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { + let mut new_reps = *rep; + update_rep(&mut new_reps, off_base, ll0); + new_reps +} + +/// `ZSTD_LLcode` +#[inline(always)] +pub(crate) fn ll_code(lit_length: u32) -> u32 { + #[rustfmt::skip] + const LL_CODE: [u8; 64] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + ]; + const LL_DELTA_CODE: u32 = 19; + match LL_CODE.get(lit_length as usize) { + Some(&code) => u32::from(code), + None => highbit32(lit_length) + LL_DELTA_CODE, + } +} + +/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. +#[inline(always)] +pub(crate) fn ml_code(ml_base: u32) -> u32 { + #[rustfmt::skip] + const ML_CODE: [u8; 128] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ]; + const ML_DELTA_CODE: u32 = 36; + match ML_CODE.get(ml_base as usize) { + Some(&code) => u32::from(code), + None => highbit32(ml_base) + ML_DELTA_CODE, + } +} + +/// `ZSTD_noCompressBlock` +pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { + let c_block_header24 = + u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); + if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header24)?; + dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; + Ok(ZSTD_BLOCKHEADERSIZE + src.len()) +} + +/// `ZSTD_rleCompressBlock` +pub(crate) fn rle_compress_block( + dst: &mut [u8], + byte: u8, + src_size: usize, + last_block: bool, +) -> Result { + let c_block_header = + u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); + if dst.len() < 4 { + return Err(ErrorCode::DstSizeTooSmall); + } + write_le24(dst, 0, c_block_header)?; + dst.put(3, byte)?; + Ok(4) +} + +/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. +#[inline] +pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { + let minlog = if strat >= Strategy::BtUltra { + strat as u32 - 1 + } else { + 6 + }; + (src_size >> minlog) + 2 +} + +// ── matching ────────────────────────────────────────────────────────────── + +/// How many bytes `a` and `b` have in common at their start. +#[inline(always)] +pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { + let (a_words, _) = a.as_chunks::<8>(); + let (b_words, _) = b.as_chunks::<8>(); + let mut len = 0usize; + for (a_word, b_word) in a_words.iter().zip(b_words) { + let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); + if diff != 0 { + return len + (diff.trailing_zeros() >> 3) as usize; + } + len += 8; + } + let a_tail = a.get(len..).unwrap_or_default(); + let b_tail = b.get(len..).unwrap_or_default(); + len + a_tail + .iter() + .zip(b_tail) + .take_while(|(a_byte, b_byte)| a_byte == b_byte) + .count() +} + +/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. +#[inline(always)] +pub(crate) fn count( + buf: &[u8], + p_in: usize, + match_buf: &[u8], + p_match: usize, + p_in_limit: usize, +) -> Result { + Ok(common_prefix( + buf.sub(p_in..p_in_limit)?, + match_buf.sub(p_match..)?, + )) +} + +/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at +/// `i_start` in `buf`. +#[inline(always)] +pub(crate) fn count_2segments( + buf: &[u8], + ip: usize, + match_buf: &[u8], + match_pos: usize, + i_end: usize, + m_end: usize, + i_start: usize, +) -> Result { + let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); + if match_pos + match_length != m_end { + return Ok(match_length); + } + Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) +} + +const PRIME_3_BYTES: u32 = 506_832_829; +const PRIME_4_BYTES: u32 = 2_654_435_761; +const PRIME_5_BYTES: u64 = 889_523_592_379; +const PRIME_6_BYTES: u64 = 227_718_039_650_203; +const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; +pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; + +/// `ZSTD_hash3` +#[inline(always)] +const fn hash3(u: u32, h: u32, s: u32) -> u32 { + (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash4(u: u32, h: u32, s: u32) -> u32 { + ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) +} +#[inline(always)] +const fn hash5(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash6(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +#[inline(always)] +const fn hash7(u: u64, h: u32, s: u64) -> usize { + ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} +/// `ZSTD_hash8` +#[inline(always)] +pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { + (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize +} + +/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. +#[inline(always)] +pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { + Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) +} + +/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, +/// else 8. +#[inline(always)] +pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { + hash_ptr_salted(buf, pos, h_bits, mls, 0) +} + +/// `ZSTD_hashPtrSalted` +#[inline(always)] +pub(crate) fn hash_ptr_salted( + buf: &[u8], + pos: usize, + h_bits: u32, + mls: u32, + hash_salt: u64, +) -> Result { + Ok(match mls { + 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), + 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), + 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), + 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), + _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, + }) +} + +// ── the window ──────────────────────────────────────────────────────────── + +/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are +/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the +/// others, and an address is turned into a position in the buffer only to read. +pub(crate) type Addr = usize; + +/// The buffers that a window can have a segment in. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum BufId { + /// Where an empty window points. + None = 0, + /// The context's own buffer of input. + InBuff = 1, + /// What the caller gave to the call in progress. + User = 2, + /// The content of the dictionary, or the prefix. + Dict = 3, +} + +impl BufId { + /// The address of the first byte of the buffer. + #[inline(always)] + pub(crate) const fn start(self) -> Addr { + (self as usize + 1) << 44 + } +} + +/// The buffers themselves, for as long as a call lasts. +#[derive(Clone, Copy)] +pub(crate) struct Buffers<'a> { + pub(crate) in_buff: &'a [u8], + pub(crate) user: &'a [u8], + pub(crate) dict: &'a [u8], +} + +impl<'a> Buffers<'a> { + #[inline] + pub(crate) fn get(&self, id: BufId) -> &'a [u8] { + match id { + BufId::None => &[], + BufId::InBuff => self.in_buff, + BufId::User => self.user, + BufId::Dict => self.dict, + } + } +} + +/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. +pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; + +/// The index past which indices are brought down. +pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; +/// The most input that can be indexed at once. +pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; + +/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the +/// index is less than `dict_limit`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Window { + /// Where input has to continue to be contiguous. + pub(crate) next_src: Addr, + pub(crate) base: Addr, + pub(crate) dict_base: Addr, + /// The buffer of the segment that starts at `dict_limit`. + pub(crate) base_buf: BufId, + /// The buffer of the segment from `low_limit` to `dict_limit`. + pub(crate) dict_buf: BufId, + pub(crate) dict_limit: u32, + /// Below this, there is nothing valid. + pub(crate) low_limit: u32, + pub(crate) nb_overflow_corrections: u32, +} + +impl Window { + /// `ZSTD_window_init` + pub(crate) const fn new() -> Self { + let base = BufId::None.start(); + Self { + next_src: base + ZSTD_WINDOW_START_INDEX as usize, + base, + dict_base: base, + base_buf: BufId::None, + dict_buf: BufId::None, + dict_limit: ZSTD_WINDOW_START_INDEX, + low_limit: ZSTD_WINDOW_START_INDEX, + nb_overflow_corrections: 0, + } + } + + /// The index of `addr`, which is in the current segment or at its end. + #[inline(always)] + pub(crate) const fn index_of(&self, addr: Addr) -> u32 { + addr.wrapping_sub(self.base) as u32 + } + + /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. + pub(crate) fn clear(&mut self) { + let end = self.index_of(self.next_src); + self.low_limit = end; + self.dict_limit = end; + } + + /// `ZSTD_window_isEmpty` + pub(crate) fn is_empty(&self) -> bool { + self.dict_limit == ZSTD_WINDOW_START_INDEX + && self.low_limit == ZSTD_WINDOW_START_INDEX + && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize + } + + /// `ZSTD_window_hasExtDict` + #[inline] + pub(crate) fn has_ext_dict(&self) -> bool { + self.low_limit < self.dict_limit + } + + /// `ZSTD_window_needOverflowCorrection` + #[inline] + pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { + self.index_of(src_end) > ZSTD_CURRENT_MAX + } + + /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns + /// what to subtract from every index that is stored. + pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { + let cycle_size = 1u32 << cycle_log; + let cycle_mask = cycle_size - 1; + let curr = self.index_of(src); + let current_cycle = curr & cycle_mask; + // Indices must not fall below the start index. + let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { + cycle_size.max(ZSTD_WINDOW_START_INDEX) + } else { + 0 + }; + let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); + let correction = curr.wrapping_sub(new_current); + + self.base = self.base.wrapping_add(correction as usize); + self.dict_base = self.dict_base.wrapping_add(correction as usize); + self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.low_limit - correction + }; + self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.dict_limit - correction + }; + self.nb_overflow_corrections += 1; + correction + } + + /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the + /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. + pub(crate) fn enforce_max_dist( + &mut self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { + let new_low_limit = block_end_idx - max_dist; + if self.low_limit < new_low_limit { + self.low_limit = new_low_limit; + } + if self.dict_limit < self.low_limit { + self.dict_limit = self.low_limit; + } + return true; + } + false + } + + /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not + /// right before the current segment, and is to be forgotten. + pub(crate) fn dict_is_invalid( + &self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit + } + + /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not + /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. + pub(crate) fn update( + &mut self, + src: Addr, + src_buf: BufId, + src_size: usize, + force_non_contiguous: bool, + ) -> bool { + let mut contiguous = true; + if src_size == 0 { + return contiguous; + } + if src != self.next_src || force_non_contiguous { + let distance_from_base = self.next_src.wrapping_sub(self.base); + self.low_limit = self.dict_limit; + self.dict_limit = distance_from_base as u32; + self.dict_base = self.base; + self.dict_buf = self.base_buf; + self.base = src.wrapping_sub(distance_from_base); + self.base_buf = src_buf; + if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { + // Too small to be of use. + self.low_limit = self.dict_limit; + } + contiguous = false; + } + self.next_src = src + src_size; + // Where the input overlaps the older segment, it has overwritten it. + if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) + & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) + { + let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); + self.low_limit = if high_input_idx > self.dict_limit as usize { + self.dict_limit + } else { + high_input_idx as u32 + }; + } + contiguous + } +} + +/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes +/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. +#[inline(always)] +pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { + prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 +} + +pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; +pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; + +/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. +#[inline(always)] +pub(crate) fn write_tagged_index( + hash_table: &mut [u32], + hash_and_tag: usize, + index: u32, +) -> Result<()> { + let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; + hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) +} + +/// `ZSTD_comparePackedTags` +#[inline(always)] +pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { + (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) + == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) +} + +// ── the state of the match finders ──────────────────────────────────────── + +/// `ZSTD_match_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Match { + /// As [`SeqDef::off_base`]. + pub(crate) off: u32, + pub(crate) len: u32, +} + +/// `rawSeq`: a sequence found by long distance matching. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct RawSeq { + pub(crate) offset: u32, + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +/// `RawSeqStore_t` +#[derive(Clone, Default)] +pub(crate) struct RawSeqStore { + /// `capacity` of them, of which `size` are in use. + pub(crate) seq: Vec, + /// The sequence that is being read. + pub(crate) pos: usize, + /// How many of its bytes, literals first, are read. + pub(crate) pos_in_sequence: usize, + pub(crate) size: usize, + pub(crate) capacity: usize, +} + +/// `ZSTD_optimal_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct Optimal { + /// The cost of getting to this position. + pub(crate) price: i32, + pub(crate) off: u32, + pub(crate) mlen: u32, + pub(crate) litlen: u32, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +/// `ZSTD_OptPrice_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum OptPrice { + Dynamic, + Predef, +} + +/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. +pub(crate) struct OptState { + pub(crate) lit_freq: Vec, + pub(crate) lit_length_freq: Vec, + pub(crate) match_length_freq: Vec, + pub(crate) off_code_freq: Vec, + pub(crate) match_table: Vec, + pub(crate) price_table: Vec, + pub(crate) lit_sum: u32, + pub(crate) lit_length_sum: u32, + pub(crate) match_length_sum: u32, + pub(crate) off_code_sum: u32, + pub(crate) lit_sum_base_price: u32, + pub(crate) lit_length_sum_base_price: u32, + pub(crate) match_length_sum_base_price: u32, + pub(crate) off_code_sum_base_price: u32, + pub(crate) price_type: OptPrice, + pub(crate) literal_compression_mode: ParamSwitch, +} + +impl OptState { + pub(crate) const fn new() -> Self { + Self { + lit_freq: Vec::new(), + lit_length_freq: Vec::new(), + match_length_freq: Vec::new(), + off_code_freq: Vec::new(), + match_table: Vec::new(), + price_table: Vec::new(), + lit_sum: 0, + lit_length_sum: 0, + match_length_sum: 0, + off_code_sum: 0, + lit_sum_base_price: 0, + lit_length_sum_base_price: 0, + match_length_sum_base_price: 0, + off_code_sum_base_price: 0, + price_type: OptPrice::Dynamic, + literal_compression_mode: ParamSwitch::Auto, + } + } +} + +pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; + +/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures +/// own, are in [`BlockSrc`]. +pub(crate) struct MatchState { + pub(crate) window: Window, + /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. + pub(crate) loaded_dict_end: u32, + /// The index from which the tables are still to be filled. + pub(crate) next_to_update: u32, + /// 0 if there is no table of 3-byte hashes. + pub(crate) hash_log3: u32, + pub(crate) row_hash_log: u32, + /// For the row-based match finder: a byte of hash for each entry of `hash_table`. + pub(crate) tag_table: Vec, + pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], + /// Makes what a previous frame left in `tag_table` meaningless. + pub(crate) hash_salt: u64, + pub(crate) hash_salt_entropy: u32, + pub(crate) hash_table: Vec, + pub(crate) hash_table3: Vec, + pub(crate) chain_table: Vec, + pub(crate) force_non_contiguous: bool, + pub(crate) opt: OptState, + /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. + pub(crate) has_dict_match_state: bool, + pub(crate) c_params: CompressionParameters, + pub(crate) lazy_skipping: bool, +} + +impl MatchState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + loaded_dict_end: 0, + next_to_update: 0, + hash_log3: 0, + row_hash_log: 0, + tag_table: Vec::new(), + hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + hash_salt: 0, + hash_salt_entropy: 0, + hash_table: Vec::new(), + hash_table3: Vec::new(), + chain_table: Vec::new(), + force_non_contiguous: false, + opt: OptState::new(), + has_dict_match_state: false, + c_params: CompressionParameters { + window_log: 0, + chain_log: 0, + hash_log: 0, + search_log: 0, + min_match: 0, + target_length: 0, + strategy: None, + }, + lazy_skipping: false, + } + } + + /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. + pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { + if self.window.has_ext_dict() { + return DictMode::ExtDict; + } + if dms.is_some() { + DictMode::DictMatchState + } else { + DictMode::NoDict + } + } + + /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. + #[inline(always)] + pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.low_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + // A dictionary is valid as a whole until it is invalidated as a whole. + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } + + /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. + #[inline(always)] + pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.dict_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } +} + +/// The match state of a dictionary, and where its indices lead. +#[derive(Clone, Copy)] +pub(crate) struct DictMatchState<'a> { + pub(crate) ms: &'a MatchState, + /// The buffer its window is in, which is one segment. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// `window.nextSrc`, as a position in `buf`. + pub(crate) end: usize, +} + +/// The current segment of a window, for the functions that index a dictionary. +#[derive(Clone, Copy)] +pub(crate) struct Prefix<'a> { + /// The buffer that the segment is in. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, +} + +impl Prefix<'_> { + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. +#[derive(Clone, Copy)] +pub(crate) struct BlockSrc<'a> { + /// The buffer that the current segment of the window is in, and so the block. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// The buffer that the older segment of the window is in. + pub(crate) dict_buf: &'a [u8], + /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. + pub(crate) dict_base: usize, + /// Where the block starts in `buf`. + pub(crate) start: usize, + /// Where it ends. + pub(crate) end: usize, + pub(crate) dms: Option>, + /// `opt.symbolCosts`: the tables of the previous block. + pub(crate) symbol_costs: &'a EntropyCTables, + /// `ldmSeqStore`: matches that the optimal parser is to consider. + pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, +} + +impl<'a> BlockSrc<'a> { + #[inline(always)] + pub(crate) const fn prefix(&self) -> Prefix<'a> { + Prefix { + buf: self.buf, + base: self.base, + } + } + + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`, which is in the current segment. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } + + /// The position in `dict_buf` of `index`, which is in the older segment. + #[inline(always)] + pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { + self.dict_base.wrapping_add(index as usize) + } +} + +impl DictMatchState<'_> { + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +// ── long distance matching ──────────────────────────────────────────────── + +/// `ldmEntry_t` +#[derive(Clone, Copy, Default)] +pub(crate) struct LdmEntry { + pub(crate) offset: u32, + pub(crate) checksum: u32, +} + +/// `ldmParams_t` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct LdmParams { + pub(crate) enable_ldm: ParamSwitch, + pub(crate) hash_log: u32, + pub(crate) bucket_size_log: u32, + pub(crate) min_match_length: u32, + pub(crate) hash_rate_log: u32, + pub(crate) window_log: u32, +} + +/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. +pub(crate) struct LdmState { + pub(crate) window: Window, + pub(crate) hash_table: Vec, + pub(crate) loaded_dict_end: u32, + /// The next entry to replace in each bucket. + pub(crate) bucket_offsets: Vec, +} + +impl LdmState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + hash_table: Vec::new(), + loaded_dict_end: 0, + bucket_offsets: Vec::new(), + } + } +} diff --git a/src/zstd/wip/work/opt/zstd/compress/lazy.rs b/src/zstd/wip/work/opt/zstd/compress/lazy.rs new file mode 100644 index 000000000000..9f653fd684f3 --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/compress/lazy.rs @@ -0,0 +1,34 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; +use crate::common::ZSTD_REP_NUM; +use crate::error::{ErrorCode, Result}; + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. Returns the index of the first candidate for `ip`. +pub(crate) fn insert_and_find_first_index( + _ms: &mut MatchState, + _prefix: Prefix<'_>, + _ip: usize, +) -> Result { + Err(ErrorCode::Generic) +} + +/// `ZSTD_row_update`: the same, in the rows of the row-based match finder. +pub(crate) fn row_update(_ms: &mut MatchState, _prefix: Prefix<'_>, _ip: usize) -> Result<()> { + Err(ErrorCode::Generic) +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + _ms: &mut MatchState, + _seq_store: &mut SeqStore, + _rep: &mut [u32; ZSTD_REP_NUM], + _src: &BlockSrc<'_>, + _strategy: Strategy, + _use_row: bool, + _dict_mode: DictMode, +) -> Result { + Err(ErrorCode::Generic) +} diff --git a/src/zstd/wip/work/opt/zstd/compress/ldm.rs b/src/zstd/wip/work/opt/zstd/compress/ldm.rs new file mode 100644 index 000000000000..2177fd945e10 --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/compress/ldm.rs @@ -0,0 +1 @@ +//! `zstd_ldm.c`: long distance matching. diff --git a/src/zstd/wip/work/opt/zstd/compress/mod.rs b/src/zstd/wip/work/opt/zstd/compress/mod.rs new file mode 100644 index 000000000000..13ddecdf82e4 --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/compress/mod.rs @@ -0,0 +1,68 @@ +//! `lib/compress` + +pub(crate) mod double_fast; +pub(crate) mod fast; +pub(crate) mod fse; +pub(crate) mod hist; +pub(crate) mod huf; +pub(crate) mod internal; +pub(crate) mod lazy; +pub(crate) mod ldm; +pub(crate) mod opt; +pub(crate) mod presplit; + +use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; + +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; + +/// `ZSTD_rowMatchFinderSupported` +pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { + strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 +} + +/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to +/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies +/// that support it look at. Returns how many bytes at the end of the block are in no sequence. +pub(crate) fn compress_block( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row_match_finder: bool, + dict_mode: DictMode, +) -> Result { + match strategy { + Strategy::Fast => match dict_mode { + DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), + DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), + DictMode::DictMatchState => { + fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::DFast => match dict_mode { + DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), + DictMode::ExtDict => { + double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { + lazy::compress_block_lazy( + ms, + seq_store, + rep, + src, + strategy, + use_row_match_finder && row_match_finder_supported(strategy), + dict_mode, + ) + } + Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { + opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) + } + } +} diff --git a/src/zstd/wip/work/opt/zstd/compress/opt.rs b/src/zstd/wip/work/opt/zstd/compress/opt.rs new file mode 100644 index 000000000000..a7a6022a3a45 --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/compress/opt.rs @@ -0,0 +1,1721 @@ +//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. + +use core::mem; + +use super::hist; +use super::huf::HufRepeat; +use super::internal::{ + BlockSrc, DictMode, EntropyCTables, Match, MatchState, OptPrice, OptState, Optimal, + ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, ZSTD_OPT_NUM, count, + count_2segments, hash_ptr, hash3_ptr, index_overlap_check, ll_code, ml_code, new_rep, + offset_to_offbase, +}; +use crate::common::{ + LL_BITS, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, +}; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +/// How much a literal adds to its frequency, so that frequencies adapt fast to new statistics. +const ZSTD_LITFREQ_ADD: u32 = 2; +const ZSTD_MAX_PRICE: i32 = 1 << 30; +/// A first block of at most this size is priced by fixed rules, not by statistics. +const ZSTD_PREDEF_THRESHOLD: usize = 8; + +const MAX_LIT: u32 = 255; + +// ── prices ──────────────────────────────────────────────────────────────── + +/// Prices are in units of `1 / BITCOST_MULTIPLIER` of a bit. +const BITCOST_ACCURACY: u32 = 8; +const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY; + +/// `WEIGHT`: approximate for `btopt`, accurate for `btultra`. +#[inline(always)] +const fn weight(stat: u32, opt_level: i32) -> u32 { + if opt_level != 0 { + frac_weight(stat) + } else { + bit_weight(stat) + } +} + +/// `ZSTD_bitWeight`: the cost of a statistic, in whole bits. +#[inline(always)] +const fn bit_weight(stat: u32) -> u32 { + highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER +} + +/// `ZSTD_fracWeight`: the cost of a statistic, with a fraction of a bit that is interpolated linearly. +#[inline(always)] +const fn frac_weight(raw_stat: u32) -> u32 { + let stat = raw_stat.wrapping_add(1); + let hb = highbit32(stat); + let b_weight = hb * BITCOST_MULTIPLIER; + // From 1 to 2, in fixed point. + let f_weight = (stat << BITCOST_ACCURACY) >> hb; + b_weight.wrapping_add(f_weight) +} + +/// `ZSTD_compressedLiterals` +#[inline(always)] +fn compressed_literals(opt_ptr: &OptState) -> bool { + opt_ptr.literal_compression_mode != ParamSwitch::Disable +} + +/// `ZSTD_setBasePrices` +#[inline(always)] +fn set_base_prices(opt_ptr: &mut OptState, opt_level: i32) { + if compressed_literals(opt_ptr) { + opt_ptr.lit_sum_base_price = weight(opt_ptr.lit_sum, opt_level); + } + opt_ptr.lit_length_sum_base_price = weight(opt_ptr.lit_length_sum, opt_level); + opt_ptr.match_length_sum_base_price = weight(opt_ptr.match_length_sum, opt_level); + opt_ptr.off_code_sum_base_price = weight(opt_ptr.off_code_sum, opt_level); +} + +/// `sum_u32` +fn sum_u32(table: &[u32]) -> u32 { + table.iter().fold(0, |total, &elt| total.wrapping_add(elt)) +} + +/// `base_directive_e`: the least a scaled down statistic can be. +#[derive(Clone, Copy, PartialEq, Eq)] +enum BaseDirective { + /// 0, if it is 0 already. + Base0Possible, + Base1Guaranteed, +} + +/// `ZSTD_downscaleStats`: returns the new sum of the elements. +fn downscale_stats( + table: &mut [u32], + last_elt_index: u32, + shift: u32, + base1: BaseDirective, +) -> Result { + let mut sum = 0u32; + for elt in table.sub_mut(..=last_elt_index as usize)? { + let base = u32::from(base1 == BaseDirective::Base1Guaranteed || *elt > 0); + let new_stat = base.wrapping_add(*elt >> (shift & 31)); + sum = sum.wrapping_add(new_stat); + *elt = new_stat; + } + Ok(sum) +} + +/// `ZSTD_scaleStats`: scales all the elements down if their sum is too large. Returns the sum that results. +fn scale_stats(table: &mut [u32], last_elt_index: u32, log_target: u32) -> Result { + let prevsum = sum_u32(table.sub(..=last_elt_index as usize)?); + let factor = prevsum >> log_target; + if factor <= 1 { + return Ok(prevsum); + } + downscale_stats( + table, + last_elt_index, + highbit32(factor), + BaseDirective::Base1Guaranteed, + ) +} + +/// Frequencies, out of about `1 << scale_log`, that make each symbol cost the bits `bit_cost` tells. Returns their sum. +fn freqs_from_bit_costs( + freqs: &mut [u32], + last_elt_index: u32, + scale_log: u32, + bit_cost: impl Fn(u32) -> u32, +) -> Result { + let mut sum = 0u32; + for (symbol, freq) in (0u32..).zip(freqs.sub_mut(..=last_elt_index as usize)?) { + let bit_cost = bit_cost(symbol); + // At least 1, to have a cost. + *freq = if bit_cost != 0 { + 1 << (scale_log.wrapping_sub(bit_cost) & 31) + } else { + 1 + }; + sum = sum.wrapping_add(*freq); + } + Ok(sum) +} + +/// `ZSTD_rescaleFreqs`: for the first block, which no statistics tell, starts them from the tables of the dictionary +/// if there is one, else from the literals of `src` and from fixed values for the sequences. For the next ones, scales +/// down what the blocks before have accumulated. +fn rescale_freqs( + opt_ptr: &mut OptState, + symbol_costs: &EntropyCTables, + src: &[u8], + opt_level: i32, +) -> Result<()> { + #[rustfmt::skip] + const BASE_LL_FREQS: [u32; MAX_LL as usize + 1] = [ + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, + ]; + #[rustfmt::skip] + const BASE_OFC_FREQS: [u32; MAX_OFF as usize + 1] = [ + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + ]; + + let compressed_literals = compressed_literals(opt_ptr); + opt_ptr.price_type = OptPrice::Dynamic; + + if opt_ptr.lit_length_sum == 0 { + if src.len() <= ZSTD_PREDEF_THRESHOLD { + opt_ptr.price_type = OptPrice::Predef; + } + + if symbol_costs.huf.repeat_mode == HufRepeat::Valid { + // A Huffman table that has all the values is presumed to come from a dictionary. + opt_ptr.price_type = OptPrice::Dynamic; + + if compressed_literals { + let huf = &symbol_costs.huf.c_table; + opt_ptr.lit_sum = + freqs_from_bit_costs(&mut opt_ptr.lit_freq, MAX_LIT, 11, |lit| { + huf.get_nb_bits(lit) + })?; + } + let fse = &symbol_costs.fse; + opt_ptr.lit_length_sum = + freqs_from_bit_costs(&mut opt_ptr.lit_length_freq, MAX_LL, 10, |ll| { + fse.litlength_c_table.get_max_nb_bits(ll) + })?; + opt_ptr.match_length_sum = + freqs_from_bit_costs(&mut opt_ptr.match_length_freq, MAX_ML, 10, |ml| { + fse.matchlength_c_table.get_max_nb_bits(ml) + })?; + opt_ptr.off_code_sum = + freqs_from_bit_costs(&mut opt_ptr.off_code_freq, MAX_OFF, 10, |of| { + fse.offcode_c_table.get_max_nb_bits(of) + })?; + } else { + if compressed_literals { + let mut lit = MAX_LIT; + hist::count_simple(&mut opt_ptr.lit_freq, &mut lit, src)?; + opt_ptr.lit_sum = downscale_stats( + &mut opt_ptr.lit_freq, + MAX_LIT, + 8, + BaseDirective::Base0Possible, + )?; + } + + opt_ptr.lit_length_freq.write_at(0, &BASE_LL_FREQS)?; + opt_ptr.lit_length_sum = sum_u32(&BASE_LL_FREQS); + + opt_ptr + .match_length_freq + .sub_mut(..=MAX_ML as usize)? + .fill(1); + opt_ptr.match_length_sum = MAX_ML + 1; + + opt_ptr.off_code_freq.write_at(0, &BASE_OFC_FREQS)?; + opt_ptr.off_code_sum = sum_u32(&BASE_OFC_FREQS); + } + } else { + if compressed_literals { + opt_ptr.lit_sum = scale_stats(&mut opt_ptr.lit_freq, MAX_LIT, 12)?; + } + opt_ptr.lit_length_sum = scale_stats(&mut opt_ptr.lit_length_freq, MAX_LL, 11)?; + opt_ptr.match_length_sum = scale_stats(&mut opt_ptr.match_length_freq, MAX_ML, 11)?; + opt_ptr.off_code_sum = scale_stats(&mut opt_ptr.off_code_freq, MAX_OFF, 11)?; + } + + set_base_prices(opt_ptr, opt_level); + Ok(()) +} + +/// `ZSTD_rawLiteralsCost`: the price of `literals`, without that of their length. +#[inline(always)] +fn raw_literals_cost(literals: &[u8], opt_ptr: &OptState, opt_level: i32) -> Result { + let lit_length = literals.len() as u32; + if lit_length == 0 { + return Ok(0); + } + + if !compressed_literals(opt_ptr) { + // 8 bits per literal. + return Ok((lit_length << 3).wrapping_mul(BITCOST_MULTIPLIER)); + } + + if opt_ptr.price_type == OptPrice::Predef { + // 6 bits per literal. + return Ok(lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER)); + } + + let mut price = opt_ptr.lit_sum_base_price.wrapping_mul(lit_length); + let lit_price_max = opt_ptr.lit_sum_base_price.wrapping_sub(BITCOST_MULTIPLIER); + for &literal in literals { + let lit_price = weight(opt_ptr.lit_freq.at(usize::from(literal))?, opt_level); + price = price.wrapping_sub(lit_price.min(lit_price_max)); + } + Ok(price) +} + +/// `ZSTD_litLengthPrice`: the price of the symbol for a length of literals. +#[inline(always)] +fn lit_length_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + if opt_ptr.price_type == OptPrice::Predef { + return Ok(weight(lit_length, opt_level)); + } + + // The format has no code for the length of a whole block of the largest size, which is then all literals. It is + // made to cost 1 bit more than the length before it. + let (lit_length, extra) = if lit_length == ZSTD_BLOCKSIZE_MAX as u32 { + (ZSTD_BLOCKSIZE_MAX as u32 - 1, BITCOST_MULTIPLIER) + } else { + (lit_length, 0) + }; + + let ll_code = ll_code(lit_length) as usize; + Ok(extra + .wrapping_add(u32::from(LL_BITS.at(ll_code)?) * BITCOST_MULTIPLIER) + .wrapping_add(opt_ptr.lit_length_sum_base_price) + .wrapping_sub(weight(opt_ptr.lit_length_freq.at(ll_code)?, opt_level))) +} + +/// `ZSTD_getMatchPrice`: the price of the offset and of the match length of a sequence. `off_base` is as in +/// [`SeqStore::store_seq`]. Below level 2, it favours small offsets, which are faster to decompress. +#[inline(always)] +fn get_match_price( + off_base: u32, + match_length: u32, + opt_ptr: &OptState, + opt_level: i32, +) -> Result { + let off_code = highbit32(off_base); + let ml_base = match_length.wrapping_sub(MINMATCH); + + if opt_ptr.price_type == OptPrice::Predef { + return Ok(weight(ml_base, opt_level).wrapping_add((16 + off_code) * BITCOST_MULTIPLIER)); + } + + let off_code_weight = weight(opt_ptr.off_code_freq.at(off_code as usize)?, opt_level); + let mut price = (off_code * BITCOST_MULTIPLIER).wrapping_add( + opt_ptr + .off_code_sum_base_price + .wrapping_sub(off_code_weight), + ); + if opt_level < 2 && off_code >= 20 { + price = price.wrapping_add((off_code - 19) * 2 * BITCOST_MULTIPLIER); + } + + let ml_code = ml_code(ml_base) as usize; + let ml_code_weight = weight(opt_ptr.match_length_freq.at(ml_code)?, opt_level); + price = price + .wrapping_add(u32::from(ML_BITS.at(ml_code)?) * BITCOST_MULTIPLIER) + .wrapping_add( + opt_ptr + .match_length_sum_base_price + .wrapping_sub(ml_code_weight), + ); + + // Matches cost a bit more, to have fewer sequences, which is faster to decompress. + Ok(price.wrapping_add(BITCOST_MULTIPLIER / 5)) +} + +/// `ZSTD_updateStats` +#[inline(always)] +fn update_stats( + opt_ptr: &mut OptState, + literals: &[u8], + off_base: u32, + match_length: u32, +) -> Result<()> { + #[inline(always)] + fn increment(freqs: &mut [u32], code: u32, sum: &mut u32) -> Result<()> { + let freq = freqs.at_mut(code as usize)?; + *freq = freq.wrapping_add(1); + *sum = sum.wrapping_add(1); + Ok(()) + } + + let lit_length = literals.len() as u32; + if compressed_literals(opt_ptr) { + for &literal in literals { + let freq = opt_ptr.lit_freq.at_mut(usize::from(literal))?; + *freq = freq.wrapping_add(ZSTD_LITFREQ_ADD); + } + opt_ptr.lit_sum = opt_ptr + .lit_sum + .wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD)); + } + + increment( + &mut opt_ptr.lit_length_freq, + ll_code(lit_length), + &mut opt_ptr.lit_length_sum, + )?; + increment( + &mut opt_ptr.off_code_freq, + highbit32(off_base), + &mut opt_ptr.off_code_sum, + )?; + increment( + &mut opt_ptr.match_length_freq, + ml_code(match_length.wrapping_sub(MINMATCH)), + &mut opt_ptr.match_length_sum, + ) +} + +/// `ZSTD_readMINMATCH`: `length` bytes, 3 or 4, to compare with others. Reads 4. +#[inline(always)] +fn read_minmatch(buf: &[u8], pos: usize, length: u32) -> Result { + let value = read_le32(buf, pos)?; + Ok(if length == 3 { value << 8 } else { value }) +} + +/// `ZSTD_insertAndFindFirstIndexHash3`: indexes the 3-byte hashes up to `ip`, which is not included, and looks it up. +/// All of it is in the current segment. +fn insert_and_find_first_index_hash3( + ms: &mut MatchState, + next_to_update3: &mut u32, + prefix: Prefix<'_>, + ip: usize, +) -> Result { + let hash_table3 = ms.hash_table3.as_mut_slice(); + let hash_log3 = ms.hash_log3; + let target = prefix.index_of(ip); + let hash3 = hash3_ptr(prefix.buf, ip, hash_log3)?; + + for idx in *next_to_update3..target { + hash_table3.put(hash3_ptr(prefix.buf, prefix.pos_of(idx), hash_log3)?, idx)?; + } + + *next_to_update3 = target; + hash_table3.at(hash3) +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// `*ptr = value`, where `ptr` is an element of `bt`, or `None` for a place that nothing reads. +#[inline(always)] +fn store(bt: &mut [u32], ptr: Option, value: u32) -> Result<()> { + match ptr { + Some(ptr) => bt.put(ptr, value), + None => Ok(()), + } +} + +/// `ZSTD_insertBt1`: adds the position `ip`, which is at most `iend - 8`, to the tree. `target` is up to where the +/// caller is filling it, and `dict` the older segment, which is looked at if `ext_dict`. Returns how many positions +/// to move forward. +#[inline(always)] +fn insert_bt1( + ms: &mut MatchState, + prefix: Prefix<'_>, + dict: Prefix<'_>, + ip: usize, + iend: usize, + target: u32, + mls: u32, + ext_dict: bool, +) -> Result { + let c_params = ms.c_params; + let h = hash_ptr(prefix.buf, ip, c_params.hash_log, mls)?; + let bt_log = c_params.chain_log.wrapping_sub(1); + let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = dict.pos_of(dict_limit); + let prefix_start = prefix.pos_of(dict_limit); + let curr = prefix.index_of(ip); + let bt_low = curr.saturating_sub(bt_mask); + let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); + let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); + // Only the positions that are still in the window once the tree is filled are of use. + let window_low = ms.get_lowest_match_index(target, c_params.window_log); + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut best_length = 8usize; + let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); + + let bt = ms.chain_table.as_mut_slice(); + let hash_entry = ms.hash_table.at_mut(h)?; + let mut match_index = *hash_entry; + *hash_entry = curr; + + while nb_compares != 0 && match_index >= window_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + // What is sure to be common. + let mut match_length = common_length_smaller.min(common_length_larger); + + // A position of the older segment, as one of the current segment, only makes sense with a length that + // brings it there. + let mut match_buf = prefix.buf; + let mut match_pos = prefix.pos_of(match_index); + if !ext_dict || match_index as usize + match_length >= dict_limit as usize { + match_length += count( + prefix.buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + iend, + )?; + } else { + match_length += count_2segments( + prefix.buf, + ip + match_length, + dict.buf, + dict.pos_of(match_index) + match_length, + iend, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + match_buf = dict.buf; + match_pos = dict.pos_of(match_index); + } + } + + if match_length > best_length { + best_length = match_length; + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + } + + if ip + match_length == iend { + // They are equal: there is no telling which is smaller. Dropping it keeps the tree consistent. + break; + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < prefix.buf.at(ip + match_length)? { + // All the smaller ones will have at least this much in common. + store(bt, smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. + smaller_ptr = None; + break; + } + smaller_ptr = Some(next_ptr + 1); + match_index = bt.at(next_ptr + 1)?; + } else { + store(bt, larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = None; + break; + } + larger_ptr = Some(next_ptr); + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + store(bt, smaller_ptr, 0)?; + store(bt, larger_ptr, 0)?; + + let positions = if best_length > 384 { + 192.min((best_length - 384) as u32) + } else { + 0 + }; + Ok(positions.max(match_end_idx.wrapping_sub(curr.wrapping_add(8)))) +} + +/// `ZSTD_updateTree_internal` +#[inline(always)] +fn update_tree_internal( + ms: &mut MatchState, + prefix: Prefix<'_>, + dict: Prefix<'_>, + ip: usize, + iend: usize, + mls: u32, + dict_mode: DictMode, +) -> Result<()> { + let target = prefix.index_of(ip); + let mut idx = ms.next_to_update; + + while idx < target { + let forward = insert_bt1( + ms, + prefix, + dict, + prefix.pos_of(idx), + iend, + target, + mls, + dict_mode == DictMode::ExtDict, + )?; + idx = idx.wrapping_add(forward); + } + ms.next_to_update = target; + Ok(()) +} + +/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. +/// Matches are not compared past the position `iend`. +pub(crate) fn update_tree( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + iend: usize, +) -> Result<()> { + let no_dict = Prefix { buf: &[], base: 0 }; + let mls = ms.c_params.min_match; + update_tree_internal(ms, prefix, no_dict, ip, iend, mls, DictMode::NoDict) +} + +/// `ZSTD_insertBtAndGetAllMatches`: adds the position `ip` to the tree, and stores in `matches` the matches found on +/// the way, each longer than the one before and all at least `length_to_beat` long. `ll0` is whether no literals +/// come before `ip`. Returns how many there are. +#[inline(always)] +fn insert_bt_and_get_all_matches( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_limit: usize, + dict_mode: DictMode, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, + mls: u32, +) -> Result { + let buf = src.buf; + let c_params = ms.c_params; + let sufficient_len = c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); + let curr = src.index_of(ip); + let min_match = if mls == 3 { 3 } else { 4 }; + let h = hash_ptr(buf, ip, c_params.hash_log, mls)?; + let bt_log = c_params.chain_log.wrapping_sub(1); + let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let prefix_start = src.pos_of(dict_limit); + let bt_low = curr.saturating_sub(bt_mask); + let window_low = ms.get_lowest_match_index(curr, c_params.window_log); + let match_low = window_low.max(1); + let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); + let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); + // The farthest position any match refers to, which detects repetitive patterns. + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut mnum = 0u32; + let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); + + let dms = if dict_mode == DictMode::DictMatchState { + Some(src.dms.ok_or(BUG)?) + } else { + None + }; + let dms_high_limit = dms.map_or(0, |dms| dms.end.wrapping_sub(dms.base) as u32); + let dms_low_limit = dms.map_or(0, |dms| dms.ms.window.low_limit); + let dms_index_delta = dms.map_or(0, |_| window_low.wrapping_sub(dms_high_limit)); + + let mut best_length = length_to_beat.wrapping_sub(1) as usize; + + // The repeat offsets. + let ll0 = u32::from(ll0); + let [rep0, _, _] = *rep; + for rep_code in ll0..ZSTD_REP_NUM as u32 + ll0 { + let rep_offset = match rep.get(rep_code as usize) { + Some(&offset) => offset, + None => rep0.wrapping_sub(1), + }; + let rep_index = curr.wrapping_sub(rep_offset); + let mut rep_len = 0u32; + // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. + if rep_offset.wrapping_sub(1) < curr.wrapping_sub(dict_limit) { + // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. + let rep_match = ip.wrapping_sub(rep_offset as usize); + if rep_index >= window_low + && read_minmatch(buf, ip, min_match)? == read_minmatch(buf, rep_match, min_match)? + { + rep_len = count( + buf, + ip + min_match as usize, + buf, + rep_match + min_match as usize, + i_limit, + )? as u32 + + min_match; + } + } else if dict_mode == DictMode::ExtDict { + // `curr > rep_index >= window_low` + if rep_offset.wrapping_sub(1) < curr.wrapping_sub(window_low) + && index_overlap_check(dict_limit, rep_index) + { + let rep_match = src.dict_pos_of(rep_index); + if read_minmatch(buf, ip, min_match)? + == read_minmatch(src.dict_buf, rep_match, min_match)? + { + rep_len = count_2segments( + buf, + ip + min_match as usize, + src.dict_buf, + rep_match + min_match as usize, + i_limit, + dict_end, + prefix_start, + )? as u32 + + min_match; + } + } + } else if let Some(dms) = dms { + // `curr > rep_index >= dms_low_limit` + if rep_offset.wrapping_sub(1) + < curr.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta)) + && index_overlap_check(dict_limit, rep_index) + { + let rep_match = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); + if read_minmatch(buf, ip, min_match)? + == read_minmatch(dms.buf, rep_match, min_match)? + { + rep_len = count_2segments( + buf, + ip + min_match as usize, + dms.buf, + rep_match + min_match as usize, + i_limit, + dms.end, + prefix_start, + )? as u32 + + min_match; + } + } + } + if rep_len as usize > best_length { + best_length = rep_len as usize; + matches.put( + mnum as usize, + Match { + off: rep_code - ll0 + 1, + len: rep_len, + }, + )?; + mnum += 1; + if rep_len > sufficient_len || ip + rep_len as usize == i_limit { + // The best there can be. + return Ok(mnum); + } + } + } + + // The table of 3-byte hashes. That of a dictionary is not filled. + if mls == 3 && best_length < mls as usize { + let match_index3 = + insert_and_find_first_index_hash3(ms, next_to_update3, src.prefix(), ip)?; + // Farther than that is likely to cost too much. + if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) { + let mlen = if dict_mode != DictMode::ExtDict || match_index3 >= dict_limit { + count(buf, ip, buf, src.pos_of(match_index3), i_limit)? + } else { + count_2segments( + buf, + ip, + src.dict_buf, + src.dict_pos_of(match_index3), + i_limit, + dict_end, + prefix_start, + )? + }; + + if mlen >= mls as usize { + best_length = mlen; + matches.put( + 0, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index3)), + len: mlen as u32, + }, + )?; + mnum = 1; + if mlen > sufficient_len as usize || ip + mlen == i_limit { + // The best there can be: `ip` is not inserted. + ms.next_to_update = curr.wrapping_add(1); + return Ok(1); + } + } + } + } + + let bt = ms.chain_table.as_mut_slice(); + let hash_entry = ms.hash_table.at_mut(h)?; + let mut match_index = *hash_entry; + *hash_entry = curr; + + while nb_compares != 0 && match_index >= match_low { + let next_ptr = 2 * (match_index & bt_mask) as usize; + // What is sure to be common. + let mut match_length = common_length_smaller.min(common_length_larger); + + // A position of the older segment, as one of the current segment, only makes sense with a length that + // brings it there. + let mut match_buf = buf; + let mut match_pos = src.pos_of(match_index); + if dict_mode != DictMode::ExtDict + || match_index as usize + match_length >= dict_limit as usize + { + match_length += count( + buf, + ip + match_length, + match_buf, + match_pos.wrapping_add(match_length), + i_limit, + )?; + } else { + match_length += count_2segments( + buf, + ip + match_length, + src.dict_buf, + src.dict_pos_of(match_index) + match_length, + i_limit, + dict_end, + prefix_start, + )?; + if (match_index as usize + match_length) < dict_limit as usize { + match_buf = src.dict_buf; + match_pos = src.dict_pos_of(match_index); + } + } + + if match_length > best_length { + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + best_length = match_length; + matches.put( + mnum as usize, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index)), + len: match_length as u32, + }, + )?; + mnum += 1; + if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { + // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. + if dict_mode == DictMode::DictMatchState { + // The dictionary is not to be searched either. + nb_compares = 0; + } + break; + } + } + + if match_buf.at(match_pos.wrapping_add(match_length))? < buf.at(ip + match_length)? { + // All the smaller ones will have at least this much in common. + store(bt, smaller_ptr, match_index)?; + common_length_smaller = match_length; + if match_index <= bt_low { + // Beyond the tree. + smaller_ptr = None; + break; + } + smaller_ptr = Some(next_ptr + 1); + match_index = bt.at(next_ptr + 1)?; + } else { + store(bt, larger_ptr, match_index)?; + common_length_larger = match_length; + if match_index <= bt_low { + larger_ptr = None; + break; + } + larger_ptr = Some(next_ptr); + match_index = bt.at(next_ptr)?; + } + nb_compares -= 1; + } + + store(bt, smaller_ptr, 0)?; + store(bt, larger_ptr, 0)?; + + if let Some(dms) = dms + && nb_compares != 0 + { + let dms_c_params = &dms.ms.c_params; + let dms_bt_log = dms_c_params.chain_log.wrapping_sub(1); + let dms_bt_mask = 1u32.wrapping_shl(dms_bt_log).wrapping_sub(1); + let dms_bt_low = if dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit) { + dms_high_limit.wrapping_sub(dms_bt_mask) + } else { + dms_low_limit + }; + let dms_h = hash_ptr(buf, ip, dms_c_params.hash_log, mls)?; + let mut dict_match_index = dms.ms.hash_table.at(dms_h)?; + let dms_bt = dms.ms.chain_table.as_slice(); + common_length_smaller = 0; + common_length_larger = 0; + while nb_compares != 0 && dict_match_index > dms_low_limit { + let next_ptr = 2 * (dict_match_index & dms_bt_mask) as usize; + let mut match_length = common_length_smaller.min(common_length_larger); + match_length += count_2segments( + buf, + ip + match_length, + dms.buf, + dms.pos_of(dict_match_index) + match_length, + i_limit, + dms.end, + prefix_start, + )?; + + if match_length > best_length { + match_index = dict_match_index.wrapping_add(dms_index_delta); + if match_length > match_end_idx.wrapping_sub(match_index) as usize { + match_end_idx = match_index.wrapping_add(match_length as u32); + } + best_length = match_length; + matches.put( + mnum as usize, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index)), + len: match_length as u32, + }, + )?; + mnum += 1; + if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { + break; + } + } + + if dict_match_index <= dms_bt_low { + // Beyond the tree. + break; + } + // A match that reaches the end of the dictionary goes on at the start of the current segment. + let match_byte = if dict_match_index as usize + match_length >= dms_high_limit as usize + { + let match_pos = src.pos_of(dict_match_index.wrapping_add(dms_index_delta)); + buf.at(match_pos.wrapping_add(match_length))? + } else { + dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + }; + if match_byte < buf.at(ip + match_length)? { + common_length_smaller = match_length; + dict_match_index = dms_bt.at(next_ptr + 1)?; + } else { + common_length_larger = match_length; + dict_match_index = dms_bt.at(next_ptr)?; + } + nb_compares -= 1; + } + } + + // Repetitive patterns are skipped. + ms.next_to_update = match_end_idx.wrapping_sub(8); + Ok(mnum) +} + +/// `ZSTD_getAllMatchesFn`: the arguments are `matches`, `ms`, `next_to_update3`, `src`, `ip`, `i_high_limit`, `rep`, +/// `ll0` and `length_to_beat`. +type GetAllMatchesFn = fn( + &mut [Match], + &mut MatchState, + &mut u32, + &BlockSrc<'_>, + usize, + usize, + &[u32; ZSTD_REP_NUM], + bool, + u32, +) -> Result; + +/// `ZSTD_btGetAllMatches_internal` +#[inline(always)] +fn bt_get_all_matches_internal( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, + dict_mode: DictMode, + mls: u32, +) -> Result { + if src.index_of(ip) < ms.next_to_update { + // An area that is skipped. + return Ok(0); + } + let dict = Prefix { + buf: src.dict_buf, + base: src.dict_base, + }; + update_tree_internal(ms, src.prefix(), dict, ip, i_high_limit, mls, dict_mode)?; + insert_bt_and_get_all_matches( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + dict_mode, + rep, + ll0, + length_to_beat, + mls, + ) +} + +/// `ZSTD_btGetAllMatches_noDict_3` to `ZSTD_btGetAllMatches_noDict_6` +fn bt_get_all_matches_no_dict( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::NoDict, + MLS, + ) +} + +/// `ZSTD_btGetAllMatches_extDict_3` to `ZSTD_btGetAllMatches_extDict_6` +fn bt_get_all_matches_ext_dict( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::ExtDict, + MLS, + ) +} + +/// `ZSTD_btGetAllMatches_dictMatchState_3` to `ZSTD_btGetAllMatches_dictMatchState_6` +fn bt_get_all_matches_dict_match_state( + matches: &mut [Match], + ms: &mut MatchState, + next_to_update3: &mut u32, + src: &BlockSrc<'_>, + ip: usize, + i_high_limit: usize, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + length_to_beat: u32, +) -> Result { + bt_get_all_matches_internal( + matches, + ms, + next_to_update3, + src, + ip, + i_high_limit, + rep, + ll0, + length_to_beat, + DictMode::DictMatchState, + MLS, + ) +} + +/// `ZSTD_selectBtGetAllMatches` +fn select_bt_get_all_matches(ms: &MatchState, dict_mode: DictMode) -> GetAllMatchesFn { + match (dict_mode, ms.c_params.min_match) { + (DictMode::NoDict, ..=3) => bt_get_all_matches_no_dict::<3>, + (DictMode::NoDict, 4) => bt_get_all_matches_no_dict::<4>, + (DictMode::NoDict, 5) => bt_get_all_matches_no_dict::<5>, + (DictMode::NoDict, 6..) => bt_get_all_matches_no_dict::<6>, + (DictMode::ExtDict, ..=3) => bt_get_all_matches_ext_dict::<3>, + (DictMode::ExtDict, 4) => bt_get_all_matches_ext_dict::<4>, + (DictMode::ExtDict, 5) => bt_get_all_matches_ext_dict::<5>, + (DictMode::ExtDict, 6..) => bt_get_all_matches_ext_dict::<6>, + (DictMode::DictMatchState, ..=3) => bt_get_all_matches_dict_match_state::<3>, + (DictMode::DictMatchState, 4) => bt_get_all_matches_dict_match_state::<4>, + (DictMode::DictMatchState, 5) => bt_get_all_matches_dict_match_state::<5>, + (DictMode::DictMatchState, 6..) => bt_get_all_matches_dict_match_state::<6>, + } +} + +// ── candidates from long distance matching ──────────────────────────────── + +/// `ZSTD_optLdm_t`: what it takes to decide whether to use a long distance match. +struct OptLdm<'a> { + /// `seqStore.seq`: the `seqStore.size` candidates for this block and the next ones. + seq: &'a [RawSeq], + /// `seqStore.pos` + pos: usize, + /// `seqStore.posInSequence` + pos_in_sequence: usize, + /// Where the current candidate starts. + start_pos_in_block: u32, + /// Where it ends. + end_pos_in_block: u32, + offset: u32, +} + +impl<'a> OptLdm<'a> { + fn new(seq_store: Option<&'a RawSeqStore>) -> Self { + Self { + seq: seq_store + .and_then(|store| store.seq.get(..store.size)) + .unwrap_or_default(), + pos: seq_store.map_or(0, |store| store.pos), + pos_in_sequence: seq_store.map_or(0, |store| store.pos_in_sequence), + start_pos_in_block: 0, + end_pos_in_block: 0, + offset: 0, + } + } + + /// `ZSTD_optLdm_skipRawSeqStoreBytes`: moves forward by `nb_bytes`. + fn skip_raw_seq_store_bytes(&mut self, nb_bytes: usize) { + let mut curr_pos = self.pos_in_sequence.wrapping_add(nb_bytes) as u32; + while curr_pos != 0 { + let Some(curr_seq) = self.seq.get(self.pos) else { + break; + }; + let seq_length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); + if curr_pos >= seq_length { + curr_pos -= seq_length; + self.pos += 1; + } else { + self.pos_in_sequence = curr_pos as usize; + break; + } + } + if curr_pos == 0 || self.pos == self.seq.len() { + self.pos_in_sequence = 0; + } + } + + /// `ZSTD_opt_getNextMatchAndUpdateSeqStore`: finds where the next candidate starts and ends in the block, and + /// moves past it. + fn get_next_match_and_update_seq_store( + &mut self, + curr_pos_in_block: u32, + block_bytes_remaining: u32, + ) { + let Some(&curr_seq) = self.seq.get(self.pos) else { + // No position gets there, so that there is no candidate in this block. + self.start_pos_in_block = u32::MAX; + self.end_pos_in_block = u32::MAX; + return; + }; + // What is left of the literals and of the match. + let pos_in_sequence = self.pos_in_sequence as u32; + let curr_block_end_pos = curr_pos_in_block.wrapping_add(block_bytes_remaining); + let literals_bytes_remaining = curr_seq.lit_length.saturating_sub(pos_in_sequence); + let match_bytes_remaining = if literals_bytes_remaining == 0 { + curr_seq + .match_length + .wrapping_sub(pos_in_sequence.wrapping_sub(curr_seq.lit_length)) + } else { + curr_seq.match_length + }; + + if literals_bytes_remaining >= block_bytes_remaining { + // The match starts after the block. + self.start_pos_in_block = u32::MAX; + self.end_pos_in_block = u32::MAX; + self.skip_raw_seq_store_bytes(block_bytes_remaining as usize); + return; + } + + // What is left of a match can be too short to use, which is checked later. + self.start_pos_in_block = curr_pos_in_block.wrapping_add(literals_bytes_remaining); + self.end_pos_in_block = self.start_pos_in_block.wrapping_add(match_bytes_remaining); + self.offset = curr_seq.offset; + + if self.end_pos_in_block > curr_block_end_pos { + // The match ends after the block, which can only use its start. + self.end_pos_in_block = curr_block_end_pos; + self.skip_raw_seq_store_bytes( + curr_block_end_pos.wrapping_sub(curr_pos_in_block) as usize + ); + } else { + self.skip_raw_seq_store_bytes( + literals_bytes_remaining.wrapping_add(match_bytes_remaining) as usize, + ); + } + } + + /// `ZSTD_optLdm_maybeAddMatch`: adds what is left of the candidate at `curr_pos_in_block` to `matches`, if it is + /// long enough and longer than all of them, since they are in order of length. + fn maybe_add_match( + &self, + matches: &mut [Match], + nb_matches: &mut u32, + curr_pos_in_block: u32, + min_match: u32, + ) -> Result<()> { + let pos_diff = curr_pos_in_block.wrapping_sub(self.start_pos_in_block); + let candidate_match_length = self + .end_pos_in_block + .wrapping_sub(self.start_pos_in_block) + .wrapping_sub(pos_diff); + + if curr_pos_in_block < self.start_pos_in_block + || curr_pos_in_block >= self.end_pos_in_block + || candidate_match_length < min_match + { + return Ok(()); + } + + if *nb_matches == 0 + || (candidate_match_length > matches.at(*nb_matches as usize - 1)?.len + && (*nb_matches as usize) < ZSTD_OPT_NUM) + { + matches.put( + *nb_matches as usize, + Match { + off: offset_to_offbase(self.offset), + len: candidate_match_length, + }, + )?; + *nb_matches += 1; + } + Ok(()) + } + + /// `ZSTD_optLdm_processMatchCandidate` + #[inline(always)] + fn process_match_candidate( + &mut self, + matches: &mut [Match], + nb_matches: &mut u32, + curr_pos_in_block: u32, + remaining_bytes: u32, + min_match: u32, + ) -> Result<()> { + if self.pos >= self.seq.len() { + return Ok(()); + } + + if curr_pos_in_block >= self.end_pos_in_block { + if curr_pos_in_block > self.end_pos_in_block { + // The parser does not stop at the end of each candidate: it is often some bytes past it. + let pos_overshoot = curr_pos_in_block - self.end_pos_in_block; + self.skip_raw_seq_store_bytes(pos_overshoot as usize); + } + self.get_next_match_and_update_seq_store(curr_pos_in_block, remaining_bytes); + } + self.maybe_add_match(matches, nb_matches, curr_pos_in_block, min_match) + } +} + +// ── the optimal parser ──────────────────────────────────────────────────── + +/// `LIT_PRICE`: the price of the literal at `pos`. +#[inline(always)] +fn lit_price(buf: &[u8], pos: usize, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok(raw_literals_cost(buf.span(pos, 1)?, opt_ptr, opt_level)? as i32) +} + +/// `LL_PRICE` +#[inline(always)] +fn ll_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok(lit_length_price(lit_length, opt_ptr, opt_level)? as i32) +} + +/// `LL_INCPRICE`: what one more literal, which makes `lit_length` of them, adds to the price of their length. +#[inline(always)] +fn ll_inc_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { + Ok( + ll_price(lit_length, opt_ptr, opt_level)?.wrapping_sub(ll_price( + lit_length.wrapping_sub(1), + opt_ptr, + opt_level, + )?), + ) +} + +/// `ZSTD_compressBlock_opt_generic` +#[inline(always)] +fn compress_block_opt_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + opt_level: i32, + dict_mode: DictMode, +) -> Result { + // The match finders borrow the whole of `ms`. + let mut price_table = mem::take(&mut ms.opt.price_table); + let mut match_table = mem::take(&mut ms.opt.match_table); + let result = parse_block( + ms, + seq_store, + rep, + src, + &mut price_table, + &mut match_table, + opt_level, + dict_mode, + ); + ms.opt.price_table = price_table; + ms.opt.match_table = match_table; + result +} + +/// The body of `ZSTD_compressBlock_opt_generic`. `opt` is `ms.opt.price_table` and `matches` is `ms.opt.match_table`. +#[inline(always)] +fn parse_block( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + opt: &mut [Optimal], + matches: &mut [Match], + opt_level: i32, + dict_mode: DictMode, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let ilimit = iend.saturating_sub(8); + let prefix_start = src.pos_of(ms.window.dict_limit); + + let get_all_matches = select_bt_get_all_matches(ms, dict_mode); + + let sufficient_len = ms.c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); + let min_match = if ms.c_params.min_match == 3 { 3 } else { 4 }; + let mut next_to_update3 = ms.next_to_update; + + let mut last_stretch = Optimal::default(); + let mut opt_ldm = OptLdm::new(src.ldm_seq_store); + opt_ldm.get_next_match_and_update_seq_store(0, (iend - ip) as u32); + + rescale_freqs( + &mut ms.opt, + src.symbol_costs, + buf.sub(istart..iend)?, + opt_level, + )?; + ip += usize::from(ip == prefix_start); + + 'series: while ip < ilimit { + let mut last_pos: u32; + + // The forward pass. `opt` holds stretches, each a match followed by literals, unlike a sequence, which is + // literals followed by a match: that way each position of a run of literals can have another match before it. + // It ends with the last stretch in `last_stretch`, which starts at `cur` and ends at `last_pos`. + let mut cur = 'forward: { + // Find a first match. + { + let litlen = (ip - anchor) as u32; + let mut nb_matches = get_all_matches( + matches, + ms, + &mut next_to_update3, + src, + ip, + iend, + rep, + litlen == 0, + min_match, + )?; + opt_ldm.process_match_candidate( + matches, + &mut nb_matches, + (ip - istart) as u32, + (iend - ip) as u32, + min_match, + )?; + if nb_matches == 0 { + ip += 1; + continue 'series; + } + let matches = matches.sub(..nb_matches as usize)?; + + // The price of the literals before the first match is the same whatever comes next, so it is left + // out. But how the price of their length changes depends on their length. + let first_price = ll_price(litlen, &ms.opt, opt_level)?; + { + let first = opt.at_mut(0)?; + first.mlen = 0; + first.litlen = litlen; + first.price = first_price; + first.rep = *rep; + } + + // A large match is encoded at once. + let longest = matches.at(matches.len() - 1)?; + if longest.len > sufficient_len { + last_stretch.litlen = 0; + last_stretch.mlen = longest.len; + last_stretch.off = longest.off; + last_pos = longest.len; + break 'forward 0; + } + + // Set the prices of the first matches. + let ll0_price = ll_price(0, &ms.opt, opt_level)?; + let mut pos = 1u32; + while pos < min_match { + let stretch = opt.at_mut(pos as usize)?; + stretch.price = ZSTD_MAX_PRICE; + stretch.mlen = 0; + stretch.litlen = litlen.wrapping_add(pos); + pos += 1; + } + for candidate in matches { + while pos <= candidate.len { + let match_price = + get_match_price(candidate.off, pos, &ms.opt, opt_level)? as i32; + let sequence_price = first_price.wrapping_add(match_price); + let stretch = opt.at_mut(pos as usize)?; + stretch.mlen = pos; + stretch.off = candidate.off; + stretch.litlen = 0; + stretch.price = sequence_price.wrapping_add(ll0_price); + pos += 1; + } + } + last_pos = pos - 1; + opt.at_mut(pos as usize)?.price = ZSTD_MAX_PRICE; + } + + // Check the positions that follow. + let mut cur = 0u32; + while cur < last_pos { + cur += 1; + let inr = ip + cur as usize; + + // Get to this position by a literal, if it is cheaper. + { + let before = opt.at(cur as usize - 1)?; + let litlen = before.litlen.wrapping_add(1); + let price = before + .price + .wrapping_add(lit_price(buf, inr - 1, &ms.opt, opt_level)?) + .wrapping_add(ll_inc_price(litlen, &ms.opt, opt_level)?); + let prev_match = opt.at(cur as usize)?; + if price <= prev_match.price { + opt.put( + cur as usize, + Optimal { + litlen, + price, + ..before + }, + )?; + // If this replaces a match, and 1 literal is cheaper than none, the match followed by a + // literal may be a cheaper way to the next position. + if opt_level >= 1 + && prev_match.litlen == 0 + && ll_inc_price(1, &ms.opt, opt_level)? < 0 + && inr < iend + { + let next_lit_price = lit_price(buf, inr, &ms.opt, opt_level)?; + let with_1_literal = prev_match + .price + .wrapping_add(next_lit_price) + .wrapping_add(ll_inc_price(1, &ms.opt, opt_level)?); + let with_more_literals = price + .wrapping_add(next_lit_price) + .wrapping_add(ll_inc_price( + litlen.wrapping_add(1), + &ms.opt, + opt_level, + )?); + if with_1_literal < with_more_literals + && with_1_literal < opt.at(cur as usize + 1)?.price + { + let prev = opt.at(cur.wrapping_sub(prev_match.mlen) as usize)?; + opt.put( + cur as usize + 1, + Optimal { + rep: new_rep(&prev.rep, prev_match.off, prev.litlen == 0), + litlen: 1, + price: with_1_literal, + ..prev_match + }, + )?; + last_pos = last_pos.max(cur + 1); + } + } + } + } + + // The repeat offsets are not updated while matches are compared, but here, now that one is selected. + let mut here = opt.at(cur as usize)?; + if here.litlen == 0 { + let prev = opt.at(cur.wrapping_sub(here.mlen) as usize)?; + here.rep = new_rep(&prev.rep, here.off, prev.litlen == 0); + opt.at_mut(cur as usize)?.rep = here.rep; + } + + // The last match starts at least 8 bytes before the end. + if inr > ilimit { + continue; + } + + if cur == last_pos { + break; + } + + if opt_level == 0 + && opt.at(cur as usize + 1)?.price + <= here.price.wrapping_add(BITCOST_MULTIPLIER as i32 / 2) + { + // The next position is cheaper to get to: this one is not promising. + continue; + } + + { + let base_price = here.price.wrapping_add(ll_price(0, &ms.opt, opt_level)?); + let mut nb_matches = get_all_matches( + matches, + ms, + &mut next_to_update3, + src, + inr, + iend, + &here.rep, + here.litlen == 0, + min_match, + )?; + opt_ldm.process_match_candidate( + matches, + &mut nb_matches, + (inr - istart) as u32, + (iend - inr) as u32, + min_match, + )?; + if nb_matches == 0 { + continue; + } + let matches = matches.sub(..nb_matches as usize)?; + + let longest = matches.at(matches.len() - 1)?; + if longest.len > sufficient_len + || cur.wrapping_add(longest.len) as usize >= ZSTD_OPT_NUM + || inr + longest.len as usize >= iend + { + last_stretch.mlen = longest.len; + last_stretch.off = longest.off; + last_stretch.litlen = 0; + last_pos = cur.wrapping_add(longest.len); + break 'forward cur; + } + + // Set the prices that the matches found here give. + let mut start_ml = min_match; + for candidate in matches { + let mut mlen = candidate.len; + while mlen >= start_ml { + let pos = cur + mlen; + let price = base_price.wrapping_add(get_match_price( + candidate.off, + mlen, + &ms.opt, + opt_level, + )? + as i32); + + if pos > last_pos || price < opt.at(pos as usize)?.price { + while last_pos < pos { + // Positions that nothing leads to yet. + last_pos += 1; + let stretch = opt.at_mut(last_pos as usize)?; + stretch.price = ZSTD_MAX_PRICE; + // Not 0, which would be the end of a match. + stretch.litlen = 1; + } + let stretch = opt.at_mut(pos as usize)?; + stretch.mlen = mlen; + stretch.off = candidate.off; + stretch.litlen = 0; + stretch.price = price; + } else if opt_level == 0 { + // Shorter lengths are not tried. + break; + } + mlen -= 1; + } + start_ml = candidate.len + 1; + } + } + opt.at_mut(last_pos as usize + 1)?.price = ZSTD_MAX_PRICE; + } + + last_stretch = opt.at(last_pos as usize)?; + last_pos.wrapping_sub(last_stretch.mlen) + }; + + if last_stretch.mlen == 0 { + // All the matches have been turned into literals. + ip += last_pos as usize; + continue; + } + + if last_stretch.litlen == 0 { + // It ends with a match. + let prev = opt.at(cur as usize)?; + *rep = new_rep(&prev.rep, last_stretch.off, prev.litlen == 0); + } else { + *rep = last_stretch.rep; + cur = cur.wrapping_sub(last_stretch.litlen); + } + + // Write the cheapest path in `opt`, backwards from `store_end`, over the stretches that are not on it, and as + // sequences. The literals that end the last stretch are in no sequence: they are parsed again. (The C code + // starts by making them a last sequence without a match, and overwrites it at once.) + let store_end = cur.wrapping_add(2) as usize; + let mut store_start = store_end; + let mut stretch_pos = cur; + opt.put(store_end, last_stretch)?; + loop { + let next_stretch = opt.at(stretch_pos as usize)?; + opt.at_mut(store_start)?.litlen = next_stretch.litlen; + if next_stretch.mlen == 0 { + // The start of the series. + break; + } + store_start = store_start.wrapping_sub(1); + opt.put(store_start, next_stretch)?; + stretch_pos = + stretch_pos.wrapping_sub(next_stretch.litlen.wrapping_add(next_stretch.mlen)); + } + + for sequence in opt.sub(store_start..=store_end)? { + let llen = sequence.litlen as usize; + let mlen = sequence.mlen as usize; + update_stats( + &mut ms.opt, + buf.span(anchor, llen)?, + sequence.off, + sequence.mlen, + )?; + seq_store.store_seq(llen, buf, anchor, sequence.off, mlen)?; + anchor += llen + mlen; + ip = anchor; + } + + set_base_prices(&mut ms.opt, opt_level); + } + + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_opt0` +fn compress_block_opt0( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode) +} + +/// `ZSTD_compressBlock_opt2` +fn compress_block_opt2( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode) +} + +/// `ZSTD_initStats_ultra`: a first pass over the block, only to have better statistics to start from. It has to be the +/// first block, with no dictionary and no long distance matching. Returns the block as the window then sees it, since +/// its indices change. +fn init_stats_ultra<'a>( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &[u32; ZSTD_REP_NUM], + src: &BlockSrc<'a>, +) -> Result> { + // The repeat offsets it leaves are of no use. + let mut tmp_rep = *rep; + compress_block_opt2(ms, seq_store, &mut tmp_rep, src, DictMode::NoDict)?; + + // Of the first pass, only the statistics are kept: what the tables have is made out of reach. + let src_size = src.end - src.start; + seq_store.reset(); + ms.window.base = ms.window.base.wrapping_sub(src_size); + ms.window.dict_limit = ms.window.dict_limit.wrapping_add(src_size as u32); + ms.window.low_limit = ms.window.dict_limit; + ms.next_to_update = ms.window.dict_limit; + Ok(BlockSrc { + base: src.base.wrapping_sub(src_size), + ..*src + }) +} + +/// `ZSTD_compressBlock_btultra2` +fn compress_block_btultra2( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let curr = src.index_of(src.start); + + // The first block is parsed twice, the first time to collect statistics for the second. Since the first pass is + // then forgotten, nothing must be in the tables before it: no dictionary, no prefix, no long distance matching. + if ms.opt.lit_length_sum == 0 + && seq_store.sequences.is_empty() + && ms.window.dict_limit == ms.window.low_limit + && curr == ms.window.dict_limit + && src.end - src.start > ZSTD_PREDEF_THRESHOLD + { + let src = init_stats_ultra(ms, seq_store, rep, src)?; + return compress_block_opt2(ms, seq_store, rep, &src, DictMode::NoDict); + } + + compress_block_opt2(ms, seq_store, rep, src, DictMode::NoDict) +} + +/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` +/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. +/// +/// `btultra2` can move `ms.window.base`, which `src.base` comes from. +pub(crate) fn compress_block_opt( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + dict_mode: DictMode, +) -> Result { + match (strategy, dict_mode) { + (Strategy::BtUltra2, DictMode::NoDict) => compress_block_btultra2(ms, seq_store, rep, src), + (Strategy::BtUltra | Strategy::BtUltra2, _) => { + compress_block_opt2(ms, seq_store, rep, src, dict_mode) + } + _ => compress_block_opt0(ms, seq_store, rep, src, dict_mode), + } +} diff --git a/src/zstd/wip/work/opt/zstd/compress/presplit.rs b/src/zstd/wip/work/opt/zstd/compress/presplit.rs new file mode 100644 index 000000000000..8b778e2989f9 --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/compress/presplit.rs @@ -0,0 +1 @@ +//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. diff --git a/src/zstd/wip/work/opt/zstd/lib.rs b/src/zstd/wip/work/opt/zstd/lib.rs new file mode 100644 index 000000000000..31c98c72e5e4 --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/lib.rs @@ -0,0 +1,385 @@ +//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the +//! same bytes in, in both directions, and the same errors. +#![forbid(unsafe_code)] +// Frames come from anywhere and a panic aborts the process. +#![deny( + clippy::indexing_slicing, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unreachable +)] +#![warn(unused_must_use)] + +mod common; +mod compress; +mod decompress; +mod error; +mod util; + +pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; +pub use decompress::{ + DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, + find_frame_compressed_size, get_frame_content_size, +}; +pub use error::ErrorCode; + +/// `ZSTD_inBuffer` +pub struct InBuffer<'a> { + pub src: &'a [u8], + /// How much of `src` is read. + pub pos: usize, +} + +/// `ZSTD_outBuffer` +pub struct OutBuffer<'a> { + pub dst: &'a mut [u8], + /// How much of `dst` is written. + pub pos: usize, +} + +/// `ZSTD_ResetDirective` +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ResetDirective { + SessionOnly = 1, + Parameters = 2, + SessionAndParameters = 3, +} + +// ----------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] +pub enum ZstdError { + /// The output, or the decoder state the frame's window size dictates, could not be allocated. + OutOfMemory, + InvalidZstdData, + DecompressionFailed, + ZstdDecompressionError, + ShortRead, +} + +impl core::fmt::Display for ZstdError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(<&'static str>::from(self)) + } +} +impl core::error::Error for ZstdError {} + +impl ZstdError { + /// The error for a failed decompression call; `other` is the non-allocation failure. + fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { + if code == ErrorCode::MemoryAllocation { + ZstdError::OutOfMemory + } else { + other + } + } +} + +/// `len` zeros for a decoder to write to. +fn output_buffer(len: usize) -> Result, ZstdError> { + util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) +} + +/// Decompress data, automatically allocating the output buffer. +/// Handles both frames with known and unknown content sizes. +/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. +/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. +pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { + let size = find_decompressed_size(src); + + const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit + + if size == ZSTD_CONTENTSIZE_ERROR { + return Err(ZstdError::InvalidZstdData); + } + + // Use streaming decompression if: + // 1. Content size is unknown, OR + // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) + if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { + let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { + // A frame's output is rarely smaller than its input. + src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) + } else { + // The header size is untrusted: reserve no more than the fast path below would. + MAX_PREALLOCATE_SIZE + }; + let mut list: Vec = Vec::new(); + list.try_reserve_exact(initial_capacity) + .map_err(|_| ZstdError::OutOfMemory)?; + StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; + return Ok(list); + } + + // Fast path: size is known and within reasonable limits + let mut output = output_buffer(size as usize)?; + let written = decompress(&mut output, src) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is +/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. +pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { + const MAX_SIZE: u64 = 16 * 1024 * 1024; + let size = find_decompressed_size(src); + // Also if it is unknown or invalid, which are the two largest values. + if size > MAX_SIZE { + return Err(ZstdError::InvalidZstdData); + } + let mut output = output_buffer(size as usize)?; + let written = DDict::prefix(prefix) + .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; + output.truncate(written); + Ok(output) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + Uninitialized, + Inflating, + End, + Error, +} + +/// Minimum spare output capacity offered to the decoder per call. +const STREAMING_OUTPUT_STEP: usize = 4096; + +// ────────────────────────────────────────────────────────────────────────── +// StreamingDecoder +// ────────────────────────────────────────────────────────────────────────── + +/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) +/// per call, so callers can hold the decoder across multiple body chunks. +pub struct StreamingDecoder { + stream: DCtx, + state: State, + /// The decoder filled its last window and may hold more. `max_output` can end a call there. + output_full: bool, +} + +impl Default for StreamingDecoder { + fn default() -> Self { + Self::new() + } +} + +impl StreamingDecoder { + pub fn new() -> Self { + Self { + stream: DCtx::new(), + state: State::Uninitialized, + output_full: false, + } + } + + #[inline] + pub fn is_inflating(&self) -> bool { + matches!(self.state, State::Inflating) + } + + /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is + /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. + /// Returns `ShortRead` when more input is required and `is_done` is false. + pub fn decompress( + &mut self, + input: &[u8], + out: &mut Vec, + max_output: usize, + is_done: bool, + ) -> Result { + if matches!(self.state, State::End | State::Error) { + return Ok(input.len()); + } + + let mut in_buf = InBuffer { src: input, pos: 0 }; + while matches!(self.state, State::Uninitialized | State::Inflating) { + let next_in_len = input.len() - in_buf.pos; + + // Call the decoder again with no input until it leaves the window short. + if next_in_len == 0 && !self.output_full { + if is_done { + if self.state == State::Inflating { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + self.state = State::End; + } + return Ok(in_buf.pos); + } + + if out.len() >= max_output { + return Ok(in_buf.pos); + } + + if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { + self.state = State::Error; + return Err(ZstdError::OutOfMemory); + } + // The decoder writes to initialized memory only. + let written_before = out.len(); + let window = (out.capacity() - written_before).min(max_output - written_before); + out.resize(written_before + window, 0); + let mut out_buf = OutBuffer { + dst: out.get_mut(written_before..).unwrap_or_default(), + pos: 0, + }; + let read_before = in_buf.pos; + + let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); + let bytes_written = out_buf.pos; + out.truncate(written_before + bytes_written); + let remaining = match result { + Ok(remaining) => remaining, + Err(code) => { + self.state = State::Error; + return Err(ZstdError::for_decompression( + code, + ZstdError::ZstdDecompressionError, + )); + } + }; + + self.output_full = bytes_written == window; + + if remaining == 0 { + // Frame complete, and fully flushed. + self.state = State::Uninitialized; + self.output_full = false; + if in_buf.pos >= input.len() { + if is_done { + self.state = State::End; + } + return Ok(in_buf.pos); + } + // More input available — reinitialize for the next frame. + self.stream.init_stream(); + continue; + } + + self.state = State::Inflating; + + if in_buf.pos - read_before == next_in_len { + if self.output_full { + continue; + } + if is_done { + self.state = State::Error; + return Err(ZstdError::ZstdDecompressionError); + } + return Err(ZstdError::ShortRead); + } + } + Ok(in_buf.pos) + } +} + +// ── compressed embedded assets ──────────────────────────────────────────── + +/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes +/// Bun never executes or parses itself: the shell completion scripts and the +/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, +/// error overlay/page). Anything that runs inside Bun stays uncompressed. +/// +/// The exception is text that only `bun lint`, `bun fmt` and `bun check` +/// read. It may be compressed because it isn't on the path of running a program. +/// +/// Release (`bun_codegen_embed`) builds include the `.zst` twin that +/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to +/// `/compressed/.zst`; other builds evaluate `$fallback`. +#[macro_export] +macro_rules! embed_compressed { + // A codegen output (`/$sub`); debug builds read it at runtime. + (codegen $sub:literal) => { + $crate::embed_compressed!( + ("codegen/", $sub), + ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() + ) + }; + // Same, with a trailing NUL included in the slice (for C-string consumers). + (codegen_nul $sub:literal) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/codegen/", + $sub, + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); + &__COPY.get_or_init(|| { + let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); + let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); + copy.extend_from_slice(text); + copy.push(0); + copy.into_boxed_slice() + })[..] + } + }; + __bytes + }}; + // A file under `src/`; debug builds read it from the source tree at runtime. + (src $sub:literal) => { + $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) + }; + ($name:literal, $fallback:expr) => { + $crate::embed_compressed!(($name), $fallback) + }; + (($($name:literal),+), $fallback:expr) => {{ + #[allow(unexpected_cfgs)] + let __bytes: &'static [u8] = { + #[cfg(bun_codegen_embed)] + { + static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); + __INFLATED + .get_or_init(|| { + $crate::inflate_embedded(::core::include_bytes!(::core::concat!( + ::core::env!("BUN_CODEGEN_DIR"), + "/compressed/", + $($name,)+ + ".zst" + ))) + }) + } + #[cfg(not(bun_codegen_embed))] + { + $fallback + } + }; + __bytes + }}; +} + +/// Cold, shared body of [`embed_compressed!`]'s release arm. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { + decompress_alloc(compressed) + .expect("embedded asset: invalid zstd frame") + .into_boxed_slice() +} + +/// [`inflate_embedded`] plus a trailing NUL byte. +#[cold] +#[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] +pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { + let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); + inflated.reserve_exact(1); + inflated.push(0); + inflated.into_boxed_slice() +} +pub mod testing; diff --git a/src/zstd/wip/work/opt/zstd/testing.rs b/src/zstd/wip/work/opt/zstd/testing.rs new file mode 100644 index 000000000000..36fe6bd042bb --- /dev/null +++ b/src/zstd/wip/work/opt/zstd/testing.rs @@ -0,0 +1,288 @@ +//! Hooks for the differential tests. Only in scratch copies of the crate. +#![allow(clippy::all, dead_code, unreachable_pub)] + +use crate::ErrorCode; +use crate::compress::internal::*; +use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; + +fn bitmix(mut val: u64, len: u64) -> u64 { + val ^= val.rotate_right(49) ^ val.rotate_right(24); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^= (val >> 35).wrapping_add(len); + val = val.wrapping_mul(0x9FB21C651E98DF25); + val ^ (val >> 28) +} + +#[derive(Clone, Copy)] +pub enum Dict<'a> { + None, + /// `ZSTD_CCtx_refPrefix`, raw content. + Prefix(&'a [u8]), + /// `ZSTD_CCtx_refCDict`, raw content, attached. With the parameters of the CDict. + CDict(&'a [u8], [u32; 7]), +} + +/// What is left in the match state after the frame. +#[derive(PartialEq, Eq, Debug, Default)] +pub struct State { + pub has_dms: bool, + pub next_to_update: u32, + pub loaded_dict_end: u32, + pub low_limit: u32, + pub dict_limit: u32, + pub hash_salt_entropy: u32, + pub lazy_skipping: bool, + pub hash_cache: [u32; 8], + pub rep: [u32; 3], + pub hash_table: Vec, + pub chain_table: Vec, + pub tag_table: Vec, + pub hash_table3: Vec, + pub sums: [u32; 8], + pub freqs: [Vec; 4], +} + +fn params(cp: [u32; 7]) -> Result { + Ok(CompressionParameters { + window_log: cp[0], chain_log: cp[1], hash_log: cp[2], search_log: cp[3], min_match: cp[4], target_length: cp[5], + strategy: Some(Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?), + }) +} + +/// `ZSTD_reset_matchState`, on a new one. +fn reset_match_state(c_params: CompressionParameters, use_row: bool, for_cdict: bool) -> MatchState { + let strategy = c_params.strat(); + let row_used = row_match_finder_supported(strategy) && use_row; + let mut ms = MatchState::new(); + ms.hash_log3 = if !for_cdict && c_params.min_match == 3 { c_params.window_log.min(17) } else { 0 }; + ms.window.clear(); + ms.next_to_update = ms.window.dict_limit; + ms.hash_table = vec![0; 1 << c_params.hash_log]; + ms.chain_table = vec![0; if strategy != Strategy::Fast && !row_used { 1 << c_params.chain_log } else { 0 }]; + ms.hash_table3 = vec![0; if ms.hash_log3 != 0 { 1 << ms.hash_log3 } else { 0 }]; + if row_used { + ms.tag_table = vec![0; 1 << c_params.hash_log]; + ms.hash_salt = if for_cdict { 0 } else { bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4) }; + ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); + } + if !for_cdict && strategy >= Strategy::BtOpt { + ms.opt.lit_freq = vec![0; 256]; + ms.opt.lit_length_freq = vec![0; 36]; + ms.opt.match_length_freq = vec![0; 53]; + ms.opt.off_code_freq = vec![0; 32]; + ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; + ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; + } + ms.c_params = c_params; + ms +} + +/// `ZSTD_loadDictionaryContent` +fn load_dictionary_content(ms: &mut MatchState, use_row: bool, dict: &[u8], dtlm: DictTableLoadMethod, tfp: TableFillPurpose) -> Result<(), ErrorCode> { + let c_params = ms.c_params; + let iend = dict.len(); + ms.window.update(BufId::Dict.start(), BufId::Dict, dict.len(), false); + let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); + let ip = iend - dict.len().min(max_dict_size); + let prefix = Prefix { buf: dict, base: ms.window.base.wrapping_sub(BufId::Dict.start()) }; + ms.next_to_update = prefix.index_of(ip); + ms.loaded_dict_end = prefix.index_of(iend); + if iend - ip > HASH_READ_SIZE { + match c_params.strat() { + Strategy::Fast => fast::fill_hash_table(ms, prefix, iend, dtlm, tfp)?, + Strategy::DFast => double_fast::fill_double_hash_table(ms, prefix, iend, dtlm, tfp)?, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { + if use_row { + ms.tag_table.fill(0); + lazy::row_update(ms, prefix, iend - HASH_READ_SIZE)?; + } else { + lazy::insert_and_find_first_index(ms, prefix, iend - HASH_READ_SIZE)?; + } + } + _ => opt::update_tree(ms, prefix, iend - HASH_READ_SIZE, iend)?, + } + ms.next_to_update = prefix.index_of(iend); + } + Ok(()) +} + +/// What the public parameters of the C library cannot set. +#[derive(Clone, Copy, Default)] +pub struct Extra<'a> { + /// `literalCompressionMode`: 0 auto, 1 enable, 2 disable. + pub lit_mode: u32, + /// (offset, literal length, match length), as from long distance matching. + pub ldm: &'a [(u32, u32, u32)], + /// The Huffman table, and `deltaNbBits` for the codes of literal lengths, match lengths and offsets. + pub entropy: Option<(&'a [u64; 257], &'a [u32; 36], &'a [u32; 53], &'a [u32; 32])>, + /// Input that comes before `src`, from another buffer. + pub first_chunk: &'a [u8], +} + +/// `ZSTD_ldm_skipRawSeqStoreBytes` +fn skip_raw_seq_store_bytes(store: &mut RawSeqStore, nb_bytes: usize) { + let mut curr_pos = (store.pos_in_sequence + nb_bytes) as u32; + while curr_pos != 0 && store.pos < store.size { + let seq = store.seq[store.pos]; + if curr_pos >= seq.lit_length.wrapping_add(seq.match_length) { + curr_pos -= seq.lit_length.wrapping_add(seq.match_length); + store.pos += 1; + } else { + store.pos_in_sequence = curr_pos as usize; + break; + } + } + if curr_pos == 0 || store.pos == store.size { + store.pos_in_sequence = 0; + } +} + +pub fn frame(cp: [u32; 7], use_row: bool, dict: Dict<'_>, src: &[u8]) -> Result<(Vec<(u32, u32, u32, u32)>, State), ErrorCode> { + frame2(cp, use_row, dict, src, Extra::default()) +} + +/// The sequences of all the blocks of a frame, as (literal length, match length, repeat code or 0, offset or 0). Each +/// block ends with the literals that are left as (length, 0, 0, 0). +/// +/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, as applied. +pub fn frame2(cp: [u32; 7], use_row: bool, dict: Dict<'_>, src: &[u8], extra: Extra<'_>) -> Result<(Vec<(u32, u32, u32, u32)>, State), ErrorCode> { + let c_params = params(cp)?; + let strategy = c_params.strat(); + let mut ms = reset_match_state(c_params, use_row, false); + + let block_size_max = (1usize << c_params.window_log).min(128 << 10); + let mut seq_store = SeqStore::new(); + seq_store.reserve(block_size_max / if c_params.min_match == 3 { 3 } else { 4 }, block_size_max)?; + + let dict_content = match dict { Dict::None => &[][..], Dict::Prefix(d) | Dict::CDict(d, _) => d }; + let bufs = Buffers { in_buff: extra.first_chunk, user: src, dict: dict_content }; + + let mut cdict_ms = None; + match dict { + Dict::Prefix(d) if d.len() >= 8 => load_dictionary_content(&mut ms, use_row, d, DictTableLoadMethod::Fast, TableFillPurpose::ForCCtx)?, + Dict::CDict(d, cdict_cp) if d.len() >= 8 => { + let mut c = reset_match_state(params(cdict_cp)?, use_row, true); + load_dictionary_content(&mut c, use_row, d, DictTableLoadMethod::Full, TableFillPurpose::ForCDict)?; + // ZSTD_resetCCtx_byAttachingCDict + let cdict_end = c.window.index_of(c.window.next_src); + if ms.window.dict_limit < cdict_end { + ms.window.next_src = ms.window.base + cdict_end as usize; + ms.window.clear(); + } + ms.loaded_dict_end = ms.window.dict_limit; + cdict_ms = Some(c); + } + _ => {} + } + let mut dms = cdict_ms.as_ref().map(|c| DictMatchState { + ms: c, + buf: dict_content, + base: c.window.base.wrapping_sub(BufId::Dict.start()), + end: c.window.next_src - BufId::Dict.start(), + }); + + let max_dist = 1u32 << c_params.window_log; + let mut prev = CompressedBlockState::new(); + if let Some((huf, ll, ml, of)) = extra.entropy { + prev.entropy.huf.c_table.0 = *huf; + prev.entropy.huf.repeat_mode = crate::compress::huf::HufRepeat::Valid; + for (to, from) in prev.entropy.fse.litlength_c_table.symbol_tt.iter_mut().zip(ll) { to.delta_nb_bits = *from; } + for (to, from) in prev.entropy.fse.matchlength_c_table.symbol_tt.iter_mut().zip(ml) { to.delta_nb_bits = *from; } + for (to, from) in prev.entropy.fse.offcode_c_table.symbol_tt.iter_mut().zip(of) { to.delta_nb_bits = *from; } + } + ms.opt.literal_compression_mode = match extra.lit_mode { 0 => ParamSwitch::Auto, 1 => ParamSwitch::Enable, _ => ParamSwitch::Disable }; + let mut extern_seq_store = RawSeqStore { + seq: extra.ldm.iter().map(|&(offset, lit_length, match_length)| RawSeq { offset, lit_length, match_length }).collect(), + pos: 0, pos_in_sequence: 0, size: extra.ldm.len(), capacity: extra.ldm.len(), + }; + let mut rep = prev.rep; + let mut out = Vec::new(); + for (src, src_buf) in [(extra.first_chunk, BufId::InBuff), (src, BufId::User)] { + // ZSTD_compressContinue_internal + let src_addr = src_buf.start(); + if !ms.window.update(src_addr, src_buf, src.len(), ms.force_non_contiguous) { + ms.force_non_contiguous = false; + ms.next_to_update = ms.window.dict_limit; + } + // ZSTD_compress_frameChunk + let mut pos = 0; + while pos < src.len() { + let block_size = (src.len() - pos).min(block_size_max); + if ms.window.dict_is_invalid(src_addr + pos + block_size, max_dist, ms.loaded_dict_end) { + ms.loaded_dict_end = 0; + dms = None; + } + if ms.window.enforce_max_dist(src_addr + pos, max_dist, ms.loaded_dict_end) { + ms.loaded_dict_end = 0; + dms = None; + } + if ms.next_to_update < ms.window.low_limit { + ms.next_to_update = ms.window.low_limit; + } + // ZSTD_buildSeqStore + assert!(block_size >= 7); + seq_store.reset(); + let block = BlockSrc { + buf: bufs.get(ms.window.base_buf), + base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), + dict_buf: bufs.get(ms.window.dict_buf), + dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), + start: pos, + end: pos + block_size, + dms, + symbol_costs: &prev.entropy, + ldm_seq_store: if extern_seq_store.pos < extern_seq_store.size { Some(&extern_seq_store) } else { None }, + }; + let curr = block.index_of(pos); + if curr > ms.next_to_update + 384 { + ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); + } + let dict_mode = ms.dict_mode(dms.as_ref()); + let mut reps = rep; + let last_ll = compress::compress_block(&mut ms, &mut seq_store, &mut rep, &block, strategy, use_row, dict_mode)?; + if extern_seq_store.pos < extern_seq_store.size { + skip_raw_seq_store_bytes(&mut extern_seq_store, block_size); + } + + let mut total = last_ll; + for (i, seq) in seq_store.sequences.iter().enumerate() { + let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); + if seq_store.long_length_pos as usize == i { + match seq_store.long_length_type { + LongLengthType::LiteralLength => ll += 0x10000, + LongLengthType::MatchLength => ml += 0x10000, + LongLengthType::None => {} + } + } + update_rep(&mut reps, seq.off_base, ll == 0); + total += (ll + ml) as usize; + out.push(if seq.off_base <= 3 { (ll, ml, seq.off_base, 0) } else { (ll, ml, 0, seq.off_base - 3) }); + } + out.push((last_ll as u32, 0, 0, 0)); + assert_eq!(total, block_size, "the sequences cover the block"); + assert_eq!(seq_store.lit_len + last_ll + seq_store.sequences.iter().enumerate().map(|(i, s)| s.ml_base as usize + 3 + + if seq_store.long_length_pos as usize == i && seq_store.long_length_type == LongLengthType::MatchLength { 0x10000 } else { 0 }).sum::(), block_size); + assert_eq!(reps[..2], rep[..2], "the repeat offsets the block compressor leaves are those of its sequences"); + pos += block_size; + } + } + let state = State { + has_dms: dms.is_some(), + next_to_update: ms.next_to_update, + loaded_dict_end: ms.loaded_dict_end, + low_limit: ms.window.low_limit, + dict_limit: ms.window.dict_limit, + hash_salt_entropy: ms.hash_salt_entropy, + lazy_skipping: ms.lazy_skipping, + hash_cache: ms.hash_cache, + rep, + hash_table: ms.hash_table, + chain_table: ms.chain_table, + tag_table: ms.tag_table, + hash_table3: ms.hash_table3, + sums: [ms.opt.lit_sum, ms.opt.lit_length_sum, ms.opt.match_length_sum, ms.opt.off_code_sum, ms.opt.lit_sum_base_price, + ms.opt.lit_length_sum_base_price, ms.opt.match_length_sum_base_price, ms.opt.off_code_sum_base_price], + freqs: [ms.opt.lit_freq, ms.opt.lit_length_freq, ms.opt.match_length_freq, ms.opt.off_code_freq], + }; + Ok((out, state)) +} diff --git a/test/js/node/process/process.test.js b/test/js/node/process/process.test.js index 9633bbbb89f4..256258aebb9f 100644 --- a/test/js/node/process/process.test.js +++ b/test/js/node/process/process.test.js @@ -761,7 +761,6 @@ it("process.versions", async () => { lolhtml: "lolhtml", ares: "cares", libdeflate: "libdeflate", - zstd: "zstd", lshpack: "lshpack", }; @@ -787,6 +786,9 @@ it("process.versions", async () => { expect(process.versions).toHaveProperty("sqlite"); expect(process.versions.sqlite).toMatch(/^3\.\d+\.\d+$/); + // The version of zstd that Bun's own implementation behaves like. + expect(process.versions.zstd).toBe("1.5.7"); + // The version of TypeScript that `bun check` behaves like. expect(process.versions.typescript).toMatch(/^\d+\.\d+\.\d+$/); }); From 360684e85f8b1c48194e94be236bdbd5fdd30265 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 12:25:17 +0000 Subject: [PATCH 004/199] zstd: remove what nothing uses, and what clippy finds --- src/zstd/compress/block_entropy.rs | 9 +++- src/zstd/compress/cctx.rs | 34 +++++++------- src/zstd/compress/internal.rs | 11 +---- src/zstd/compress/lazy.rs | 31 ++++++++++-- src/zstd/compress/ldm.rs | 19 -------- src/zstd/compress/params.rs | 15 ++++-- src/zstd/compress/presplit.rs | 3 -- src/zstd/compress/sequences.rs | 3 +- src/zstd/compress/superblock.rs | 35 +++++++------- src/zstd/decompress/mod.rs | 4 +- src/zstd/wip/PORTING.md | 75 ++++++++++++++++++++++++++++++ 11 files changed, 162 insertions(+), 77 deletions(-) create mode 100644 src/zstd/wip/PORTING.md diff --git a/src/zstd/compress/block_entropy.rs b/src/zstd/compress/block_entropy.rs index 2701b1af9e44..440376512a31 100644 --- a/src/zstd/compress/block_entropy.rs +++ b/src/zstd/compress/block_entropy.rs @@ -536,7 +536,9 @@ fn estimate_block_size_symbol_type( let mut max = max_code; hist::count_fast_wksp(&mut count, &mut max, code_table)?; let estimate_in_bits = match ty { - SymbolEncodingType::Basic => cross_entropy_cost(default_norm, default_norm_log, &count, max), + SymbolEncodingType::Basic => { + cross_entropy_cost(default_norm, default_norm_log, &count, max) + } SymbolEncodingType::Rle => Ok(0), SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { fse_bit_cost(fse_c_table, &count, max) @@ -638,7 +640,10 @@ pub(crate) fn build_entropy_statistics_and_estimate_sub_block_size( /// `ZSTD_countSeqStoreLiteralsBytes` pub(crate) fn count_seq_store_literals_bytes(store: &SeqStore, chunk: &SeqChunk) -> Result { let sequences = store.sequences.sub(chunk.seq_start..chunk.seq_end)?; - let mut literals_bytes: usize = sequences.iter().map(|seq| usize::from(seq.lit_length)).sum(); + let mut literals_bytes: usize = sequences + .iter() + .map(|seq| usize::from(seq.lit_length)) + .sum(); if chunk.long_length_type == LongLengthType::LiteralLength && (chunk.long_length_pos as usize) < sequences.len() { diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 3023d25df465..df7ed1a8085f 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -14,8 +14,8 @@ use super::internal::{ DictTableLoadMethod, EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqChunk, SeqStore, Strategy, TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, - ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, min_gain, - no_compress_block, offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, + ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, min_gain, no_compress_block, + offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, }; use super::params::{ CCtxParams, CParamMode, Format, MatchStateSizes, Workspace, WorkspaceNeeds, @@ -182,11 +182,7 @@ fn reset_match_state( force_reset_index: IndexResetPolicy, for_who: ResetTarget, ) -> Result<()> { - let sizes = MatchStateSizes::new( - c_params, - use_row_match_finder, - for_who == ResetTarget::CCtx, - ); + let sizes = MatchStateSizes::new(c_params, use_row_match_finder, for_who == ResetTarget::CCtx); if force_reset_index == IndexResetPolicy::Reset { ms.window = Window::new(); @@ -301,12 +297,14 @@ fn load_dictionary_content( } } - ms.window.update(src.start_addr(), src.buf, src.len(), false); + ms.window + .update(src.start_addr(), src.buf, src.len(), false); if let Some(ls) = ls && params.ldm_params.enable_ldm == ParamSwitch::Enable { - ls.window.update(src.start_addr(), src.buf, src.len(), false); + ls.window + .update(src.start_addr(), src.buf, src.len(), false); ls.loaded_dict_end = if params.force_window { 0 } else { @@ -371,7 +369,9 @@ fn dict_ncount_repeat( if dict_max_symbol_value < max_symbol_value { return FseRepeat::Check; } - let counts = normalized_counter.iter().take(max_symbol_value as usize + 1); + let counts = normalized_counter + .iter() + .take(max_symbol_value as usize + 1); if counts.into_iter().any(|&count| count == 0) { return FseRepeat::Check; } @@ -460,11 +460,8 @@ fn load_c_entropy(bs: &mut CompressedBlockState, dict: &[u8]) -> Result { let max_offset = dict_content_size as u32 + (128 << 10); offcode_max = highbit32(max_offset); } - bs.entropy.fse.offcode_repeat_mode = dict_ncount_repeat( - &offcode_ncount, - offcode_max_value, - offcode_max.min(MAX_OFF), - ); + bs.entropy.fse.offcode_repeat_mode = + dict_ncount_repeat(&offcode_ncount, offcode_max_value, offcode_max.min(MAX_OFF)); for rep in bs.rep { if rep == 0 || rep as usize > dict_content_size { @@ -1643,7 +1640,12 @@ impl FrameCompressor { } let ms = &mut self.match_state; - if !(ms.window).update(src.start_addr(), src.buf, src.len(), ms.force_non_contiguous) { + if !(ms.window).update( + src.start_addr(), + src.buf, + src.len(), + ms.force_non_contiguous, + ) { ms.force_non_contiguous = false; ms.next_to_update = ms.window.dict_limit; } diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 74db66398416..3f2aab1365fe 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -18,7 +18,7 @@ pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; -pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; +pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53usize * 9 + 36 * 9 + 32 * 8).div_ceil(8); /// `ZSTD_strategy` #[repr(u8)] @@ -416,8 +416,6 @@ impl SeqStore { } pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; -pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; -pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; /// `OFFSET_TO_OFFBASE`: `offset` is not 0. #[inline(always)] @@ -775,13 +773,6 @@ impl Window { self.dict_limit = end; } - /// `ZSTD_window_isEmpty` - pub(crate) fn is_empty(&self) -> bool { - self.dict_limit == ZSTD_WINDOW_START_INDEX - && self.low_limit == ZSTD_WINDOW_START_INDEX - && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize - } - /// `ZSTD_window_hasExtDict` #[inline] pub(crate) fn has_ext_dict(&self) -> bool { diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 267b0e7c9577..5a4f9eaf7a53 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -651,7 +651,12 @@ fn row_next_cached_hash( mls, hash_salt, )? as u32; - row_prefetch(hash_table, tag_table, ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize, row_log); + row_prefetch( + hash_table, + tag_table, + ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize, + row_log, + ); let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; Ok(core::mem::replace(cached, new_hash)) } @@ -675,7 +680,17 @@ fn row_update_internal_impl( for idx in update_start_idx..update_end_idx { let hash = if use_cache { - row_next_cached_hash(&mut ms.hash_cache, hash_table, tag_table, row_log, prefix, idx, hash_log, mls, hash_salt)? + row_next_cached_hash( + &mut ms.hash_cache, + hash_table, + tag_table, + row_log, + prefix, + idx, + hash_log, + mls, + hash_salt, + )? } else { hash_ptr_salted( prefix.buf, @@ -826,7 +841,17 @@ fn row_find_best_match usize { - size -} - -/// `ZSTD_ldm_getTableSize`: how many bytes of workspace the C code takes for the tables. -pub(crate) fn get_table_size(params: LdmParams) -> usize { - let ldm_h_size = 1usize << (params.hash_log & 63); - let ldm_bucket_size_log = params.bucket_size_log.min(params.hash_log); - let ldm_bucket_size = 1usize << ((params.hash_log - ldm_bucket_size_log) & 63); - let total_size = cwksp_alloc_size(ldm_bucket_size) - + cwksp_alloc_size(ldm_h_size.wrapping_mul(size_of::())); - if params.enable_ldm == ParamSwitch::Enable { - total_size - } else { - 0 - } -} - /// `ZSTD_ldm_getMaxNbSeq`: the most sequences that `max_chunk_size` bytes can give. pub(crate) fn get_max_nb_seq(params: LdmParams, max_chunk_size: usize) -> usize { if params.enable_ldm == ParamSwitch::Enable { diff --git a/src/zstd/compress/params.rs b/src/zstd/compress/params.rs index 44d9fe2ae0b4..bc7a74097961 100644 --- a/src/zstd/compress/params.rs +++ b/src/zstd/compress/params.rs @@ -104,7 +104,6 @@ pub(crate) enum CParamMode { /// The dictionary keeps its own tables, so its size does not count. AttachDict, CreateCDict, - Unknown, } /// `ZSTD_CCtx_params`, without the fields that nothing can change from their default. @@ -459,7 +458,7 @@ pub(crate) fn adjust_c_params_internal( const MAX_WINDOW_RESIZE: u64 = 1 << (ZSTD_WINDOWLOG_MAX - 1); match mode { - CParamMode::Unknown | CParamMode::NoAttachDict => {} + CParamMode::NoAttachDict => {} CParamMode::CreateCDict => { if dict_size != 0 && src_size == ZSTD_CONTENTSIZE_UNKNOWN { src_size = MIN_SRC_SIZE; @@ -540,8 +539,12 @@ pub(crate) fn get_c_params_from_cctx_params( dict_size: usize, mode: CParamMode, ) -> CompressionParameters { - let mut c_params = - get_c_params_internal(cctx_params.compression_level, src_size_hint, dict_size, mode); + let mut c_params = get_c_params_internal( + cctx_params.compression_level, + src_size_hint, + dict_size, + mode, + ); if cctx_params.ldm_params.enable_ldm == ParamSwitch::Enable { c_params.window_log = ZSTD_LDM_DEFAULT_WINDOW_LOG; } @@ -734,7 +737,9 @@ impl WorkspaceNeeds<'_> { total += aligned64(max_nb_seq * SIZEOF_SEQ_DEF); if self.ldm_params.enable_ldm == ParamSwitch::Enable { total += aligned64((1usize << self.ldm_params.hash_log) * SIZEOF_LDM_ENTRY) - + aligned64(ldm::get_max_nb_seq(*self.ldm_params, self.block_size) * SIZEOF_RAW_SEQ) + + aligned64( + ldm::get_max_nb_seq(*self.ldm_params, self.block_size) * SIZEOF_RAW_SEQ, + ) + (1usize << (self.ldm_params.hash_log - self.ldm_params.bucket_size_log)); } total diff --git a/src/zstd/compress/presplit.rs b/src/zstd/compress/presplit.rs index 6676175f0c1f..00ab8c5a217f 100644 --- a/src/zstd/compress/presplit.rs +++ b/src/zstd/compress/presplit.rs @@ -4,9 +4,6 @@ use super::hist; use crate::error::Result; use crate::util::SliceExt; -/// What the C code needs of workspace, which counts in the size of a context. -pub(crate) const ZSTD_SLIPBLOCK_WORKSPACESIZE: usize = 8208; - const THRESHOLD_PENALTY_RATE: u64 = 16; const THRESHOLD_BASE: i32 = THRESHOLD_PENALTY_RATE as i32 - 2; const THRESHOLD_PENALTY: i32 = 3; diff --git a/src/zstd/compress/sequences.rs b/src/zstd/compress/sequences.rs index d05a6abf85b4..c3eb497cb86b 100644 --- a/src/zstd/compress/sequences.rs +++ b/src/zstd/compress/sequences.rs @@ -141,8 +141,7 @@ pub(crate) fn select_encoding_type( if *repeat_mode == FseRepeat::Valid && nb_seq < STATIC_FSE_NB_SEQ_MAX { return Ok(SymbolEncodingType::Repeat); } - if nb_seq < dynamic_fse_nb_seq_min - || most_frequent < (nb_seq >> (default_norm_log - 1)) + if nb_seq < dynamic_fse_nb_seq_min || most_frequent < (nb_seq >> (default_norm_log - 1)) { *repeat_mode = FseRepeat::None; return Ok(SymbolEncodingType::Basic); diff --git a/src/zstd/compress/superblock.rs b/src/zstd/compress/superblock.rs index 56134573a1ee..cffc902e2352 100644 --- a/src/zstd/compress/superblock.rs +++ b/src/zstd/compress/superblock.rs @@ -11,10 +11,9 @@ use super::params::CCtxParams; use super::sequences::{cross_entropy_cost, encode_sequences, fse_bit_cost}; use super::{hist, huf}; use crate::common::{ - BlockType, DEFAULT_MAX_OFF, HUF_SYMBOLVALUE_MAX, LL_BITS, LL_DEFAULT_NORM, - LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MINMATCH, ML_BITS, ML_DEFAULT_NORM, - ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, SymbolEncodingType, - ZSTD_BLOCKHEADERSIZE, + BlockType, DEFAULT_MAX_OFF, HUF_SYMBOLVALUE_MAX, LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, + MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MINMATCH, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, + OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, SymbolEncodingType, ZSTD_BLOCKHEADERSIZE, }; use crate::error::{ErrorCode, Result}; use crate::util::{SliceExt, write_le24, write_le32}; @@ -76,9 +75,7 @@ fn compress_sub_block_literal( return Ok((no_compress_literals(dst, literals)?, false)); } // The size of the header was a guess. - if lh_size - < 3 + usize::from(c_lit_size >= 1 << 10) + usize::from(c_lit_size >= 16 << 10) - { + if lh_size < 3 + usize::from(c_lit_size >= 1 << 10) + usize::from(c_lit_size >= 16 << 10) { return Ok((no_compress_literals(dst, literals)?, false)); } } @@ -163,7 +160,10 @@ fn compress_sub_block_sequences( op += tables.len(); } else { let repeat = SymbolEncodingType::Repeat as u32; - dst.put(seq_head, ((repeat << 6) + (repeat << 4) + (repeat << 2)) as u8)?; + dst.put( + seq_head, + ((repeat << 6) + (repeat << 4) + (repeat << 2)) as u8, + )?; } let bitstream_size = encode_sequences( @@ -402,7 +402,8 @@ fn size_block_sequences( if budget > target_budget { return Ok(1); } - let mut in_size = usize::from(first.lit_length) + usize::from(first.ml_base) + MINMATCH as usize; + let mut in_size = + usize::from(first.lit_length) + usize::from(first.ml_base) + MINMATCH as usize; let mut n = 1; for seq in rest { @@ -462,13 +463,12 @@ fn compress_sub_block_multi( write_seq_entropy, )? + est_lit_size + ZSTD_BLOCKHEADERSIZE; - let avg_lit_cost = if nb_literals != 0 { - (est_lit_size * BYTESCALE) / nb_literals - } else { - BYTESCALE - }; + let avg_lit_cost = (est_lit_size * BYTESCALE) + .checked_div(nb_literals) + .unwrap_or(BYTESCALE); let avg_seq_cost = ((est_block_size - est_lit_size) * BYTESCALE) / nb_seqs; - let nb_sub_blocks = ((est_block_size + target_c_block_size / 2) / target_c_block_size).max(1); + let nb_sub_blocks = + ((est_block_size + target_c_block_size / 2) / target_c_block_size).max(1); let avg_block_budget = (est_block_size * BYTESCALE) / nb_sub_blocks; if est_block_size > src.len() { return Ok(0); @@ -544,7 +544,10 @@ fn compress_sub_block_multi( if write_lit_entropy { // The table was not written, so the next block cannot use it again. - next_c_block.entropy.huf.clone_from(&prev_c_block.entropy.huf); + next_c_block + .entropy + .huf + .clone_from(&prev_c_block.entropy.huf); } if write_seq_entropy && need_sequence_entropy_tables(&entropy_metadata.fse_metadata) { // Neither were these, and there is no going back to the previous ones. diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 0b5beda4b583..cfd310a9f876 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -254,7 +254,9 @@ pub fn frame_extent(src: &[u8]) -> Result { }; let mut ip = header.header_size; loop { - let Some(remaining) = src.get(ip..).filter(|rest| rest.len() >= ZSTD_BLOCKHEADERSIZE) + let Some(remaining) = src + .get(ip..) + .filter(|rest| rest.len() >= ZSTD_BLOCKHEADERSIZE) else { return Ok(FrameExtent::AtLeast(ip + ZSTD_BLOCKHEADERSIZE)); }; diff --git a/src/zstd/wip/PORTING.md b/src/zstd/wip/PORTING.md new file mode 100644 index 000000000000..007ba3a706e6 --- /dev/null +++ b/src/zstd/wip/PORTING.md @@ -0,0 +1,75 @@ +# Porting zstd 1.5.7 to safe Rust for Bun (`src/zstd`, crate `bun_zstd`) + +Goal: replace the C library with a port that gives **byte-for-byte the same compressed output, the same decompressed +output and the same error codes** as the C code, for every input. It is checked by differential tests against the C +library, so "equivalent but different" algorithms are wrong whenever they can change a single output byte. + +## Where things are + +- C source (pin f8745da6 = v1.5.7, with Bun's two patches): `/root/zstd-ref/zstd/lib/` +- The same without comments/asserts-noise, quicker to read: `/root/zstd-ref/stripped/` (read the original when a + comment would help to understand intent) +- The Rust crate: `/root/repos/oven-sh/bun/src/zstd/` (`common/`, `decompress/` are done and are the style reference; + `compress/internal.rs` is the port of `zstd_compress_internal.h` and is what the parts of the compressor share) +- Reference C static libs: `/root/zstd-ref/build/ref/libzstd.a` (as Bun built it), `/root/zstd-ref/build/noasm/` +- Differential harness (not part of the repo): `/root/zstd-ref/harness/` (`src/c.rs` has the FFI to the C lib). Build: + `. ~/.bun-dev-env && cd /root/zstd-ref/harness && RUSTFLAGS="--cap-lints=warn" cargo build --release` + It is built with overflow checks and debug assertions ON, on purpose. +- Type-check the crate: `. ~/.bun-dev-env && cd /root/repos/oven-sh/bun && cargo check -p bun_zstd` (dead-code errors + for items that nothing calls yet are expected while the port is incomplete; every other error is yours to fix). + Lints: `cargo clippy -p bun_zstd` must have no finding in your files other than dead code. + +## Hard rules + +1. `#![forbid(unsafe_code)]`. No `unsafe`, no `core::arch`, no `std::simd`, no new dependencies. +2. **No panics**: the crate denies `clippy::indexing_slicing`, `unwrap_used`, `expect_used`, `panic`, `unreachable`. + No `a[i]`, no `&a[i..j]`, no `.unwrap()`. Use `crate::util::SliceExt` (`at`, `at_mut`, `put`, `sub`, `sub_mut`, + `span`, `span_mut`, `array_at`, `array_at_mut`, `move_within`, `write_at`) and `read_le16/24/32/64`, + `write_le16/24/32/64`, all of which return `crate::error::Result` (`Err(ErrorCode::Generic)` = "broken + invariant, no input should cause it") and are used with `?`. Also avoid the std methods that panic + (`split_at`, `copy_from_slice` on lengths that are not provably equal, `copy_within`, `chunks(0)`, `/ 0`, ...). + Destructuring fixed arrays (`let [a, b, c] = arr;`), iterators, `zip`, `as_chunks`, `first_chunk`, `get` are good. + An index masked to a power-of-two array length (`arr.get(i & (N - 1))`) lets LLVM drop the check. +3. **No arithmetic overflow**: debug and fuzz builds have overflow checks, and a failed check aborts the process. + Wherever the C code relies on unsigned wrap-around (index arithmetic such as `curr - matchIndex`, `(U32)(a - b)`, + hash multiplications, `offset_1 > maxRep` tricks, `x - 1` on a possibly-0 value) use `wrapping_*`. Shifts by a + value that is not provably less than the width need a mask or `checked_sh*`. Casts: `as` truncates like C. + Keep the C integer widths (`U32` -> `u32`, `size_t` -> `usize`, `int` -> `i32`, `U64` -> `u64`, `BYTE` -> `u8`, + `S16`/`short` -> `i16`). All of Bun's targets are 64-bit little-endian, so drop `MEM_32bits()` branches and + big-endian branches. +4. Functions that return `size_t` with error codes return `Result`; `ERROR(x)` is `Err(ErrorCode::X)` + (`src/zstd/error.rs`). Keep the **same error code in the same order of checks**. Void C functions that need checked + accesses return `Result<()>`. +5. Pointers become positions (`usize`) into slices. Never keep a reference where C keeps a pointer into a buffer + that is mutated or that someone else owns: pass the slice down as a parameter. +6. Workspaces (`void* workSpace, size_t wkspSize`) become local arrays/structs or typed scratch structs. If a + `wkspSize` check can fail for some *input* (not just for a programming error) keep it, computed from the C sizes. +7. C "templates" (FORCE_INLINE_TEMPLATE functions specialised by constant arguments, macro-generated variants) become + `#[inline(always)]` generic functions with `const` generic parameters (or plain arguments that are constants at + the call sites), plus small non-inline wrappers/dispatch `match`es, so that each variant is compiled separately as + in C. +8. Drop what cannot be reached (listed per task), BMI2/ASM/NEON/SSE variants (keep the portable scalar code path that + gives the same result), tracing, DEBUGLOG, and `assert`s (a few `debug_assert!`s on real invariants are welcome). + PREFETCH is dropped. +9. Performance matters (this replaces optimised C): keep the structure of the hot loops, avoid allocation in them, + avoid bounds checks where a cheap restructuring (iterators, chunking, masks, hoisted sub-slices) removes them, mark + hot small helpers `#[inline(always)]`. But correctness and exact equivalence come first. + +## Style + +- Rust names in snake_case derived from the C names without the `ZSTD_`/`HUF_`/`FSE_`/`HIST_` prefix where the module + already says it (e.g. `HUF_buildCTable_wksp` -> `huf::build_ctable`). Every ported function gets a one-line doc + comment that starts with the C name in backticks: ``/// `HUF_buildCTable_wksp` `` plus a short sentence only if it + says something a reader needs (a precondition, what is returned). +- Items are `pub(crate)` at most (`unreachable_pub` is denied), private when only the module uses them. +- Comments: few, short, plain English, about the code as it is. Never mention the porting process, this + document, a conversation or "the original"; say "the C code" only when explaining a deliberate difference. + No commented-out code. No `TODO`. +- `rustfmt` (run `cargo fmt -p bun_zstd`), lines up to 120 columns in comments. `#[rustfmt::skip]` on big tables. +- Workspace lints are strict (see `[workspace.lints]` in `/root/repos/oven-sh/bun/Cargo.toml`), e.g. + `needless_pass_by_value`, `trivially_copy_pass_by_ref`, `large_types_passed_by_value`, `manual_memcpy`, + `redundant_clone`, `precedence`, `implicit_saturating_sub`, `manual_swap`; `too_many_arguments`, + `needless_range_loop` and `collapsible_if` are allowed. +- Do not edit files that are not yours (listed per task). If `compress/internal.rs`, `util.rs` or `common/` lacks + something you need, add a helper to your own file and say so in your report. +- Do not run `git commit`, `git add`, `git stash` or anything else that changes the repository's history or index. From 09ce1323122f3541d12419c38efea0fed3060db8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 12:32:15 +0000 Subject: [PATCH 005/199] zstd: the match finders and the entropy coders as verified against C --- src/zstd/compress/double_fast.rs | 8 +- src/zstd/compress/fast.rs | 6 +- src/zstd/compress/fse.rs | 16 +- src/zstd/compress/hist.rs | 20 +- src/zstd/compress/huf.rs | 261 +-- src/zstd/compress/lazy.rs | 52 +- src/zstd/wip/work/entropy/STATUS.md | 46 + .../work/entropy/zstd/compress/double_fast.rs | 48 - .../wip/work/entropy/zstd/compress/fast.rs | 48 - .../wip/work/entropy/zstd/compress/fse.rs | 678 ------- .../wip/work/entropy/zstd/compress/hist.rs | 132 -- .../wip/work/entropy/zstd/compress/huf.rs | 1162 ----------- .../work/entropy/zstd/compress/internal.rs | 1212 ------------ .../wip/work/entropy/zstd/compress/lazy.rs | 34 - .../wip/work/entropy/zstd/compress/ldm.rs | 1 - .../wip/work/entropy/zstd/compress/mod.rs | 68 - .../wip/work/entropy/zstd/compress/opt.rs | 29 - .../work/entropy/zstd/compress/presplit.rs | 1 - src/zstd/wip/work/fast/STATUS.md | 35 + .../work/fast/zstd/compress/double_fast.rs | 785 -------- src/zstd/wip/work/fast/zstd/compress/fast.rs | 829 -------- src/zstd/wip/work/fast/zstd/compress/fse.rs | 70 - src/zstd/wip/work/fast/zstd/compress/hist.rs | 33 - src/zstd/wip/work/fast/zstd/compress/huf.rs | 38 - .../wip/work/fast/zstd/compress/internal.rs | 1212 ------------ src/zstd/wip/work/fast/zstd/compress/lazy.rs | 34 - src/zstd/wip/work/fast/zstd/compress/ldm.rs | 1 - src/zstd/wip/work/fast/zstd/compress/mod.rs | 68 - src/zstd/wip/work/fast/zstd/compress/opt.rs | 29 - .../wip/work/fast/zstd/compress/presplit.rs | 1 - src/zstd/wip/work/lazy/STATUS.md | 53 + .../work/lazy/zstd/compress/double_fast.rs | 48 - src/zstd/wip/work/lazy/zstd/compress/fast.rs | 48 - src/zstd/wip/work/lazy/zstd/compress/fse.rs | 70 - src/zstd/wip/work/lazy/zstd/compress/hist.rs | 33 - src/zstd/wip/work/lazy/zstd/compress/huf.rs | 38 - .../wip/work/lazy/zstd/compress/internal.rs | 1212 ------------ src/zstd/wip/work/lazy/zstd/compress/lazy.rs | 1420 -------------- src/zstd/wip/work/lazy/zstd/compress/ldm.rs | 1 - src/zstd/wip/work/lazy/zstd/compress/mod.rs | 68 - src/zstd/wip/work/lazy/zstd/compress/opt.rs | 96 - .../wip/work/lazy/zstd/compress/presplit.rs | 1 - src/zstd/wip/work/ldm/snap/double_fast.rs | 785 -------- src/zstd/wip/work/ldm/snap/fast.rs | 829 -------- src/zstd/wip/work/ldm/snap/lazy.rs | 1353 ------------- src/zstd/wip/work/ldm/snap/opt.rs | 1721 ----------------- src/zstd/wip/work/ldm/stubs/double_fast.rs | 48 - src/zstd/wip/work/ldm/stubs/fast.rs | 48 - src/zstd/wip/work/ldm/stubs/lazy.rs | 34 - src/zstd/wip/work/ldm/stubs/opt.rs | 29 - .../wip/work/ldm/zstd/compress/double_fast.rs | 785 -------- src/zstd/wip/work/ldm/zstd/compress/fast.rs | 829 -------- src/zstd/wip/work/ldm/zstd/compress/fse.rs | 70 - src/zstd/wip/work/ldm/zstd/compress/hist.rs | 33 - src/zstd/wip/work/ldm/zstd/compress/huf.rs | 38 - .../wip/work/ldm/zstd/compress/internal.rs | 1212 ------------ src/zstd/wip/work/ldm/zstd/compress/lazy.rs | 1353 ------------- src/zstd/wip/work/ldm/zstd/compress/ldm.rs | 779 -------- src/zstd/wip/work/ldm/zstd/compress/mod.rs | 68 - src/zstd/wip/work/ldm/zstd/compress/opt.rs | 1721 ----------------- .../wip/work/ldm/zstd/compress/presplit.rs | 184 -- src/zstd/wip/work/opt/STATUS.md | 38 + .../wip/work/opt/zstd/compress/double_fast.rs | 48 - src/zstd/wip/work/opt/zstd/compress/fast.rs | 48 - src/zstd/wip/work/opt/zstd/compress/fse.rs | 70 - src/zstd/wip/work/opt/zstd/compress/hist.rs | 33 - src/zstd/wip/work/opt/zstd/compress/huf.rs | 38 - .../wip/work/opt/zstd/compress/internal.rs | 1212 ------------ src/zstd/wip/work/opt/zstd/compress/lazy.rs | 34 - src/zstd/wip/work/opt/zstd/compress/ldm.rs | 1 - src/zstd/wip/work/opt/zstd/compress/mod.rs | 68 - src/zstd/wip/work/opt/zstd/compress/opt.rs | 1721 ----------------- .../wip/work/opt/zstd/compress/presplit.rs | 1 - 73 files changed, 369 insertions(+), 24907 deletions(-) create mode 100644 src/zstd/wip/work/entropy/STATUS.md delete mode 100644 src/zstd/wip/work/entropy/zstd/compress/double_fast.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/compress/fast.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/compress/fse.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/compress/hist.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/compress/huf.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/compress/internal.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/compress/lazy.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/compress/ldm.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/compress/mod.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/compress/opt.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/compress/presplit.rs create mode 100644 src/zstd/wip/work/fast/STATUS.md delete mode 100644 src/zstd/wip/work/fast/zstd/compress/double_fast.rs delete mode 100644 src/zstd/wip/work/fast/zstd/compress/fast.rs delete mode 100644 src/zstd/wip/work/fast/zstd/compress/fse.rs delete mode 100644 src/zstd/wip/work/fast/zstd/compress/hist.rs delete mode 100644 src/zstd/wip/work/fast/zstd/compress/huf.rs delete mode 100644 src/zstd/wip/work/fast/zstd/compress/internal.rs delete mode 100644 src/zstd/wip/work/fast/zstd/compress/lazy.rs delete mode 100644 src/zstd/wip/work/fast/zstd/compress/ldm.rs delete mode 100644 src/zstd/wip/work/fast/zstd/compress/mod.rs delete mode 100644 src/zstd/wip/work/fast/zstd/compress/opt.rs delete mode 100644 src/zstd/wip/work/fast/zstd/compress/presplit.rs create mode 100644 src/zstd/wip/work/lazy/STATUS.md delete mode 100644 src/zstd/wip/work/lazy/zstd/compress/double_fast.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/compress/fast.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/compress/fse.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/compress/hist.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/compress/huf.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/compress/internal.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/compress/lazy.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/compress/ldm.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/compress/mod.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/compress/opt.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/compress/presplit.rs delete mode 100644 src/zstd/wip/work/ldm/snap/double_fast.rs delete mode 100644 src/zstd/wip/work/ldm/snap/fast.rs delete mode 100644 src/zstd/wip/work/ldm/snap/lazy.rs delete mode 100644 src/zstd/wip/work/ldm/snap/opt.rs delete mode 100644 src/zstd/wip/work/ldm/stubs/double_fast.rs delete mode 100644 src/zstd/wip/work/ldm/stubs/fast.rs delete mode 100644 src/zstd/wip/work/ldm/stubs/lazy.rs delete mode 100644 src/zstd/wip/work/ldm/stubs/opt.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/compress/double_fast.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/compress/fast.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/compress/fse.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/compress/hist.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/compress/huf.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/compress/internal.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/compress/lazy.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/compress/ldm.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/compress/mod.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/compress/opt.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/compress/presplit.rs create mode 100644 src/zstd/wip/work/opt/STATUS.md delete mode 100644 src/zstd/wip/work/opt/zstd/compress/double_fast.rs delete mode 100644 src/zstd/wip/work/opt/zstd/compress/fast.rs delete mode 100644 src/zstd/wip/work/opt/zstd/compress/fse.rs delete mode 100644 src/zstd/wip/work/opt/zstd/compress/hist.rs delete mode 100644 src/zstd/wip/work/opt/zstd/compress/huf.rs delete mode 100644 src/zstd/wip/work/opt/zstd/compress/internal.rs delete mode 100644 src/zstd/wip/work/opt/zstd/compress/lazy.rs delete mode 100644 src/zstd/wip/work/opt/zstd/compress/ldm.rs delete mode 100644 src/zstd/wip/work/opt/zstd/compress/mod.rs delete mode 100644 src/zstd/wip/work/opt/zstd/compress/opt.rs delete mode 100644 src/zstd/wip/work/opt/zstd/compress/presplit.rs diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 0b3fd60ef1d7..741c372c3cb6 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -115,7 +115,7 @@ fn compress_block_double_fast_no_dict_generic( let h_bits_l = ms.c_params.hash_log; let h_bits_s = ms.c_params.chain_log; let window_log = ms.c_params.window_log; - let buf = src.buf; + let buf = src.buf.sub(..src.end)?; let istart = src.start; let iend = src.end; let end_index = src.index_of(iend); @@ -295,7 +295,7 @@ fn compress_block_double_fast_dict_match_state_generic( ) -> Result { let h_bits_l = ms.c_params.hash_log; let h_bits_s = ms.c_params.chain_log; - let buf = src.buf; + let buf = src.buf.sub(..src.end)?; let istart = src.start; let iend = src.end; let end_index = src.index_of(iend); @@ -420,7 +420,7 @@ fn compress_block_double_fast_dict_match_state_generic( } } - let short_found = if match_index_s >= prefix_lowest_index { + let short_found = if match_index_s > prefix_lowest_index { read_le32(buf, src.pos_of(match_index_s))? == read_le32(buf, ip)? } else if dict_tags_match_s { let dict_match_index_s = @@ -598,7 +598,7 @@ fn compress_block_double_fast_ext_dict_generic( ) -> Result { let h_bits_l = ms.c_params.hash_log; let h_bits_s = ms.c_params.chain_log; - let buf = src.buf; + let buf = src.buf.sub(..src.end)?; let istart = src.start; let iend = src.end; let end_index = src.index_of(iend); diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 4bec128a6786..f543cad9ae16 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -184,7 +184,7 @@ fn compress_block_fast_no_dict_generic( let target_length = ms.c_params.target_length; // The least step is 2, as two positions are looked at per turn. let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; - let buf = src.buf; + let buf = src.buf.sub(..src.end)?; let istart = src.start; let iend = src.end; let end_index = src.index_of(iend); @@ -378,7 +378,7 @@ fn compress_block_fast_dict_match_state_generic( let hlog = ms.c_params.hash_log; let target_length = ms.c_params.target_length; let step_size = (target_length + u32::from(target_length == 0)) as usize; - let buf = src.buf; + let buf = src.buf.sub(..src.end)?; let istart = src.start; let iend = src.end; let prefix_start_index = ms.window.dict_limit; @@ -607,7 +607,7 @@ fn compress_block_fast_ext_dict_generic( let hlog = ms.c_params.hash_log; let target_length = ms.c_params.target_length; let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; - let buf = src.buf; + let buf = src.buf.sub(..src.end)?; let istart = src.start; let iend = src.end; let end_index = src.index_of(iend); diff --git a/src/zstd/compress/fse.rs b/src/zstd/compress/fse.rs index 82a4a8821b92..7d57ceda36e4 100644 --- a/src/zstd/compress/fse.rs +++ b/src/zstd/compress/fse.rs @@ -69,8 +69,10 @@ impl FseCTable { let step = fse_table_step(table_size); let counts = normalized_counter.sub(..=max_symbol_value as usize)?; - let mut cumul = [0u16; MAX_SEQ as usize + 1]; - let mut table_symbol = [0u8; 1 << ML_FSE_LOG]; + const STATE_TABLE_SIZE: usize = 1 << ML_FSE_LOG; + const CUMUL_SIZE: usize = (MAX_SEQ as usize + 1).next_power_of_two(); + let mut cumul = [0u16; CUMUL_SIZE]; + let mut table_symbol = [0u8; STATE_TABLE_SIZE]; let mut high_threshold = table_size - 1; self.table_log = table_log; @@ -127,10 +129,14 @@ impl FseCTable { // The states, sorted by symbol. for (u, &symbol) in table_symbol.sub(..table_size)?.iter().enumerate() { - let next = cumul.at_mut(usize::from(symbol))?; - self.state_table - .put(usize::from(*next), (table_size + u) as u16)?; + let Some(next) = cumul.get_mut(usize::from(symbol) & (CUMUL_SIZE - 1)) else { + continue; + }; + let state = usize::from(*next) & (STATE_TABLE_SIZE - 1); *next = next.wrapping_add(1); + if let Some(state) = self.state_table.get_mut(state) { + *state = (table_size + u) as u16; + } } let mut total = 0u32; diff --git a/src/zstd/compress/hist.rs b/src/zstd/compress/hist.rs index 472292b90e2c..745fc94e641f 100644 --- a/src/zstd/compress/hist.rs +++ b/src/zstd/compress/hist.rs @@ -60,28 +60,30 @@ fn count_parallel( let mut counting4 = [0u32; 256]; // Each word is read before the bytes of the one before it are counted. - let (words, rest) = src.as_chunks::<4>(); + let (words, rest) = src.as_chunks::<8>(); if let Some((&first, words)) = words.split_first() { - let mut cached = u32::from_le_bytes(first); - let mut count_word = |next: [u8; 4]| { + let mut cached = u64::from_le_bytes(first); + let mut count_word = |next: [u8; 8]| { let c = cached; - cached = u32::from_le_bytes(next); + cached = u64::from_le_bytes(next); increment(&mut counting1, c as u8); increment(&mut counting2, (c >> 8) as u8); increment(&mut counting3, (c >> 16) as u8); increment(&mut counting4, (c >> 24) as u8); + increment(&mut counting1, (c >> 32) as u8); + increment(&mut counting2, (c >> 40) as u8); + increment(&mut counting3, (c >> 48) as u8); + increment(&mut counting4, (c >> 56) as u8); }; - let (stripes, words) = words.as_chunks::<4>(); - for &[a, b, c, d] in stripes { + let (stripes, words) = words.as_chunks::<2>(); + for &[a, b] in stripes { count_word(a); count_word(b); - count_word(c); - count_word(d); } for &word in words { count_word(word); } - count_word([0; 4]); + count_word([0; 8]); } for &byte in rest { increment(&mut counting1, byte); diff --git a/src/zstd/compress/huf.rs b/src/zstd/compress/huf.rs index 85fe95701c70..25db9d506b29 100644 --- a/src/zstd/compress/huf.rs +++ b/src/zstd/compress/huf.rs @@ -6,14 +6,12 @@ use super::internal::Strategy; use crate::common::entropy::huf_read_stats; use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, highbit32, write_le16}; +use crate::util::{BUG, SliceExt, highbit32, write_le16}; pub(crate) const HUF_BLOCKSIZE_MAX: usize = 128 * 1024; -pub(crate) const HUF_WORKSPACE_SIZE: usize = (8 << 10) + 512; pub(crate) const HUF_TABLELOG_DEFAULT: u32 = 11; /// The strategies from this one on look for the best depth of the tree. pub(crate) const HUF_OPTIMAL_DEPTH_THRESHOLD: Strategy = Strategy::BtUltra; -const HUF_CTABLEBOUND: usize = 129; /// `HUF_flags_e` #[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] @@ -292,41 +290,74 @@ struct NodeElt { const STARTNODE: i32 = HUF_SYMBOLVALUE_MAX as i32 + 1; /// `huffNodeTable`: the leaves from 0 on, sorted from the most frequent symbol to the rarest, the other nodes from -/// [`STARTNODE`] on, and a sentinel at -1. +/// [`STARTNODE`] on, and a sentinel at -1. A number that is out of the table wraps around, which spares the checks. struct HuffNodes([NodeElt; 2 * (HUF_SYMBOLVALUE_MAX as usize + 1)]); impl HuffNodes { #[inline(always)] - fn at(&self, n: i32) -> Result { - self.0.at(n.wrapping_add(1) as usize) + const fn index(n: i32) -> usize { + n.wrapping_add(1) as usize & (2 * (HUF_SYMBOLVALUE_MAX as usize + 1) - 1) } #[inline(always)] - fn at_mut(&mut self, n: i32) -> Result<&mut NodeElt> { - self.0.at_mut(n.wrapping_add(1) as usize) + fn get(&self, n: i32) -> NodeElt { + self.0.get(Self::index(n)).copied().unwrap_or_default() } #[inline(always)] - fn nb_bits(&self, n: i32) -> Result { - Ok(u32::from(self.at(n)?.nb_bits)) + fn set(&mut self, n: i32, node: NodeElt) { + if let Some(slot) = self.0.get_mut(Self::index(n)) { + *slot = node; + } + } + + #[inline(always)] + fn count(&self, n: i32) -> u32 { + self.get(n).count + } + + #[inline(always)] + fn nb_bits(&self, n: i32) -> u32 { + u32::from(self.get(n).nb_bits) + } + + #[inline(always)] + fn set_count(&mut self, n: i32, count: u32) { + if let Some(node) = self.0.get_mut(Self::index(n)) { + node.count = count; + } + } + + #[inline(always)] + fn set_parent(&mut self, n: i32, parent: i32) { + if let Some(node) = self.0.get_mut(Self::index(n)) { + node.parent = parent as u16; + } + } + + #[inline(always)] + fn set_nb_bits(&mut self, n: i32, nb_bits: u32) { + if let Some(node) = self.0.get_mut(Self::index(n)) { + node.nb_bits = nb_bits as u8; + } } /// The nodes from 0 on. #[inline(always)] - fn from_0(&mut self) -> &mut [NodeElt] { - let [_, nodes @ ..] = &mut self.0; + fn leaves(&self) -> &[NodeElt] { + let [_, nodes @ ..] = &self.0; nodes } } -/// `HUF_setMaxHeight`: makes no code longer than `target_nb_bits`, which is enough for a code for every symbol. -/// `last_non_null` is the rarest symbol, which has the longest code. Returns the length of the longest code. +/// `HUF_setMaxHeight`: makes no code longer than `target_nb_bits`, which is not 0 and is enough for a code for every +/// symbol. `last_non_null` is the rarest symbol, which has the longest code. Returns the length of the longest code. fn set_max_height( huff_node: &mut HuffNodes, last_non_null: u32, target_nb_bits: u32, ) -> Result { - let largest_bits = huff_node.nb_bits(last_non_null as i32)?; + let largest_bits = huff_node.nb_bits(last_non_null as i32); if largest_bits <= target_nb_bits { return Ok(largest_bits); } @@ -338,15 +369,14 @@ fn set_max_height( let mut n = last_non_null as i32; // Shorten the codes that are too long. `total_cost` is how much more than the whole code space is taken then. - while huff_node.nb_bits(n)? > target_nb_bits { - let node = huff_node.at_mut(n)?; - let cost = base_cost.wrapping_sub(cost_of(u32::from(node.nb_bits))); + while huff_node.nb_bits(n) > target_nb_bits { + let cost = base_cost.wrapping_sub(cost_of(huff_node.nb_bits(n))); total_cost = total_cost.wrapping_add(cost as i32); - node.nb_bits = target_nb_bits as u8; + huff_node.set_nb_bits(n, target_nb_bits); n -= 1; } - // `n` ends at the rarest symbol with a code shorter than `target_nb_bits`. - while huff_node.nb_bits(n)? == target_nb_bits { + // `n` ends at the rarest symbol with a code shorter than `target_nb_bits`, or at the sentinel. + while huff_node.nb_bits(n) == target_nb_bits { n -= 1; } @@ -360,7 +390,7 @@ fn set_max_height( let mut current_nb_bits = target_nb_bits; for pos in (0..=n).rev() { - let nb_bits = huff_node.nb_bits(pos)?; + let nb_bits = huff_node.nb_bits(pos); if nb_bits >= current_nb_bits { continue; } @@ -379,8 +409,8 @@ fn set_max_height( break; } // It is cheaper to lengthen this one than two of the rank below. - let high_total = huff_node.at(high_pos as i32)?.count; - let low_total = huff_node.at(low_pos as i32)?.count.wrapping_mul(2); + let high_total = huff_node.count(high_pos as i32); + let low_total = huff_node.count(low_pos as i32).wrapping_mul(2); if high_total <= low_total { break; } @@ -393,9 +423,11 @@ fn set_max_height( n_bits_to_decrease += 1; } let pos = rank_last.at(n_bits_to_decrease)?; + if pos == NO_SYMBOL { + return Err(BUG); + } total_cost -= 1 << (n_bits_to_decrease - 1); - let node = huff_node.at_mut(pos as i32)?; - node.nb_bits = node.nb_bits.wrapping_add(1); + huff_node.set_nb_bits(pos as i32, huff_node.nb_bits(pos as i32) + 1); // The symbol is now of the rank below, where it is the rarest only if there was none. let below = rank_last.at_mut(n_bits_to_decrease - 1)?; @@ -404,7 +436,7 @@ fn set_max_height( } // The next symbol takes its place, if it is of the same rank. let same_rank = target_nb_bits.wrapping_sub(n_bits_to_decrease as u32); - let next = if pos != 0 && huff_node.nb_bits(pos as i32 - 1)? == same_rank { + let next = if pos != 0 && huff_node.nb_bits(pos as i32 - 1) == same_rank { pos - 1 } else { NO_SYMBOL @@ -416,18 +448,14 @@ fn set_max_height( let [_, rank_last_1, ..] = &mut rank_last; while total_cost < 0 { if *rank_last_1 == NO_SYMBOL { - while huff_node.nb_bits(n)? == target_nb_bits { + while huff_node.nb_bits(n) == target_nb_bits { n -= 1; } - let node = huff_node.at_mut(n + 1)?; - node.nb_bits = node.nb_bits.wrapping_sub(1); - *rank_last_1 = (n + 1) as u32; - total_cost += 1; - continue; + *rank_last_1 = n as u32; } - let node = huff_node.at_mut(rank_last_1.wrapping_add(1) as i32)?; - node.nb_bits = node.nb_bits.wrapping_sub(1); - *rank_last_1 += 1; + *rank_last_1 = rank_last_1.wrapping_add(1); + let pos = *rank_last_1 as i32; + huff_node.set_nb_bits(pos, huff_node.nb_bits(pos).wrapping_sub(1)); total_cost += 1; } @@ -462,65 +490,62 @@ fn get_index(count: u32) -> usize { /// `HUF_swapNodes` #[inline(always)] -fn swap_nodes(arr: &mut [NodeElt], a: i32, b: i32) -> Result<()> { - let tmp = arr.at(a as usize)?; - arr.put(a as usize, arr.at(b as usize)?)?; - arr.put(b as usize, tmp) +fn swap_nodes(arr: &mut HuffNodes, a: i32, b: i32) { + let tmp = arr.get(a); + arr.set(a, arr.get(b)); + arr.set(b, tmp); } /// `HUF_insertionSort`: by decreasing count, from `low` to `high` included. #[inline] -fn insertion_sort(huff_node: &mut [NodeElt], low: i32, high: i32) -> Result<()> { - let size = high - low + 1; - let huff_node = huff_node.sub_mut(low as usize..)?; - for i in 1..size { - let key = huff_node.at(i as usize)?; +fn insertion_sort(huff_node: &mut HuffNodes, low: i32, high: i32) { + for i in low + 1..=high { + let key = huff_node.get(i); let mut j = i - 1; - while j >= 0 && huff_node.at(j as usize)?.count < key.count { - huff_node.put((j + 1) as usize, huff_node.at(j as usize)?)?; + while j >= low && huff_node.count(j) < key.count { + huff_node.set(j + 1, huff_node.get(j)); j -= 1; } - huff_node.put((j + 1) as usize, key)?; + huff_node.set(j + 1, key); } - Ok(()) } /// `HUF_quickSortPartition`: the pivot is the last element. Returns where it ends up. -fn quick_sort_partition(arr: &mut [NodeElt], low: i32, high: i32) -> Result { - let pivot = arr.at(high as usize)?.count; +fn quick_sort_partition(arr: &mut HuffNodes, low: i32, high: i32) -> i32 { + let pivot = arr.count(high); let mut i = low - 1; for j in low..high { - if arr.at(j as usize)?.count > pivot { + if arr.count(j) > pivot { i += 1; - swap_nodes(arr, i, j)?; + swap_nodes(arr, i, j); } } - swap_nodes(arr, i + 1, high)?; - Ok(i + 1) + swap_nodes(arr, i + 1, high); + i + 1 } /// `HUF_simpleQuickSort`: by decreasing count, from `low` to `high` included. It recurses into the smaller part only, /// which bounds the depth. The order it leaves symbols of the same count in decides their codes. -fn simple_quick_sort(arr: &mut [NodeElt], mut low: i32, mut high: i32) -> Result<()> { +fn simple_quick_sort(arr: &mut HuffNodes, mut low: i32, mut high: i32) { const INSERTION_SORT_THRESHOLD: i32 = 8; if high - low < INSERTION_SORT_THRESHOLD { - return insertion_sort(arr, low, high); + insertion_sort(arr, low, high); + return; } while low < high { - let idx = quick_sort_partition(arr, low, high)?; + let idx = quick_sort_partition(arr, low, high); if idx - low < high - idx { - simple_quick_sort(arr, low, idx - 1)?; + simple_quick_sort(arr, low, idx - 1); low = idx + 1; } else { - simple_quick_sort(arr, idx + 1, high)?; + simple_quick_sort(arr, idx + 1, high); high = idx - 1; } } - Ok(()) } -/// `HUF_sort`: fills `count` and `byte` of the first nodes with the symbols, by decreasing count. -fn sort(huff_node: &mut [NodeElt], count: &[u32]) -> Result<()> { +/// `HUF_sort`: makes the first nodes, which are all zeros, the symbols by decreasing count. At most 256 counts. +fn sort(huff_node: &mut HuffNodes, count: &[u32]) -> Result<()> { let mut rank_position = [RankPos::default(); RANK_POSITION_TABLE_SIZE]; // A bucket starts after the symbols of all the buckets above it. @@ -536,19 +561,19 @@ fn sort(huff_node: &mut [NodeElt], count: &[u32]) -> Result<()> { for (n, &c) in count.iter().enumerate() { let rank = rank_position.at_mut(get_index(c) + 1)?; - let pos = usize::from(rank.curr); + let node = NodeElt { + count: c, + byte: n as u8, + ..NodeElt::default() + }; + huff_node.set(i32::from(rank.curr), node); rank.curr += 1; - let node = huff_node.at_mut(pos)?; - node.count = c; - node.byte = n as u8; } let shared = RANK_POSITION_DISTINCT_COUNT_CUTOFF as usize..RANK_POSITION_TABLE_SIZE - 1; for rank in rank_position.sub(shared)? { - let bucket_size = i32::from(rank.curr) - i32::from(rank.base); - if bucket_size > 1 { - let bucket = huff_node.sub_mut(usize::from(rank.base)..)?; - simple_quick_sort(bucket, 0, bucket_size - 1)?; + if rank.curr > rank.base + 1 { + simple_quick_sort(huff_node, i32::from(rank.base), i32::from(rank.curr) - 1); } } Ok(()) @@ -557,56 +582,50 @@ fn sort(huff_node: &mut [NodeElt], count: &[u32]) -> Result<()> { /// `HUF_buildTree`: `huff_node` is sorted, and some count is not 0. Sets the depth of every leaf, and returns the /// rarest symbol whose count is not 0. fn build_tree(huff_node: &mut HuffNodes, max_symbol_value: u32) -> Result { - let mut non_null_rank = max_symbol_value as i32; - while huff_node.at(non_null_rank)?.count == 0 { - non_null_rank -= 1; - } + let mut symbols = (0..=max_symbol_value as i32).rev(); + let non_null_rank = symbols.find(|&n| huff_node.count(n) != 0).ok_or(BUG)?; let mut low_s = non_null_rank; let mut node_nb = STARTNODE; let node_root = node_nb + low_s - 1; let mut low_n = node_nb; - let mut join = |huff_node: &mut HuffNodes, n1: i32, n2: i32| -> Result<()> { - let count = huff_node - .at(n1)? - .count - .wrapping_add(huff_node.at(n2)?.count); - huff_node.at_mut(node_nb)?.count = count; - huff_node.at_mut(n1)?.parent = node_nb as u16; - huff_node.at_mut(n2)?.parent = node_nb as u16; + let mut join = |huff_node: &mut HuffNodes, n1: i32, n2: i32| { + let count = huff_node.count(n1).wrapping_add(huff_node.count(n2)); + huff_node.set_count(node_nb, count); + huff_node.set_parent(n1, node_nb); + huff_node.set_parent(n2, node_nb); node_nb += 1; - Ok(()) }; - join(huff_node, low_s, low_s - 1)?; + join(huff_node, low_s, low_s - 1); low_s -= 2; for n in STARTNODE + 1..=node_root { - huff_node.at_mut(n)?.count = 1 << 30; + huff_node.set_count(n, 1 << 30); } // The sentinel is never the smaller of the two. - huff_node.at_mut(-1)?.count = 1 << 31; + huff_node.set_count(-1, 1 << 31); // Join the two nodes of the lowest counts, which are the next leaves or the next parents. for _ in STARTNODE + 1..=node_root { - let mut lowest = |huff_node: &HuffNodes| -> Result { - Ok(if huff_node.at(low_s)?.count < huff_node.at(low_n)?.count { + let mut lowest = |huff_node: &HuffNodes| { + if huff_node.count(low_s) < huff_node.count(low_n) { low_s -= 1; low_s + 1 } else { low_n += 1; low_n - 1 - }) + } }; - let n1 = lowest(huff_node)?; - let n2 = lowest(huff_node)?; - join(huff_node, n1, n2)?; + let n1 = lowest(huff_node); + let n2 = lowest(huff_node); + join(huff_node, n1, n2); } // The parents come after their children, so that this goes from the root down. - huff_node.at_mut(node_root)?.nb_bits = 0; + huff_node.set_nb_bits(node_root, 0); for n in (STARTNODE..node_root).rev().chain(0..=non_null_rank) { - let parent = i32::from(huff_node.at(n)?.parent); - huff_node.at_mut(n)?.nb_bits = huff_node.at(parent)?.nb_bits.wrapping_add(1); + let parent = i32::from(huff_node.get(n).parent); + huff_node.set_nb_bits(n, huff_node.nb_bits(parent) + 1); } Ok(non_null_rank) @@ -650,7 +669,7 @@ pub(crate) fn build_ctable( } let mut huff_node = HuffNodes([NodeElt::default(); _]); - sort(huff_node.from_0(), count.sub(..=max_symbol_value as usize)?)?; + sort(&mut huff_node, count.sub(..=max_symbol_value as usize)?)?; let non_null_rank = build_tree(&mut huff_node, max_symbol_value)?; max_nb_bits = set_max_height(&mut huff_node, non_null_rank as u32, max_nb_bits)?; if max_nb_bits > HUF_TABLELOG_MAX { @@ -658,7 +677,7 @@ pub(crate) fn build_ctable( } build_ctable_from_tree( ctable, - huff_node.from_0(), + huff_node.leaves(), non_null_rank, max_symbol_value, max_nb_bits, @@ -689,11 +708,6 @@ pub(crate) fn validate_ctable(ctable: &HufCTable, count: &[u32], max_symbol_valu .all(|(&elt, &count)| count == 0 || get_nb_bits(elt) != 0) } -/// `HUF_compressBound` -pub(crate) const fn compress_bound(size: usize) -> usize { - HUF_CTABLEBOUND + size + (size >> 8) + 8 -} - /// One `bitContainer` of `HUF_CStream_t` and its `bitPos`. #[derive(Clone, Copy, Default)] struct BitContainer { @@ -752,21 +766,20 @@ impl<'a> HufCStream<'a> { self.container.pos = self.container.pos.wrapping_add(second.pos); } - /// `HUF_flushBits`: writes the whole bytes. With `FAST`, the caller knows that the output is large enough. - /// Without, the stream stops advancing at the end of the output, which [`Self::close`] reports. + /// `HUF_flushBits`: writes the whole bytes. At the end of the output the stream stops advancing, which + /// [`Self::close`] reports. The C code has a variant without that limit for when the output is known to be large + /// enough, but here the limit is what spares a check on the write. #[inline(always)] - fn flush_bits(&mut self) { + fn flush_bits(&mut self) { let nb_bits = self.container.nb_bits(); let nb_bytes = (nb_bits >> 3) as usize; let bits = self.container.bits >> (u64::from(u64::BITS).wrapping_sub(nb_bits) & 63); self.container.pos &= 7; - if let Ok(out) = self.dst.array_at_mut(self.pos) { + let pos = self.pos.min(self.end); + if let Ok(out) = self.dst.array_at_mut(pos) { *out = bits.to_le_bytes(); } - self.pos += nb_bytes; - if !FAST && self.pos > self.end { - self.pos = self.end; - } + self.pos = pos + nb_bytes; } /// `HUF_closeCStream`: the size of the stream, or 0 if it did not fit. @@ -775,7 +788,7 @@ impl<'a> HufCStream<'a> { /// `HUF_endMark`: one bit, set. const END_MARK: u64 = 1 | (1 << 63); self.container.add_bits::(END_MARK); - self.flush_bits::(); + self.flush_bits(); if self.pos >= self.end { return 0; } @@ -809,7 +822,7 @@ fn encode_symbols( /// `HUF_compress1X_usingCTable_internal_body_loop`: a container has room for `UNROLL` symbols on top of the 7 bits /// that a flush can leave, and, with `LAST_FAST`, for 4 bits more. #[inline(always)] -fn compress_1x_body_loop( +fn compress_1x_body_loop( bit_c: &mut HufCStream<'_>, src: &[u8], ct: &Elements, @@ -819,21 +832,21 @@ fn compress_1x_body_loop(&mut bit_c.container, symbol, ct); } - bit_c.flush_bits::(); + bit_c.flush_bits(); } let (pairs, odd) = groups.as_chunks::<2>(); if let [group] = odd { encode_symbols::(&mut bit_c.container, group, ct); - bit_c.flush_bits::(); + bit_c.flush_bits(); } for [second, first] in pairs.iter().rev() { encode_symbols::(&mut bit_c.container, first, ct); - bit_c.flush_bits::(); + bit_c.flush_bits(); // The second group does not depend on the flush, since it has a container of its own. let mut container = BitContainer::default(); encode_symbols::(&mut container, second, ct); bit_c.merge(container); - bit_c.flush_bits::(); + bit_c.flush_bits(); } } @@ -859,15 +872,15 @@ pub(crate) fn compress_1x_using_ctable( let bit_c_ref = &mut bit_c; if dst_size < tight_compress_bound(src.len(), usize::from(table_log)) || table_log > 11 { - compress_1x_body_loop::<4, false, false>(bit_c_ref, src, ct); + compress_1x_body_loop::<4, false>(bit_c_ref, src, ct); } else { match table_log { - 11 => compress_1x_body_loop::<5, true, false>(bit_c_ref, src, ct), - 10 => compress_1x_body_loop::<5, true, true>(bit_c_ref, src, ct), - 9 => compress_1x_body_loop::<6, true, false>(bit_c_ref, src, ct), - 8 => compress_1x_body_loop::<7, true, false>(bit_c_ref, src, ct), - 7 => compress_1x_body_loop::<8, true, false>(bit_c_ref, src, ct), - _ => compress_1x_body_loop::<9, true, true>(bit_c_ref, src, ct), + 11 => compress_1x_body_loop::<5, false>(bit_c_ref, src, ct), + 10 => compress_1x_body_loop::<5, true>(bit_c_ref, src, ct), + 9 => compress_1x_body_loop::<6, false>(bit_c_ref, src, ct), + 8 => compress_1x_body_loop::<7, false>(bit_c_ref, src, ct), + 7 => compress_1x_body_loop::<8, false>(bit_c_ref, src, ct), + _ => compress_1x_body_loop::<9, true>(bit_c_ref, src, ct), } } Ok(bit_c.close()) diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 5a4f9eaf7a53..c19d790c6159 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -1,5 +1,7 @@ //! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. +use core::hint::{Locality, prefetch_read}; + use super::internal::{ BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, @@ -17,6 +19,12 @@ const NO_DICT: u8 = DictMode::NoDict as u8; const EXT_DICT: u8 = DictMode::ExtDict as u8; const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; +/// `PREFETCH_L1`: a hint that `slice[index]` is about to be read. `index` may be anything. +#[inline(always)] +fn prefetch_l1(slice: &[T], index: usize) { + prefetch_read(slice.as_ptr().wrapping_add(index), Locality::L1); +} + /// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. #[inline(always)] const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { @@ -586,11 +594,7 @@ fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { next } -#[inline(always)] -fn prefetch_l1(slice: &[T], index: usize) { - core::hint::prefetch_read(slice.as_ptr().wrapping_add(index), core::hint::Locality::L1); -} - +/// `ZSTD_row_prefetch`: of the row that starts at `rel_row`. #[inline(always)] fn row_prefetch(hash_table: &[u32], tag_table: &[u8], rel_row: usize, row_log: u32) { prefetch_l1(hash_table, rel_row); @@ -608,6 +612,7 @@ fn row_prefetch(hash_table: &[u32], tag_table: &[u8], rel_row: usize, row_log: u fn row_fill_hash_cache( ms: &mut MatchState, prefix: Prefix<'_>, + row_log: u32, mls: u32, idx: u32, i_limit: usize, @@ -624,6 +629,8 @@ fn row_fill_hash_cache( mls, ms.hash_salt, )? as u32; + let row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + row_prefetch(&ms.hash_table, &ms.tag_table, row, row_log); let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; ms.hash_cache.put(cached, hash)?; } @@ -637,10 +644,10 @@ fn row_next_cached_hash( cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], hash_table: &[u32], tag_table: &[u8], - row_log: u32, prefix: Prefix<'_>, idx: u32, hash_log: u32, + row_log: u32, mls: u32, hash_salt: u64, ) -> Result { @@ -651,12 +658,8 @@ fn row_next_cached_hash( mls, hash_salt, )? as u32; - row_prefetch( - hash_table, - tag_table, - ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize, - row_log, - ); + let row = ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + row_prefetch(hash_table, tag_table, row, row_log); let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; Ok(core::mem::replace(cached, new_hash)) } @@ -684,10 +687,10 @@ fn row_update_internal_impl( &mut ms.hash_cache, hash_table, tag_table, - row_log, prefix, idx, hash_log, + row_log, mls, hash_salt, )? @@ -731,7 +734,7 @@ fn row_update_internal( let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; - row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; + row_fill_hash_cache(ms, prefix, row_log, mls, idx, ip + 1)?; } row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; ms.next_to_update = target; @@ -772,7 +775,7 @@ fn row_get_match_mask( /// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, /// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than -/// `low_limit`. Returns how many there are. +/// `low_limit`. `prefetch` is called with each. Returns how many there are. #[inline(always)] fn row_collect_matches( row: &[u32; ROW_ENTRIES], @@ -833,6 +836,16 @@ fn row_find_best_match> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; + row_prefetch(&dms.ms.hash_table, &dms.ms.tag_table, dms_rel_row, row_log); + } + // Update the tables up to, and not including, `ip`. let hash = if ms.lazy_skipping { // While skipping, not every position is inserted. @@ -845,10 +858,10 @@ fn row_find_best_match> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; let num_matches = row_collect_matches( dms.ms.hash_table.array_at::(dms_rel_row)?, dms.ms.tag_table.array_at::(dms_rel_row)?, @@ -1155,7 +1165,7 @@ fn compress_block_lazy_generic [all|hist|fse_logs|fse|fse_tables|fse_bad_norm|huf_table|huf_read|huf_repeat] +ENTROPY_TRACE=1 ... # prints the seed of each case +# speed, C against Rust. Its own target dir: the `bench` profile writes to target/release and would replace the checked build. +CARGO_TARGET_DIR=$PWD/target-bench RUSTFLAGS="--cap-lints=warn" cargo build --profile bench +./target-bench/release/zstd_harness entropy-bench +``` + +## Not reached by the tests (all are broken-invariant or unreachable paths) +- `normalize_m2`: `to_distribute == 0` (cannot happen once `min_table_log` has passed), `weight < 1` -> Generic, and my guard + against an endless loop (`no norm > 0` -> Generic). +- `read_ctable`: `table_log > 12` (`huf_read_stats` refuses it first). +- `optimal_table_log`: `build_ctable` failing inside the search. + +## Open +- Speed, not correctness (see the report): Huffman encoding 1.08-1.20x the C time (C without BMI2), histogram 1.08x, + `build_ctable` 1.07-1.28x, `write_ctable` 1.25x, `write_ncount` 1.35x (50 ns), `FseCTable::build` 1.0x, + three-state FSE encoding 1.34x, which falls to 1.20x with a 3-line change to `BitCStream::flush` in + `common/bitstream.rs` (not my file): `let pos = self.pos.min(self.end);` write at `pos`, `self.pos = pos + nb_bytes;`. + LLVM then drops the bounds check on the write. The same trick is in `HufCStream::flush_bits`. +- The last speed numbers for `build_ctable` and the Huffman encoder were measured on a machine under load (load average 17). diff --git a/src/zstd/wip/work/entropy/zstd/compress/double_fast.rs b/src/zstd/wip/work/entropy/zstd/compress/double_fast.rs deleted file mode 100644 index 65d401675d1b..000000000000 --- a/src/zstd/wip/work/entropy/zstd/compress/double_fast.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. - -use super::internal::{ - BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_double_hash_table( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _end: usize, - _dtlm: DictTableLoadMethod, - _tfp: TableFillPurpose, -) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast` -pub(crate) fn compress_block_double_fast( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast_dictMatchState` -pub(crate) fn compress_block_double_fast_dict_match_state( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast_extDict` -pub(crate) fn compress_block_double_fast_ext_dict( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/entropy/zstd/compress/fast.rs b/src/zstd/wip/work/entropy/zstd/compress/fast.rs deleted file mode 100644 index 1f4788af28b4..000000000000 --- a/src/zstd/wip/work/entropy/zstd/compress/fast.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. - -use super::internal::{ - BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_hash_table( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _end: usize, - _dtlm: DictTableLoadMethod, - _tfp: TableFillPurpose, -) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast` -pub(crate) fn compress_block_fast( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast_dictMatchState` -pub(crate) fn compress_block_fast_dict_match_state( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast_extDict` -pub(crate) fn compress_block_fast_ext_dict( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/entropy/zstd/compress/fse.rs b/src/zstd/wip/work/entropy/zstd/compress/fse.rs deleted file mode 100644 index 7d57ceda36e4..000000000000 --- a/src/zstd/wip/work/entropy/zstd/compress/fse.rs +++ /dev/null @@ -1,678 +0,0 @@ -//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. - -use crate::common::bitstream::BitCStream; -use crate::common::{FSE_MAX_TABLELOG, FSE_MIN_TABLELOG, MAX_SEQ, ML_FSE_LOG, fse_table_step}; -use crate::error::{ErrorCode, Result}; -use crate::util::{BUG, SliceExt, highbit32, write_le16}; - -pub(crate) const FSE_NCOUNTBOUND: usize = 512; -const FSE_DEFAULT_TABLELOG: u32 = 11; - -/// `FSE_BLOCKBOUND` -const fn block_bound(size: usize) -> usize { - size + (size >> 7) + 4 + size_of::() -} - -/// `FSE_repeat` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum FseRepeat { - /// The table cannot be used again. - None, - /// It can, if it has a code for every symbol to encode. - Check, - /// It can. - Valid, -} - -/// `FSE_symbolCompressionTransform` -#[derive(Clone, Copy, Default)] -pub(crate) struct FseSymbolTransform { - pub(crate) delta_find_state: i32, - pub(crate) delta_nb_bits: u32, -} - -/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. -#[derive(Clone)] -pub(crate) struct FseCTable { - pub(crate) table_log: u32, - pub(crate) max_symbol_value: u32, - pub(crate) state_table: [u16; 1 << ML_FSE_LOG], - pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], -} - -impl FseCTable { - pub(crate) const fn new() -> Self { - Self { - table_log: 0, - max_symbol_value: 0, - state_table: [0; 1 << ML_FSE_LOG], - symbol_tt: [FseSymbolTransform { - delta_find_state: 0, - delta_nb_bits: 0, - }; MAX_SEQ as usize + 1], - } - } - - /// `FSE_buildCTable_wksp`: `normalized_counter` adds up to `1 << table_log`, a count of -1 standing for 1. - /// [`ErrorCode::TableLogTooLarge`] if the table has no room for it. - pub(crate) fn build( - &mut self, - normalized_counter: &[i16], - max_symbol_value: u32, - table_log: u32, - ) -> Result<()> { - if table_log > ML_FSE_LOG || max_symbol_value > MAX_SEQ { - return Err(ErrorCode::TableLogTooLarge); - } - let table_size = 1usize << table_log; - let table_mask = table_size - 1; - let step = fse_table_step(table_size); - let counts = normalized_counter.sub(..=max_symbol_value as usize)?; - - const STATE_TABLE_SIZE: usize = 1 << ML_FSE_LOG; - const CUMUL_SIZE: usize = (MAX_SEQ as usize + 1).next_power_of_two(); - let mut cumul = [0u16; CUMUL_SIZE]; - let mut table_symbol = [0u8; STATE_TABLE_SIZE]; - let mut high_threshold = table_size - 1; - - self.table_log = table_log; - self.max_symbol_value = max_symbol_value; - - // Where the states of each symbol start. The symbols of low probability get the highest positions. - let mut total = 0u16; - for (symbol, (&count, start)) in counts.iter().zip(&mut cumul).enumerate() { - *start = total; - if count == -1 { - total = total.wrapping_add(1); - table_symbol.put(high_threshold, symbol as u8)?; - high_threshold = high_threshold.wrapping_sub(1); - } else { - total = total.wrapping_add(count as u16); - } - } - - // Spread the symbols. - if high_threshold == table_size - 1 { - // No symbol of low probability: lay the symbols down in order, and then every position is used. - let mut spread = [0u8; (1 << ML_FSE_LOG) + 8]; - let mut pos = 0usize; - let mut sv = 0u64; - for &count in counts { - let n = usize::try_from(count).map_err(|_| BUG)?; - *spread.array_at_mut(pos)? = sv.to_le_bytes(); - for i in (8..n).step_by(8) { - *spread.array_at_mut(pos + i)? = sv.to_le_bytes(); - } - pos += n; - sv = sv.wrapping_add(0x0101_0101_0101_0101); - } - - let mut position = 0usize; - let (pairs, _) = spread.sub(..table_size)?.as_chunks::<2>(); - for &[first, second] in pairs { - table_symbol.put(position, first)?; - table_symbol.put((position + step) & table_mask, second)?; - position = (position + 2 * step) & table_mask; - } - } else { - let mut position = 0usize; - for (symbol, &freq) in counts.iter().enumerate() { - for _ in 0..freq { - table_symbol.put(position, symbol as u8)?; - position = (position + step) & table_mask; - while position > high_threshold { - position = (position + step) & table_mask; - } - } - } - } - - // The states, sorted by symbol. - for (u, &symbol) in table_symbol.sub(..table_size)?.iter().enumerate() { - let Some(next) = cumul.get_mut(usize::from(symbol) & (CUMUL_SIZE - 1)) else { - continue; - }; - let state = usize::from(*next) & (STATE_TABLE_SIZE - 1); - *next = next.wrapping_add(1); - if let Some(state) = self.state_table.get_mut(state) { - *state = (table_size + u) as u16; - } - } - - let mut total = 0u32; - for (&count, transform) in counts.iter().zip(&mut self.symbol_tt) { - match count { - // For `get_max_nb_bits` and `bit_cost`. - 0 => transform.delta_nb_bits = ((table_log + 1) << 16) - (1 << table_log), - -1 | 1 => { - transform.delta_nb_bits = (table_log << 16).wrapping_sub(1 << table_log); - transform.delta_find_state = total.wrapping_sub(1) as i32; - total += 1; - } - _ => { - let count = count as u32; - let max_bits_out = table_log.wrapping_sub(highbit32(count.wrapping_sub(1))); - let min_state_plus = count.wrapping_shl(max_bits_out); - transform.delta_nb_bits = (max_bits_out << 16).wrapping_sub(min_state_plus); - transform.delta_find_state = total.wrapping_sub(count) as i32; - total = total.wrapping_add(count); - } - } - } - Ok(()) - } - - /// `FSE_buildCTable_rle`: a table that encodes `symbol_value`, and nothing else, in 0 bits. - pub(crate) fn build_rle(&mut self, symbol_value: u8) -> Result<()> { - self.table_log = 0; - self.max_symbol_value = u32::from(symbol_value); - self.state_table.sub_mut(..2)?.fill(0); - self.symbol_tt - .put(usize::from(symbol_value), FseSymbolTransform::default()) - } - - /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log - /// if the table has no code for it. - #[inline(always)] - pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { - let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); - transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 - } - - /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. - /// The table has a code for it. - #[inline(always)] - pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { - let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); - let min_nb_bits = transform.delta_nb_bits >> 16; - let threshold = (min_nb_bits + 1) << 16; - let table_size = 1u32 << self.table_log; - let delta_from_threshold = - threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); - // A linear interpolation, which is very approximate. - let normalized_delta_from_threshold = - (delta_from_threshold << accuracy_log) >> self.table_log; - let bit_multiplier = 1u32 << accuracy_log; - ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) - } -} - -/// `FSE_CState_t` -pub(crate) struct FseCState<'a> { - value: usize, - table: &'a FseCTable, -} - -impl<'a> FseCState<'a> { - /// `FSE_initCState` - #[inline] - pub(crate) fn new(table: &'a FseCTable) -> Self { - Self { - value: 1 << (table.table_log & 15), - table, - } - } - - /// `FSE_initCState2`: the first symbol costs the fewest bits this way. The table has a code for `symbol`. - #[inline] - pub(crate) fn new2(table: &'a FseCTable, symbol: u32) -> Self { - let mut state = Self::new(table); - let transform = table.symbol_tt.at(symbol as usize).unwrap_or_default(); - let nb_bits_out = transform.delta_nb_bits.wrapping_add(1 << 15) >> 16; - state.value = (nb_bits_out << 16).wrapping_sub(transform.delta_nb_bits) as usize; - state.step(nb_bits_out, transform.delta_find_state); - state - } - - #[inline(always)] - fn step(&mut self, nb_bits_out: u32, delta_find_state: i32) { - let state_table = &self.table.state_table; - let index = (self.value >> (nb_bits_out & (usize::BITS - 1))) - .wrapping_add_signed(delta_find_state as isize); - let next = state_table.get(index & (state_table.len() - 1)); - self.value = usize::from(next.copied().unwrap_or(0)); - } - - /// `FSE_encodeSymbol`: the table has a code for `symbol`. - #[inline(always)] - pub(crate) fn encode_symbol(&mut self, bit_c: &mut BitCStream<'_>, symbol: u32) { - let symbol_tt = &self.table.symbol_tt; - let transform = symbol_tt.at(symbol as usize).unwrap_or_default(); - let nb_bits_out = ((self.value + transform.delta_nb_bits as usize) >> 16) as u32; - bit_c.add_bits(self.value as u64, nb_bits_out); - self.step(nb_bits_out, transform.delta_find_state); - } - - /// `FSE_flushCState` - #[inline(always)] - pub(crate) fn flush(&self, bit_c: &mut BitCStream<'_>) { - bit_c.add_bits(self.value as u64, self.table.table_log); - bit_c.flush_bits(); - } -} - -/// `FSE_NCountWriteBound` -pub(crate) fn ncount_write_bound(max_symbol_value: u32, table_log: u32) -> usize { - let max_header_size = - ((max_symbol_value as usize + 1) * table_log as usize + 4 + 2) / 8 + 1 + 2; - if max_symbol_value != 0 { - max_header_size - } else { - FSE_NCOUNTBOUND - } -} - -/// `FSE_writeNCount_generic`: `table_log` is from 5 to 12. -fn write_ncount_generic( - header: &mut [u8], - normalized_counter: &[i16], - max_symbol_value: u32, - table_log: u32, - write_is_safe: bool, -) -> Result { - let counts = normalized_counter.sub(..=max_symbol_value as usize)?; - let alphabet_size = counts.len(); - let mut out = 0usize; - let table_size = 1i32 << table_log; - let mut bit_stream = table_log - FSE_MIN_TABLELOG; - let mut bit_count = 4i32; - let mut symbol = 0usize; - let mut previous_is_0 = false; - - // The lowest 16 bits of the stream leave it. - let mut write16 = |bit_stream: &mut u32, out: &mut usize| -> Result<()> { - if !write_is_safe && *out + 2 > header.len() { - return Err(ErrorCode::DstSizeTooSmall); - } - write_le16(header, *out, *bit_stream as u16)?; - *out += 2; - *bit_stream >>= 16; - Ok(()) - }; - let shifted = |value: u32, bit_count: i32| value.wrapping_shl(bit_count as u32); - - let mut remaining = table_size + 1; - let mut threshold = table_size; - let mut nb_bits = table_log as i32 + 1; - - while symbol < alphabet_size && remaining > 1 { - if previous_is_0 { - // After a count of 0 comes how many more follow it. - let mut start = symbol; - while symbol < alphabet_size && counts.at(symbol)? == 0 { - symbol += 1; - } - if symbol == alphabet_size { - // The counts do not add up. - break; - } - while symbol >= start + 24 { - start += 24; - bit_stream = bit_stream.wrapping_add(shifted(0xFFFF, bit_count)); - write16(&mut bit_stream, &mut out)?; - } - while symbol >= start + 3 { - start += 3; - bit_stream = bit_stream.wrapping_add(shifted(3, bit_count)); - bit_count += 2; - } - bit_stream = bit_stream.wrapping_add(shifted((symbol - start) as u32, bit_count)); - bit_count += 2; - if bit_count > 16 { - write16(&mut bit_stream, &mut out)?; - bit_count -= 16; - } - } - { - let mut count = i32::from(counts.at(symbol)?); - symbol += 1; - let max = (2 * threshold - 1) - remaining; - remaining -= count.abs(); - // The stored value is the count plus one, since a count can be -1. - count += 1; - if count >= threshold { - count += max; - } - bit_stream = bit_stream.wrapping_add(shifted(count as u32, bit_count)); - bit_count += nb_bits; - bit_count -= i32::from(count < max); - previous_is_0 = count == 1; - if remaining < 1 { - return Err(ErrorCode::Generic); - } - while remaining < threshold { - nb_bits -= 1; - threshold >>= 1; - } - } - if bit_count > 16 { - write16(&mut bit_stream, &mut out)?; - bit_count -= 16; - } - } - - if remaining != 1 { - return Err(ErrorCode::Generic); - } - - let last = out; - write16(&mut bit_stream, &mut out)?; - Ok(last + ((bit_count + 7) / 8) as usize) -} - -/// `FSE_writeNCount`: writes the header that [`crate::common::entropy::read_ncount`] reads, and returns its size. -pub(crate) fn write_ncount( - buffer: &mut [u8], - normalized_counter: &[i16], - max_symbol_value: u32, - table_log: u32, -) -> Result { - if table_log > FSE_MAX_TABLELOG { - return Err(ErrorCode::TableLogTooLarge); - } - if table_log < FSE_MIN_TABLELOG { - return Err(ErrorCode::Generic); - } - let write_is_safe = buffer.len() >= ncount_write_bound(max_symbol_value, table_log); - write_ncount_generic( - buffer, - normalized_counter, - max_symbol_value, - table_log, - write_is_safe, - ) -} - -/// `FSE_minTableLog`: the smallest table log that can represent the distribution. `src_size` is more than 1. -fn min_table_log(src_size: usize, max_symbol_value: u32) -> u32 { - let min_bits_src = highbit32(src_size as u32) + 1; - let min_bits_symbols = highbit32(max_symbol_value) + 2; - min_bits_src.min(min_bits_symbols) -} - -/// `FSE_optimalTableLog_internal`: `src_size` is more than 1. -pub(crate) fn optimal_table_log_internal( - max_table_log: u32, - src_size: usize, - max_symbol_value: u32, - minus: u32, -) -> u32 { - let max_bits_src = highbit32(src_size.wrapping_sub(1) as u32).wrapping_sub(minus); - let mut table_log = max_table_log; - let min_bits = min_table_log(src_size, max_symbol_value); - if table_log == 0 { - table_log = FSE_DEFAULT_TABLELOG; - } - // More accuracy than the input has symbols for is of no use. - if max_bits_src < table_log { - table_log = max_bits_src; - } - if min_bits > table_log { - table_log = min_bits; - } - table_log.clamp(FSE_MIN_TABLELOG, FSE_MAX_TABLELOG) -} - -/// `FSE_optimalTableLog` -pub(crate) fn optimal_table_log(max_table_log: u32, src_size: usize, max_symbol_value: u32) -> u32 { - optimal_table_log_internal(max_table_log, src_size, max_symbol_value, 2) -} - -/// `FSE_normalizeM2`: the second normalization method, for when the first one would take too much from the most -/// frequent symbol. `norm` and `count` are of the same length. -fn normalize_m2( - norm: &mut [i16], - table_log: u32, - count: &[u32], - mut total: usize, - low_prob_count: i16, -) -> Result<()> { - const NOT_YET_ASSIGNED: i16 = -2; - let table_size = 1u32 << table_log; - let mut distributed = 0u32; - - // The symbols of low probability first. - let low_threshold = (total >> table_log) as u32; - let mut low_one = (total.wrapping_mul(3) >> (table_log + 1)) as u32; - - for (norm, &count) in norm.iter_mut().zip(count) { - if count == 0 { - *norm = 0; - } else if count <= low_threshold { - *norm = low_prob_count; - distributed += 1; - total = total.wrapping_sub(count as usize); - } else if count <= low_one { - *norm = 1; - distributed += 1; - total = total.wrapping_sub(count as usize); - } else { - *norm = NOT_YET_ASSIGNED; - } - } - let mut to_distribute = table_size.wrapping_sub(distributed); - - if to_distribute == 0 { - return Ok(()); - } - - if total / to_distribute as usize > low_one as usize { - // The rest would get too much each: more symbols get 1. - low_one = total - .wrapping_mul(3) - .checked_div(to_distribute.wrapping_mul(2) as usize) - .ok_or(BUG)? as u32; - for (norm, &count) in norm.iter_mut().zip(count) { - if *norm == NOT_YET_ASSIGNED && count <= low_one { - *norm = 1; - distributed += 1; - total = total.wrapping_sub(count as usize); - } - } - to_distribute = table_size.wrapping_sub(distributed); - } - - if distributed as usize == count.len() { - // All the values are small. The first symbol of the largest count gets what is left. - let mut max_v = 0usize; - let mut max_c = 0u32; - for (s, &count) in count.iter().enumerate() { - if count > max_c { - max_v = s; - max_c = count; - } - } - let norm = norm.at_mut(max_v)?; - *norm = norm.wrapping_add(to_distribute as i16); - return Ok(()); - } - - if total == 0 { - // All the symbols were of low enough probability to get 1 or less: what is left goes round. - if !norm.iter().any(|&norm| norm > 0) { - return Err(BUG); - } - while to_distribute > 0 { - for norm in norm.iter_mut().filter(|norm| **norm > 0) { - if to_distribute == 0 { - break; - } - to_distribute -= 1; - *norm = norm.wrapping_add(1); - } - } - return Ok(()); - } - - let v_step_log = 62 - table_log; - let mid = (1u64 << (v_step_log - 1)) - 1; - // A scale on what remains. - let r_step = ((1u64 << v_step_log) * u64::from(to_distribute) + mid) - .checked_div(u64::from(total as u32)) - .ok_or(BUG)?; - let mut tmp_total = mid; - for (norm, &count) in norm.iter_mut().zip(count) { - if *norm == NOT_YET_ASSIGNED { - let end = tmp_total.wrapping_add(u64::from(count).wrapping_mul(r_step)); - let s_start = (tmp_total >> v_step_log) as u32; - let s_end = (end >> v_step_log) as u32; - let weight = s_end.wrapping_sub(s_start); - if weight < 1 { - return Err(ErrorCode::Generic); - } - *norm = weight as i16; - tmp_total = end; - } - } - Ok(()) -} - -/// `FSE_normalizeCount`: scales `count`, which adds up to `total`, to add up to `1 << table_log`. Returns the table -/// log, or 0 if there is only one symbol, which is a case for run-length encoding. -pub(crate) fn normalize_count( - normalized_counter: &mut [i16], - mut table_log: u32, - count: &[u32], - total: usize, - max_symbol_value: u32, - use_low_prob_count: bool, -) -> Result { - if table_log == 0 { - table_log = FSE_DEFAULT_TABLELOG; - } - if table_log < FSE_MIN_TABLELOG { - return Err(ErrorCode::Generic); - } - if table_log > FSE_MAX_TABLELOG { - return Err(ErrorCode::TableLogTooLarge); - } - if table_log < min_table_log(total, max_symbol_value) { - return Err(ErrorCode::Generic); - } - - const RTB_TABLE: [u32; 8] = [0, 473195, 504333, 520860, 550000, 700000, 750000, 830000]; - let low_prob_count: i16 = if use_low_prob_count { -1 } else { 1 }; - let scale = 62 - table_log; - let step = (1u64 << 62) - .checked_div(u64::from(total as u32)) - .ok_or(BUG)?; - let v_step = 1u64 << (scale - 20); - let mut still_to_distribute = 1i32 << table_log; - let mut largest = 0usize; - let mut largest_p = 0i16; - let low_threshold = (total >> table_log) as u32; - - let count = count.sub(..=max_symbol_value as usize)?; - let norm = normalized_counter.sub_mut(..=max_symbol_value as usize)?; - - for (s, (norm, &count)) in norm.iter_mut().zip(count).enumerate() { - if count as usize == total { - return Ok(0); - } - if count == 0 { - *norm = 0; - } else if count <= low_threshold { - *norm = low_prob_count; - still_to_distribute -= 1; - } else { - let scaled = u64::from(count).wrapping_mul(step); - let mut proba = (scaled >> scale) as i16; - if proba < 8 { - let rest_to_beat = v_step * u64::from(RTB_TABLE.at(proba as usize)?); - proba += i16::from(scaled.wrapping_sub((proba as u64) << scale) > rest_to_beat); - } - if proba > largest_p { - largest_p = proba; - largest = s; - } - *norm = proba; - still_to_distribute -= i32::from(proba); - } - } - let largest = norm.at_mut(largest)?; - if -still_to_distribute >= i32::from(*largest >> 1) { - // The correction is too large for one symbol to take. - normalize_m2(norm, table_log, count, total, low_prob_count)?; - } else { - *largest = largest.wrapping_add(still_to_distribute as i16); - } - Ok(table_log as usize) -} - -/// `FSE_compress_usingCTable_generic` -#[inline(always)] -fn compress_using_ctable_generic( - dst: &mut [u8], - src: &[u8], - ct: &FseCTable, -) -> usize { - let flush = |bit_c: &mut BitCStream<'_>| { - if FAST { - bit_c.flush_bits_fast(); - } else { - bit_c.flush_bits(); - } - }; - - if src.len() <= 2 { - return 0; - } - let Ok(mut bit_c) = BitCStream::new(dst) else { - return 0; - }; - let encode = |state: &mut FseCState<'_>, bit_c: &mut BitCStream<'_>, symbol: u8| { - state.encode_symbol(bit_c, u32::from(symbol)); - }; - - // From the last symbol to the first, two states taking turns. - let mut state1; - let mut state2; - let mut rest; - if src.len() & 1 != 0 { - let [head @ .., c, b, a] = src else { - return 0; - }; - state1 = FseCState::new2(ct, u32::from(*a)); - state2 = FseCState::new2(ct, u32::from(*b)); - encode(&mut state1, &mut bit_c, *c); - flush(&mut bit_c); - rest = head; - } else { - let [head @ .., b, a] = src else { - return 0; - }; - state2 = FseCState::new2(ct, u32::from(*a)); - state1 = FseCState::new2(ct, u32::from(*b)); - rest = head; - } - - if let ([head @ .., b, a], true) = (rest, rest.len() & 2 != 0) { - encode(&mut state2, &mut bit_c, *a); - encode(&mut state1, &mut bit_c, *b); - flush(&mut bit_c); - rest = head; - } - - let (_, quads) = rest.as_rchunks::<4>(); - for &[d, c, b, a] in quads.iter().rev() { - encode(&mut state2, &mut bit_c, a); - encode(&mut state1, &mut bit_c, b); - encode(&mut state2, &mut bit_c, c); - encode(&mut state1, &mut bit_c, d); - flush(&mut bit_c); - } - - state2.flush(&mut bit_c); - state1.flush(&mut bit_c); - bit_c.close() -} - -/// `FSE_compress_usingCTable`: the table has a code for every symbol in `src`. Returns the size of the output, or 0 -/// if `src` has less than 3 symbols or `dst` is too small. -pub(crate) fn compress_using_ctable(dst: &mut [u8], src: &[u8], ct: &FseCTable) -> usize { - if dst.len() >= block_bound(src.len()) { - compress_using_ctable_generic::(dst, src, ct) - } else { - compress_using_ctable_generic::(dst, src, ct) - } -} diff --git a/src/zstd/wip/work/entropy/zstd/compress/hist.rs b/src/zstd/wip/work/entropy/zstd/compress/hist.rs deleted file mode 100644 index 745fc94e641f..000000000000 --- a/src/zstd/wip/work/entropy/zstd/compress/hist.rs +++ /dev/null @@ -1,132 +0,0 @@ -//! `hist.c`: how many times each byte value occurs. - -use crate::error::{ErrorCode, Result}; -use crate::util::{BUG, SliceExt}; - -/// `HIST_add`: `count` has room for every value in `src`. -pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { - for &byte in src { - *count.at_mut(usize::from(byte))? += 1; - } - Ok(()) -} - -/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. -/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. -pub(crate) fn count_simple( - count: &mut [u32], - max_symbol_value: &mut u32, - src: &[u8], -) -> Result { - let mut max_symbol = *max_symbol_value as usize; - count.sub_mut(..=max_symbol)?.fill(0); - if src.is_empty() { - *max_symbol_value = 0; - return Ok(0); - } - add(count, src)?; - while count.at(max_symbol)? == 0 { - max_symbol = max_symbol.checked_sub(1).ok_or(BUG)?; - } - *max_symbol_value = max_symbol as u32; - Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) -} - -#[inline(always)] -fn increment(counting: &mut [u32; 256], byte: u8) { - if let Some(count) = counting.get_mut(usize::from(byte)) { - *count = count.wrapping_add(1); - } -} - -/// `HIST_count_parallel_wksp`: four tables, so that a run of one value does not wait for its own counter. -/// `max_symbol_value` is at most 255. With `check`, a value above it is an error rather than trusted not to occur. -fn count_parallel( - count: &mut [u32], - max_symbol_value: &mut u32, - src: &[u8], - check: bool, -) -> Result { - let count_size = *max_symbol_value as usize + 1; - if src.is_empty() { - count.sub_mut(..count_size)?.fill(0); - *max_symbol_value = 0; - return Ok(0); - } - - let mut counting1 = [0u32; 256]; - let mut counting2 = [0u32; 256]; - let mut counting3 = [0u32; 256]; - let mut counting4 = [0u32; 256]; - - // Each word is read before the bytes of the one before it are counted. - let (words, rest) = src.as_chunks::<8>(); - if let Some((&first, words)) = words.split_first() { - let mut cached = u64::from_le_bytes(first); - let mut count_word = |next: [u8; 8]| { - let c = cached; - cached = u64::from_le_bytes(next); - increment(&mut counting1, c as u8); - increment(&mut counting2, (c >> 8) as u8); - increment(&mut counting3, (c >> 16) as u8); - increment(&mut counting4, (c >> 24) as u8); - increment(&mut counting1, (c >> 32) as u8); - increment(&mut counting2, (c >> 40) as u8); - increment(&mut counting3, (c >> 48) as u8); - increment(&mut counting4, (c >> 56) as u8); - }; - let (stripes, words) = words.as_chunks::<2>(); - for &[a, b] in stripes { - count_word(a); - count_word(b); - } - for &word in words { - count_word(word); - } - count_word([0; 8]); - } - for &byte in rest { - increment(&mut counting1, byte); - } - - let mut max = 0u32; - let others = counting2.iter().zip(&counting3).zip(&counting4); - for (sum, ((&c2, &c3), &c4)) in counting1.iter_mut().zip(others) { - *sum = sum.wrapping_add(c2).wrapping_add(c3).wrapping_add(c4); - max = max.max(*sum); - } - - let max_symbol = counting1.iter().rposition(|&c| c != 0).ok_or(BUG)? as u32; - if check && max_symbol > *max_symbol_value { - return Err(ErrorCode::MaxSymbolValueTooSmall); - } - *max_symbol_value = max_symbol; - count.write_at(0, counting1.sub(..count_size)?)?; - Ok(max as usize) -} - -/// `HIST_countFast_wksp`: as [`count_simple`], faster on large inputs. -pub(crate) fn count_fast_wksp( - count: &mut [u32], - max_symbol_value: &mut u32, - src: &[u8], -) -> Result { - if src.len() < 1500 { - return Ok(count_simple(count, max_symbol_value, src)? as usize); - } - count_parallel(count, max_symbol_value, src, false) -} - -/// `HIST_count_wksp`: as [`count_fast_wksp`], but a value above `max_symbol_value` in `src` is -/// [`ErrorCode::MaxSymbolValueTooSmall`]. -pub(crate) fn count_wksp( - count: &mut [u32], - max_symbol_value: &mut u32, - src: &[u8], -) -> Result { - if *max_symbol_value < 255 { - return count_parallel(count, max_symbol_value, src, true); - } - *max_symbol_value = 255; - count_fast_wksp(count, max_symbol_value, src) -} diff --git a/src/zstd/wip/work/entropy/zstd/compress/huf.rs b/src/zstd/wip/work/entropy/zstd/compress/huf.rs deleted file mode 100644 index 7052bc0be190..000000000000 --- a/src/zstd/wip/work/entropy/zstd/compress/huf.rs +++ /dev/null @@ -1,1162 +0,0 @@ -//! `huf_compress.c`: Huffman coding of literals. - -use super::fse::{self, FseCTable}; -use super::hist; -use super::internal::Strategy; -use crate::common::entropy::huf_read_stats; -use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; -use crate::error::{ErrorCode, Result}; -use crate::util::{BUG, SliceExt, highbit32, write_le16}; - -pub(crate) const HUF_BLOCKSIZE_MAX: usize = 128 * 1024; -pub(crate) const HUF_WORKSPACE_SIZE: usize = (8 << 10) + 512; -pub(crate) const HUF_TABLELOG_DEFAULT: u32 = 11; -/// The strategies from this one on look for the best depth of the tree. -pub(crate) const HUF_OPTIMAL_DEPTH_THRESHOLD: Strategy = Strategy::BtUltra; -const HUF_CTABLEBOUND: usize = 129; - -/// `HUF_flags_e` -#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] -pub(crate) struct HufFlags { - /// Try every depth of the tree and keep the one that gives the smallest output, instead of a guess. - pub(crate) optimal_depth: bool, - /// Use the previous table again whenever it is possible, without looking for a better one. - pub(crate) prefer_repeat: bool, - /// Sample the input first, and give up at once if it looks like it cannot be compressed. - pub(crate) suspect_uncompressible: bool, -} - -/// `HUF_repeat` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum HufRepeat { - /// The table cannot be used again. - None, - /// It can, if it has a code for every symbol to encode. - Check, - /// It can. - Valid, -} - -/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the -/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in -/// its highest bits. -#[derive(Clone)] -pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); - -impl HufCTable { - pub(crate) const fn new() -> Self { - Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) - } - - /// The elements, by symbol. - #[inline(always)] - fn elements(&self) -> &[u64; HUF_SYMBOLVALUE_MAX as usize + 1] { - let [_, elements @ ..] = &self.0; - elements - } - - #[inline(always)] - fn elements_mut(&mut self) -> &mut [u64; HUF_SYMBOLVALUE_MAX as usize + 1] { - let [_, elements @ ..] = &mut self.0; - elements - } - - /// `HUF_writeCTableHeader` - fn write_header(&mut self, table_log: u32, max_symbol_value: u32) { - let [header, ..] = &mut self.0; - *header = u64::from(table_log as u8) | (u64::from(max_symbol_value as u8) << 8); - } - - /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. - #[inline] - pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { - let [header, elements @ ..] = &self.0; - let max_symbol_value = (header >> 8) as u8; - if symbol_value > u32::from(max_symbol_value) { - return 0; - } - (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 - } -} - -/// `HUF_CTableHeader` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) struct HufCTableHeader { - pub(crate) table_log: u8, - pub(crate) max_symbol_value: u8, -} - -/// `HUF_readCTableHeader` -#[inline] -pub(crate) fn read_ctable_header(ctable: &HufCTable) -> HufCTableHeader { - let [header, ..] = ctable.0; - HufCTableHeader { - table_log: header as u8, - max_symbol_value: (header >> 8) as u8, - } -} - -/// `HUF_getNbBits` -#[inline(always)] -const fn get_nb_bits(elt: u64) -> u64 { - elt & 0xFF -} - -/// `HUF_setValue`: `elt` has its number of bits, and nothing else. -#[inline(always)] -fn set_value(elt: &mut u64, value: u64) { - let nb_bits = get_nb_bits(*elt) as u32; - if nb_bits > 0 { - *elt |= value.wrapping_shl(u64::BITS.wrapping_sub(nb_bits)); - } -} - -/// Gives each symbol its code. `ct` has the number of bits of each, at most `table_log`, and `nb_per_rank` has how -/// many symbols there are of each number of bits. -fn set_values( - ct: &mut [u64], - nb_per_rank: &[u16; RANKS], - table_log: u32, -) -> Result<()> { - let mut val_per_rank = [0u16; RANKS]; - // The first code of each rank: the longest codes have the lowest values. - let mut min = 0u16; - for n in (1..=table_log as usize).rev() { - val_per_rank.put(n, min)?; - min = min.wrapping_add(nb_per_rank.at(n)?) >> 1; - } - for elt in ct { - let value = val_per_rank.at_mut(get_nb_bits(*elt) as usize)?; - set_value(elt, u64::from(*value)); - *value = value.wrapping_add(1); - } - Ok(()) -} - -const MAX_FSE_TABLELOG_FOR_HUFF_HEADER: u32 = 6; - -/// `HUF_compressWeights`: returns 0 if the weights cannot be compressed, and 1 if they are all the same. -fn compress_weights(dst: &mut [u8], weight_table: &[u8]) -> Result { - let mut max_symbol_value = HUF_TABLELOG_MAX; - let mut count = [0u32; HUF_TABLELOG_MAX as usize + 1]; - let mut norm = [0i16; HUF_TABLELOG_MAX as usize + 1]; - let wt_size = weight_table.len(); - - if wt_size <= 1 { - return Ok(0); - } - - let max_count = hist::count_simple(&mut count, &mut max_symbol_value, weight_table)?; - if max_count as usize == wt_size { - return Ok(1); - } - if max_count == 1 { - return Ok(0); - } - - let table_log = - fse::optimal_table_log(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, wt_size, max_symbol_value); - fse::normalize_count( - &mut norm, - table_log, - &count, - wt_size, - max_symbol_value, - false, - )?; - - let h_size = fse::write_ncount(dst, &norm, max_symbol_value, table_log)?; - - let mut ctable = FseCTable::new(); - ctable.build(&norm, max_symbol_value, table_log)?; - let c_size = fse::compress_using_ctable(dst.sub_mut(h_size..)?, weight_table, &ctable); - if c_size == 0 { - return Ok(0); - } - Ok(h_size + c_size) -} - -/// `HUF_writeCTable_wksp`: writes the header that [`huf_read_stats`] reads, and returns its size. `max_symbol_value` -/// and `huff_log` are those of `ctable`. -pub(crate) fn write_ctable( - dst: &mut [u8], - ctable: &HufCTable, - max_symbol_value: u32, - huff_log: u32, -) -> Result { - let mut bits_to_weight = [0u8; HUF_TABLELOG_MAX as usize + 1]; - let mut huff_weight = [0u8; HUF_SYMBOLVALUE_MAX as usize]; - - if max_symbol_value > HUF_SYMBOLVALUE_MAX { - return Err(ErrorCode::MaxSymbolValueTooLarge); - } - // The weight of the last symbol is not stored. - let nb_weights = max_symbol_value as usize; - - for n in 1..=huff_log { - bits_to_weight.put(n as usize, (huff_log + 1 - n) as u8)?; - } - let weights = huff_weight.sub_mut(..nb_weights)?; - for (weight, &elt) in weights.iter_mut().zip(ctable.elements()) { - *weight = bits_to_weight.at(get_nb_bits(elt) as usize)?; - } - - let max_dst_size = dst.len(); - let Some((first, rest)) = dst.split_first_mut() else { - return Err(ErrorCode::DstSizeTooSmall); - }; - let h_size = compress_weights(rest, weights)?; - if h_size > 1 && h_size < nb_weights / 2 { - *first = h_size as u8; - return Ok(h_size + 1); - } - - // Not compressed: 4 bits to a weight. - if max_symbol_value > 256 - 128 { - return Err(ErrorCode::Generic); - } - let raw_size = nb_weights.div_ceil(2); - if raw_size + 1 > max_dst_size { - return Err(ErrorCode::DstSizeTooSmall); - } - *first = 128u32.wrapping_add(max_symbol_value.wrapping_sub(1)) as u8; - // An odd number of weights is completed with a 0, which `huff_weight` still has there. - let (pairs, _) = huff_weight.sub(..raw_size * 2)?.as_chunks::<2>(); - for (out, &[high, low]) in rest.iter_mut().zip(pairs) { - *out = (high << 4).wrapping_add(low); - } - Ok(raw_size + 1) -} - -/// What [`read_ctable`] found besides the table. -pub(crate) struct HufCTableStats { - /// How many bytes of the input the header takes. - pub(crate) size: usize, - pub(crate) max_symbol_value: u32, - /// Some symbol up to `max_symbol_value` has no code. - pub(crate) has_zero_weights: bool, -} - -/// `HUF_readCTable`: [`ErrorCode::MaxSymbolValueTooSmall`] if the table has symbols above `max_symbol_value`. -pub(crate) fn read_ctable( - ctable: &mut HufCTable, - max_symbol_value: u32, - src: &[u8], -) -> Result { - let mut huff_weight = [0u8; HUF_SYMBOLVALUE_MAX as usize + 1]; - let mut rank_val = [0u32; HUF_TABLELOG_MAX as usize + 1]; - - let stats = huf_read_stats(&mut huff_weight, &mut rank_val, src)?; - let table_log = stats.table_log; - let [zero_weights, ..] = rank_val; - - if table_log > HUF_TABLELOG_MAX { - return Err(ErrorCode::TableLogTooLarge); - } - if stats.nb_symbols > max_symbol_value.wrapping_add(1) { - return Err(ErrorCode::MaxSymbolValueTooSmall); - } - let max_symbol_value = stats.nb_symbols - 1; - ctable.write_header(table_log, max_symbol_value); - - let mut nb_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 2]; - let ct = ctable.elements_mut().sub_mut(..stats.nb_symbols as usize)?; - for (elt, &weight) in ct.iter_mut().zip(&huff_weight) { - let nb_bits = if weight != 0 { - (table_log + 1).wrapping_sub(u32::from(weight)) as u8 - } else { - 0 - }; - *elt = u64::from(nb_bits); - *nb_per_rank.at_mut(usize::from(nb_bits))? += 1; - } - set_values(ct, &nb_per_rank, table_log)?; - - Ok(HufCTableStats { - size: stats.size, - max_symbol_value, - has_zero_weights: zero_weights > 0, - }) -} - -/// `nodeElt` -#[derive(Clone, Copy, Default)] -struct NodeElt { - count: u32, - parent: u16, - byte: u8, - nb_bits: u8, -} - -/// The first node that is not a leaf. -const STARTNODE: i32 = HUF_SYMBOLVALUE_MAX as i32 + 1; - -/// `huffNodeTable`: the leaves from 0 on, sorted from the most frequent symbol to the rarest, the other nodes from -/// [`STARTNODE`] on, and a sentinel at -1. A number that is out of the table wraps around, which spares the checks. -struct HuffNodes([NodeElt; 2 * (HUF_SYMBOLVALUE_MAX as usize + 1)]); - -impl HuffNodes { - #[inline(always)] - const fn index(n: i32) -> usize { - n.wrapping_add(1) as usize & (2 * (HUF_SYMBOLVALUE_MAX as usize + 1) - 1) - } - - #[inline(always)] - fn get(&self, n: i32) -> NodeElt { - self.0.get(Self::index(n)).copied().unwrap_or_default() - } - - #[inline(always)] - fn set(&mut self, n: i32, node: NodeElt) { - if let Some(slot) = self.0.get_mut(Self::index(n)) { - *slot = node; - } - } - - #[inline(always)] - fn count(&self, n: i32) -> u32 { - self.get(n).count - } - - #[inline(always)] - fn nb_bits(&self, n: i32) -> u32 { - u32::from(self.get(n).nb_bits) - } - - #[inline(always)] - fn set_count(&mut self, n: i32, count: u32) { - if let Some(node) = self.0.get_mut(Self::index(n)) { - node.count = count; - } - } - - #[inline(always)] - fn set_parent(&mut self, n: i32, parent: i32) { - if let Some(node) = self.0.get_mut(Self::index(n)) { - node.parent = parent as u16; - } - } - - #[inline(always)] - fn set_nb_bits(&mut self, n: i32, nb_bits: u32) { - if let Some(node) = self.0.get_mut(Self::index(n)) { - node.nb_bits = nb_bits as u8; - } - } - - /// The nodes from 0 on. - #[inline(always)] - fn leaves(&self) -> &[NodeElt] { - let [_, nodes @ ..] = &self.0; - nodes - } -} - -/// `HUF_setMaxHeight`: makes no code longer than `target_nb_bits`, which is not 0 and is enough for a code for every -/// symbol. `last_non_null` is the rarest symbol, which has the longest code. Returns the length of the longest code. -fn set_max_height( - huff_node: &mut HuffNodes, - last_non_null: u32, - target_nb_bits: u32, -) -> Result { - let largest_bits = huff_node.nb_bits(last_non_null as i32); - if largest_bits <= target_nb_bits { - return Ok(largest_bits); - } - - // The unit of cost is the share of the code space that a code of `largest_bits` takes. - let cost_of = |nb_bits: u32| 1u32.wrapping_shl(largest_bits.wrapping_sub(nb_bits)); - let mut total_cost = 0i32; - let base_cost = cost_of(target_nb_bits); - let mut n = last_non_null as i32; - - // Shorten the codes that are too long. `total_cost` is how much more than the whole code space is taken then. - while huff_node.nb_bits(n) > target_nb_bits { - let cost = base_cost.wrapping_sub(cost_of(huff_node.nb_bits(n))); - total_cost = total_cost.wrapping_add(cost as i32); - huff_node.set_nb_bits(n, target_nb_bits); - n -= 1; - } - // `n` ends at the rarest symbol with a code shorter than `target_nb_bits`, or at the sentinel. - while huff_node.nb_bits(n) == target_nb_bits { - n -= 1; - } - - // From here on the unit is the share of a code of `target_nb_bits`. - total_cost = total_cost.wrapping_shr(largest_bits - target_nb_bits); - - // Repay the cost by making other codes longer. - const NO_SYMBOL: u32 = 0xF0F0_F0F0; - // The rarest symbol whose code is that many bits shorter than `target_nb_bits`. - let mut rank_last = [NO_SYMBOL; HUF_TABLELOG_MAX as usize + 2]; - - let mut current_nb_bits = target_nb_bits; - for pos in (0..=n).rev() { - let nb_bits = huff_node.nb_bits(pos); - if nb_bits >= current_nb_bits { - continue; - } - current_nb_bits = nb_bits; - rank_last.put((target_nb_bits - current_nb_bits) as usize, pos as u32)?; - } - - while total_cost > 0 { - // One more bit on a code that is `n_bits_to_decrease` shorter repays `1 << (n_bits_to_decrease - 1)`. - let mut n_bits_to_decrease = highbit32(total_cost as u32) as usize + 1; - while n_bits_to_decrease > 1 { - let high_pos = rank_last.at(n_bits_to_decrease)?; - let low_pos = rank_last.at(n_bits_to_decrease - 1)?; - if high_pos != NO_SYMBOL { - if low_pos == NO_SYMBOL { - break; - } - // It is cheaper to lengthen this one than two of the rank below. - let high_total = huff_node.count(high_pos as i32); - let low_total = huff_node.count(low_pos as i32).wrapping_mul(2); - if high_total <= low_total { - break; - } - } - n_bits_to_decrease -= 1; - } - while n_bits_to_decrease <= HUF_TABLELOG_MAX as usize - && rank_last.at(n_bits_to_decrease)? == NO_SYMBOL - { - n_bits_to_decrease += 1; - } - let pos = rank_last.at(n_bits_to_decrease)?; - if pos == NO_SYMBOL { - return Err(BUG); - } - total_cost -= 1 << (n_bits_to_decrease - 1); - huff_node.set_nb_bits(pos as i32, huff_node.nb_bits(pos as i32) + 1); - - // The symbol is now of the rank below, where it is the rarest only if there was none. - let below = rank_last.at_mut(n_bits_to_decrease - 1)?; - if *below == NO_SYMBOL { - *below = pos; - } - // The next symbol takes its place, if it is of the same rank. - let same_rank = target_nb_bits.wrapping_sub(n_bits_to_decrease as u32); - let next = if pos != 0 && huff_node.nb_bits(pos as i32 - 1) == same_rank { - pos - 1 - } else { - NO_SYMBOL - }; - rank_last.put(n_bits_to_decrease, next)?; - } - - // Too much may have been repaid: shorten codes of `target_nb_bits`, the most frequent symbols first. - let [_, rank_last_1, ..] = &mut rank_last; - while total_cost < 0 { - if *rank_last_1 == NO_SYMBOL { - while huff_node.nb_bits(n) == target_nb_bits { - n -= 1; - } - *rank_last_1 = n as u32; - } - *rank_last_1 = rank_last_1.wrapping_add(1); - let pos = *rank_last_1 as i32; - huff_node.set_nb_bits(pos, huff_node.nb_bits(pos).wrapping_sub(1)); - total_cost += 1; - } - - Ok(target_nb_bits) -} - -/// `rankPos` -#[derive(Clone, Copy, Default)] -struct RankPos { - base: u16, - curr: u16, -} - -/// How many buckets [`sort`] has. -const RANK_POSITION_TABLE_SIZE: usize = 192; -const RANK_POSITION_MAX_COUNT_LOG: u32 = 32; -const RANK_POSITION_LOG_BUCKETS_BEGIN: u32 = - (RANK_POSITION_TABLE_SIZE as u32 - 1) - RANK_POSITION_MAX_COUNT_LOG - 1; -/// The counts below this one have a bucket each. The others share one with the counts of the same binary logarithm. -const RANK_POSITION_DISTINCT_COUNT_CUTOFF: u32 = - RANK_POSITION_LOG_BUCKETS_BEGIN + highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN); - -/// `HUF_getIndex`: the bucket of a count. -#[inline(always)] -fn get_index(count: u32) -> usize { - if count < RANK_POSITION_DISTINCT_COUNT_CUTOFF { - count as usize - } else { - (highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN) as usize - } -} - -/// `HUF_swapNodes` -#[inline(always)] -fn swap_nodes(arr: &mut HuffNodes, a: i32, b: i32) { - let tmp = arr.get(a); - arr.set(a, arr.get(b)); - arr.set(b, tmp); -} - -/// `HUF_insertionSort`: by decreasing count, from `low` to `high` included. -#[inline] -fn insertion_sort(huff_node: &mut HuffNodes, low: i32, high: i32) { - for i in low + 1..=high { - let key = huff_node.get(i); - let mut j = i - 1; - while j >= low && huff_node.count(j) < key.count { - huff_node.set(j + 1, huff_node.get(j)); - j -= 1; - } - huff_node.set(j + 1, key); - } -} - -/// `HUF_quickSortPartition`: the pivot is the last element. Returns where it ends up. -fn quick_sort_partition(arr: &mut HuffNodes, low: i32, high: i32) -> i32 { - let pivot = arr.count(high); - let mut i = low - 1; - for j in low..high { - if arr.count(j) > pivot { - i += 1; - swap_nodes(arr, i, j); - } - } - swap_nodes(arr, i + 1, high); - i + 1 -} - -/// `HUF_simpleQuickSort`: by decreasing count, from `low` to `high` included. It recurses into the smaller part only, -/// which bounds the depth. The order it leaves symbols of the same count in decides their codes. -fn simple_quick_sort(arr: &mut HuffNodes, mut low: i32, mut high: i32) { - const INSERTION_SORT_THRESHOLD: i32 = 8; - if high - low < INSERTION_SORT_THRESHOLD { - insertion_sort(arr, low, high); - return; - } - while low < high { - let idx = quick_sort_partition(arr, low, high); - if idx - low < high - idx { - simple_quick_sort(arr, low, idx - 1); - low = idx + 1; - } else { - simple_quick_sort(arr, idx + 1, high); - high = idx - 1; - } - } -} - -/// `HUF_sort`: makes the first nodes, which are all zeros, the symbols by decreasing count. At most 256 counts. -fn sort(huff_node: &mut HuffNodes, count: &[u32]) -> Result<()> { - let mut rank_position = [RankPos::default(); RANK_POSITION_TABLE_SIZE]; - - // A bucket starts after the symbols of all the buckets above it. - for &c in count { - rank_position.at_mut(get_index(c))?.base += 1; - } - let mut above = 0u16; - for rank in rank_position.iter_mut().rev() { - rank.base += above; - rank.curr = rank.base; - above = rank.base; - } - - for (n, &c) in count.iter().enumerate() { - let rank = rank_position.at_mut(get_index(c) + 1)?; - let node = NodeElt { - count: c, - byte: n as u8, - ..NodeElt::default() - }; - huff_node.set(i32::from(rank.curr), node); - rank.curr += 1; - } - - let shared = RANK_POSITION_DISTINCT_COUNT_CUTOFF as usize..RANK_POSITION_TABLE_SIZE - 1; - for rank in rank_position.sub(shared)? { - if rank.curr > rank.base + 1 { - simple_quick_sort(huff_node, i32::from(rank.base), i32::from(rank.curr) - 1); - } - } - Ok(()) -} - -/// `HUF_buildTree`: `huff_node` is sorted, and some count is not 0. Sets the depth of every leaf, and returns the -/// rarest symbol whose count is not 0. -fn build_tree(huff_node: &mut HuffNodes, max_symbol_value: u32) -> Result { - let mut symbols = (0..=max_symbol_value as i32).rev(); - let non_null_rank = symbols.find(|&n| huff_node.count(n) != 0).ok_or(BUG)?; - let mut low_s = non_null_rank; - let mut node_nb = STARTNODE; - let node_root = node_nb + low_s - 1; - let mut low_n = node_nb; - - let mut join = |huff_node: &mut HuffNodes, n1: i32, n2: i32| { - let count = huff_node.count(n1).wrapping_add(huff_node.count(n2)); - huff_node.set_count(node_nb, count); - huff_node.set_parent(n1, node_nb); - huff_node.set_parent(n2, node_nb); - node_nb += 1; - }; - - join(huff_node, low_s, low_s - 1); - low_s -= 2; - for n in STARTNODE + 1..=node_root { - huff_node.set_count(n, 1 << 30); - } - // The sentinel is never the smaller of the two. - huff_node.set_count(-1, 1 << 31); - - // Join the two nodes of the lowest counts, which are the next leaves or the next parents. - for _ in STARTNODE + 1..=node_root { - let mut lowest = |huff_node: &HuffNodes| { - if huff_node.count(low_s) < huff_node.count(low_n) { - low_s -= 1; - low_s + 1 - } else { - low_n += 1; - low_n - 1 - } - }; - let n1 = lowest(huff_node); - let n2 = lowest(huff_node); - join(huff_node, n1, n2); - } - - // The parents come after their children, so that this goes from the root down. - huff_node.set_nb_bits(node_root, 0); - for n in (STARTNODE..node_root).rev().chain(0..=non_null_rank) { - let parent = i32::from(huff_node.get(n).parent); - huff_node.set_nb_bits(n, huff_node.nb_bits(parent) + 1); - } - - Ok(non_null_rank) -} - -/// `HUF_buildCTableFromTree`: no leaf is deeper than `max_nb_bits`, which is at most 12. -fn build_ctable_from_tree( - ctable: &mut HufCTable, - huff_node: &[NodeElt], - non_null_rank: i32, - max_symbol_value: u32, - max_nb_bits: u32, -) -> Result<()> { - let alphabet_size = max_symbol_value as usize + 1; - let ct = ctable.elements_mut().sub_mut(..alphabet_size)?; - let mut nb_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 1]; - for node in huff_node.sub(..=non_null_rank as usize)? { - *nb_per_rank.at_mut(usize::from(node.nb_bits))? += 1; - } - for node in huff_node.sub(..alphabet_size)? { - ct.put(usize::from(node.byte), u64::from(node.nb_bits))?; - } - set_values(ct, &nb_per_rank, max_nb_bits)?; - ctable.write_header(max_nb_bits, max_symbol_value); - Ok(()) -} - -/// `HUF_buildCTable_wksp`: some count up to `max_symbol_value` is not 0, and `max_nb_bits` is enough for a code for -/// each that is not, or 0 for the default. Returns the length of the longest code. -pub(crate) fn build_ctable( - ctable: &mut HufCTable, - count: &[u32], - max_symbol_value: u32, - mut max_nb_bits: u32, -) -> Result { - if max_nb_bits == 0 { - max_nb_bits = HUF_TABLELOG_DEFAULT; - } - if max_symbol_value > HUF_SYMBOLVALUE_MAX { - return Err(ErrorCode::MaxSymbolValueTooLarge); - } - let mut huff_node = HuffNodes([NodeElt::default(); _]); - - sort(&mut huff_node, count.sub(..=max_symbol_value as usize)?)?; - let non_null_rank = build_tree(&mut huff_node, max_symbol_value)?; - max_nb_bits = set_max_height(&mut huff_node, non_null_rank as u32, max_nb_bits)?; - if max_nb_bits > HUF_TABLELOG_MAX { - return Err(ErrorCode::Generic); - } - build_ctable_from_tree( - ctable, - huff_node.leaves(), - non_null_rank, - max_symbol_value, - max_nb_bits, - )?; - Ok(max_nb_bits as usize) -} - -/// `HUF_estimateCompressedSize`: how many bytes the symbols counted in `count` take, without the last partial one. -pub(crate) fn estimate_compressed_size( - ctable: &HufCTable, - count: &[u32], - max_symbol_value: u32, -) -> usize { - let nb_bits: usize = (ctable.elements().iter().zip(count)) - .take(max_symbol_value as usize + 1) - .map(|(&elt, &count)| get_nb_bits(elt) as usize * count as usize) - .sum(); - nb_bits >> 3 -} - -/// `HUF_validateCTable`: whether the table has a code for every symbol whose count is not 0. -pub(crate) fn validate_ctable(ctable: &HufCTable, count: &[u32], max_symbol_value: u32) -> bool { - if u32::from(read_ctable_header(ctable).max_symbol_value) < max_symbol_value { - return false; - } - (ctable.elements().iter().zip(count)) - .take(max_symbol_value as usize + 1) - .all(|(&elt, &count)| count == 0 || get_nb_bits(elt) != 0) -} - -/// `HUF_compressBound` -pub(crate) const fn compress_bound(size: usize) -> usize { - HUF_CTABLEBOUND + size + (size >> 8) + 8 -} - -/// One `bitContainer` of `HUF_CStream_t` and its `bitPos`. -#[derive(Clone, Copy, Default)] -struct BitContainer { - /// The bits are at the top. The lowest bits can be dirty. - bits: u64, - /// The number of bits is in the lowest byte. The rest is dirty. - pos: u64, -} - -impl BitContainer { - /// `HUF_addBits`: there is room for the code of `elt`. With `FAST`, the lowest byte of `elt` is not cleared - /// first, which takes that there is room for 4 bits more. - #[inline(always)] - fn add_bits(&mut self, elt: u64) { - self.bits >>= get_nb_bits(elt) & 63; - self.bits |= if FAST { elt } else { elt & !0xFF }; - self.pos = self.pos.wrapping_add(elt); - } - - #[inline(always)] - const fn nb_bits(self) -> u64 { - self.pos & 0xFF - } -} - -/// `HUF_CStream_t`. Its second container is a local variable of the loop that uses it. -struct HufCStream<'a> { - container: BitContainer, - dst: &'a mut [u8], - pos: usize, - /// The last position a whole container can be written to. - end: usize, -} - -impl<'a> HufCStream<'a> { - /// `HUF_initCStream` - #[inline] - fn new(dst: &'a mut [u8]) -> Result { - if dst.len() <= size_of::() { - return Err(ErrorCode::DstSizeTooSmall); - } - let end = dst.len() - size_of::(); - Ok(Self { - container: BitContainer::default(), - dst, - pos: 0, - end, - }) - } - - /// `HUF_mergeIndex1`: there is room for the bits of `second`, which come after those the stream has. - #[inline(always)] - fn merge(&mut self, second: BitContainer) { - self.container.bits >>= second.nb_bits() & 63; - self.container.bits |= second.bits; - self.container.pos = self.container.pos.wrapping_add(second.pos); - } - - /// `HUF_flushBits`: writes the whole bytes. At the end of the output the stream stops advancing, which - /// [`Self::close`] reports. The C code has a variant without that limit for when the output is known to be large - /// enough, but here the limit is what spares a check on the write. - #[inline(always)] - fn flush_bits(&mut self) { - let nb_bits = self.container.nb_bits(); - let nb_bytes = (nb_bits >> 3) as usize; - let bits = self.container.bits >> (u64::from(u64::BITS).wrapping_sub(nb_bits) & 63); - self.container.pos &= 7; - let pos = self.pos.min(self.end); - if let Ok(out) = self.dst.array_at_mut(pos) { - *out = bits.to_le_bytes(); - } - self.pos = pos + nb_bytes; - } - - /// `HUF_closeCStream`: the size of the stream, or 0 if it did not fit. - #[inline] - fn close(mut self) -> usize { - /// `HUF_endMark`: one bit, set. - const END_MARK: u64 = 1 | (1 << 63); - self.container.add_bits::(END_MARK); - self.flush_bits(); - if self.pos >= self.end { - return 0; - } - self.pos + usize::from(self.container.nb_bits() > 0) - } -} - -type Elements = [u64; HUF_SYMBOLVALUE_MAX as usize + 1]; - -/// `HUF_encodeSymbol` -#[inline(always)] -fn encode_symbol(container: &mut BitContainer, symbol: u8, ct: &Elements) { - container.add_bits::(ct.get(usize::from(symbol)).copied().unwrap_or(0)); -} - -/// Encodes `symbols` from the last to the first. -#[inline(always)] -fn encode_symbols( - container: &mut BitContainer, - symbols: &[u8; UNROLL], - ct: &Elements, -) { - if let Some((&last, symbols)) = symbols.split_first() { - for &symbol in symbols.iter().rev() { - encode_symbol::(container, symbol, ct); - } - encode_symbol::(container, last, ct); - } -} - -/// `HUF_compress1X_usingCTable_internal_body_loop`: a container has room for `UNROLL` symbols on top of the 7 bits -/// that a flush can leave, and, with `LAST_FAST`, for 4 bits more. -#[inline(always)] -fn compress_1x_body_loop( - bit_c: &mut HufCStream<'_>, - src: &[u8], - ct: &Elements, -) { - let (groups, rem) = src.as_chunks::(); - if !rem.is_empty() { - for &symbol in rem.iter().rev() { - encode_symbol::(&mut bit_c.container, symbol, ct); - } - bit_c.flush_bits(); - } - let (pairs, odd) = groups.as_chunks::<2>(); - if let [group] = odd { - encode_symbols::(&mut bit_c.container, group, ct); - bit_c.flush_bits(); - } - for [second, first] in pairs.iter().rev() { - encode_symbols::(&mut bit_c.container, first, ct); - bit_c.flush_bits(); - // The second group does not depend on the flush, since it has a container of its own. - let mut container = BitContainer::default(); - encode_symbols::(&mut container, second, ct); - bit_c.merge(container); - bit_c.flush_bits(); - } -} - -/// `HUF_tightCompressBound`: no output is larger, given the longest code. -const fn tight_compress_bound(src_size: usize, table_log: usize) -> usize { - ((src_size * table_log) >> 3) + 8 -} - -/// `HUF_compress1X_usingCTable`: the table has a code for every symbol in `src`. Returns the size of the output, or 0 -/// if `dst` is too small. -pub(crate) fn compress_1x_using_ctable( - dst: &mut [u8], - src: &[u8], - ctable: &HufCTable, -) -> Result { - let table_log = read_ctable_header(ctable).table_log; - let ct = ctable.elements(); - let dst_size = dst.len(); - - let Ok(mut bit_c) = HufCStream::new(dst) else { - return Ok(0); - }; - let bit_c_ref = &mut bit_c; - - if dst_size < tight_compress_bound(src.len(), usize::from(table_log)) || table_log > 11 { - compress_1x_body_loop::<4, false>(bit_c_ref, src, ct); - } else { - match table_log { - 11 => compress_1x_body_loop::<5, false>(bit_c_ref, src, ct), - 10 => compress_1x_body_loop::<5, true>(bit_c_ref, src, ct), - 9 => compress_1x_body_loop::<6, false>(bit_c_ref, src, ct), - 8 => compress_1x_body_loop::<7, false>(bit_c_ref, src, ct), - 7 => compress_1x_body_loop::<8, false>(bit_c_ref, src, ct), - _ => compress_1x_body_loop::<9, true>(bit_c_ref, src, ct), - } - } - Ok(bit_c.close()) -} - -/// `HUF_compress4X_usingCTable`: four streams, after the sizes of the first three. The table has a code for every -/// symbol in `src`. Returns the size of the output, or 0 if `src` is too small for it, `dst` is, or a stream is too -/// large. -pub(crate) fn compress_4x_using_ctable( - dst: &mut [u8], - src: &[u8], - ctable: &HufCTable, -) -> Result { - let segment_size = src.len().div_ceil(4); - if dst.len() < 6 + 1 + 1 + 1 + 8 { - return Ok(0); - } - if src.len() < 12 { - return Ok(0); - } - let mut op = 6usize; - for (stream, segment) in src.chunks(segment_size).enumerate() { - let c_size = compress_1x_using_ctable(dst.sub_mut(op..)?, segment, ctable)?; - if c_size == 0 || c_size > 65535 { - return Ok(0); - } - if stream < 3 { - write_le16(dst, stream * 2, c_size as u16)?; - } - op += c_size; - } - Ok(op) -} - -/// `HUF_nbStreams_e` -#[derive(Clone, Copy, PartialEq, Eq)] -enum NbStreams { - Single, - Four, -} - -/// `HUF_compressCTable_internal`: writes from `op` on. Returns how much of `dst` is used, or 0 if that saves nothing. -fn compress_ctable_internal( - dst: &mut [u8], - op: usize, - src: &[u8], - nb_streams: NbStreams, - ctable: &HufCTable, -) -> Result { - let out = dst.sub_mut(op..)?; - let c_size = match nb_streams { - NbStreams::Single => compress_1x_using_ctable(out, src, ctable)?, - NbStreams::Four => compress_4x_using_ctable(out, src, ctable)?, - }; - if c_size == 0 { - return Ok(0); - } - if op + c_size >= src.len().wrapping_sub(1) { - return Ok(0); - } - Ok(op + c_size) -} - -const SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE: usize = 4096; -const SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO: usize = 10; - -/// `HUF_cardinality`: how many counts up to `max_symbol_value` are not 0. -pub(crate) fn cardinality(count: &[u32], max_symbol_value: u32) -> u32 { - let counts = count.iter().take(max_symbol_value as usize + 1); - counts.filter(|&&count| count != 0).count() as u32 -} - -/// `HUF_minTableLog`: the shortest that the longest code can be. `symbol_cardinality` is not 0. -pub(crate) fn min_table_log(symbol_cardinality: u32) -> u32 { - highbit32(symbol_cardinality) + 1 -} - -/// `HUF_optimalTableLog`: the limit to put on the length of the codes. `src_size` is more than 1. With -/// `flags.optimal_depth`, `table` is overwritten. -pub(crate) fn optimal_table_log( - max_table_log: u32, - src_size: usize, - max_symbol_value: u32, - table: &mut HufCTable, - count: &[u32], - flags: HufFlags, -) -> u32 { - if !flags.optimal_depth { - return fse::optimal_table_log_internal(max_table_log, src_size, max_symbol_value, 1); - } - - // Large enough that no step of `write_ctable` is short of room. - let mut dst = [0u8; 512]; - let symbol_cardinality = cardinality(count, max_symbol_value); - let min_table_log = min_table_log(symbol_cardinality); - let mut opt_size = usize::MAX - 1; - let mut opt_log = max_table_log; - - // Search until the size increases. - for opt_log_guess in min_table_log..=max_table_log { - let Ok(max_bits) = build_ctable(table, count, max_symbol_value, opt_log_guess) else { - continue; - }; - if max_bits < opt_log_guess as usize && opt_log_guess > min_table_log { - break; - } - let Ok(h_size) = write_ctable(&mut dst, table, max_symbol_value, max_bits as u32) else { - continue; - }; - let new_size = estimate_compressed_size(table, count, max_symbol_value) + h_size; - if new_size > opt_size + 1 { - break; - } - if new_size < opt_size { - opt_size = new_size; - opt_log = opt_log_guess; - } - } - opt_log -} - -/// `HUF_compress_internal` -fn compress_internal( - dst: &mut [u8], - src: &[u8], - mut max_symbol_value: u32, - mut huff_log: u32, - nb_streams: NbStreams, - old_huf_table: &mut HufCTable, - repeat: &mut HufRepeat, - flags: HufFlags, -) -> Result { - let src_size = src.len(); - let Some(&first_byte) = src.first() else { - return Ok(0); - }; - if dst.is_empty() { - return Ok(0); - } - if src_size > HUF_BLOCKSIZE_MAX { - return Err(ErrorCode::SrcSizeWrong); - } - if huff_log > HUF_TABLELOG_MAX { - return Err(ErrorCode::TableLogTooLarge); - } - if max_symbol_value > HUF_SYMBOLVALUE_MAX { - return Err(ErrorCode::MaxSymbolValueTooLarge); - } - if max_symbol_value == 0 { - max_symbol_value = HUF_SYMBOLVALUE_MAX; - } - if huff_log == 0 { - huff_log = HUF_TABLELOG_DEFAULT; - } - - if flags.prefer_repeat && *repeat == HufRepeat::Valid { - return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); - } - - let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; - - // If the input may well be incompressible, look at its start and its end before all of it. - if flags.suspect_uncompressible - && src_size >= SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO - { - let mut largest_total = 0usize; - for sample in [ - src.sub(..SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE)?, - src.sub(src_size - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE..)?, - ] { - let mut max_symbol_value_sample = max_symbol_value; - largest_total += - hist::count_simple(&mut count, &mut max_symbol_value_sample, sample)? as usize; - } - if largest_total <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7) + 4 { - return Ok(0); - } - } - - let largest = hist::count_wksp(&mut count, &mut max_symbol_value, src)?; - if largest == src_size { - // A single symbol: run-length encoding. - dst.put(0, first_byte)?; - return Ok(1); - } - if largest <= (src_size >> 7) + 4 { - // Probably not compressible enough. - return Ok(0); - } - - if *repeat == HufRepeat::Check && !validate_ctable(old_huf_table, &count, max_symbol_value) { - *repeat = HufRepeat::None; - } - if flags.prefer_repeat && *repeat != HufRepeat::None { - return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); - } - - let mut ctable = HufCTable::new(); - huff_log = optimal_table_log( - huff_log, - src_size, - max_symbol_value, - &mut ctable, - &count, - flags, - ); - huff_log = build_ctable(&mut ctable, &count, max_symbol_value, huff_log)? as u32; - - let h_size = write_ctable(dst, &ctable, max_symbol_value, huff_log)?; - // Is the previous table the better choice, since it takes no header? - if *repeat != HufRepeat::None { - let old_size = estimate_compressed_size(old_huf_table, &count, max_symbol_value); - let new_size = estimate_compressed_size(&ctable, &count, max_symbol_value); - if old_size <= h_size + new_size || h_size + 12 >= src_size { - return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); - } - } - - if h_size + 12 >= src_size { - return Ok(0); - } - *repeat = HufRepeat::None; - old_huf_table.clone_from(&ctable); - compress_ctable_internal(dst, h_size, src, nb_streams, &ctable) -} - -/// `HUF_compress1X_repeat`: one stream, after the header of its table, or without one if `huf_table` is used again, -/// which `repeat` says is allowed. If it is not used again, `repeat` becomes [`HufRepeat::None`] and `huf_table` the -/// new table. Returns the size of the output, 0 if `src` is not compressible, or 1 if it is one byte repeated, which -/// is then the output. -pub(crate) fn compress_1x_repeat( - dst: &mut [u8], - src: &[u8], - max_symbol_value: u32, - huff_log: u32, - huf_table: &mut HufCTable, - repeat: &mut HufRepeat, - flags: HufFlags, -) -> Result { - compress_internal( - dst, - src, - max_symbol_value, - huff_log, - NbStreams::Single, - huf_table, - repeat, - flags, - ) -} - -/// `HUF_compress4X_repeat`: as [`compress_1x_repeat`], with four streams. -pub(crate) fn compress_4x_repeat( - dst: &mut [u8], - src: &[u8], - max_symbol_value: u32, - huff_log: u32, - huf_table: &mut HufCTable, - repeat: &mut HufRepeat, - flags: HufFlags, -) -> Result { - compress_internal( - dst, - src, - max_symbol_value, - huff_log, - NbStreams::Four, - huf_table, - repeat, - flags, - ) -} diff --git a/src/zstd/wip/work/entropy/zstd/compress/internal.rs b/src/zstd/wip/work/entropy/zstd/compress/internal.rs deleted file mode 100644 index 51157ea6c1ce..000000000000 --- a/src/zstd/wip/work/entropy/zstd/compress/internal.rs +++ /dev/null @@ -1,1212 +0,0 @@ -//! `zstd_compress_internal.h`: what the parts of the compressor share. - -use super::fse::{FseCTable, FseRepeat}; -use super::huf::{HufCTable, HufRepeat}; -use crate::common::{ - BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, - ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, -}; -use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; - -pub(crate) const K_SEARCH_STRENGTH: u32 = 8; -pub(crate) const HASH_READ_SIZE: usize = 8; -/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. -pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; - -pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; -pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; - -pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; -pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; - -/// `ZSTD_strategy` -#[repr(u8)] -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] -pub(crate) enum Strategy { - Fast = 1, - DFast = 2, - Greedy = 3, - Lazy = 4, - Lazy2 = 5, - BtLazy2 = 6, - BtOpt = 7, - BtUltra = 8, - BtUltra2 = 9, -} - -impl Strategy { - pub(crate) const fn from_int(value: i32) -> Option { - Some(match value { - 1 => Self::Fast, - 2 => Self::DFast, - 3 => Self::Greedy, - 4 => Self::Lazy, - 5 => Self::Lazy2, - 6 => Self::BtLazy2, - 7 => Self::BtOpt, - 8 => Self::BtUltra, - 9 => Self::BtUltra2, - _ => return None, - }) - } -} - -/// `ZSTD_ParamSwitch_e` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum ParamSwitch { - Auto = 0, - Enable = 1, - Disable = 2, -} - -/// `ZSTD_compressionParameters` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) struct CompressionParameters { - pub(crate) window_log: u32, - pub(crate) chain_log: u32, - pub(crate) hash_log: u32, - pub(crate) search_log: u32, - pub(crate) min_match: u32, - pub(crate) target_length: u32, - /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. - pub(crate) strategy: Option, -} - -impl CompressionParameters { - /// The strategy of parameters that are ready to be applied. - #[inline(always)] - pub(crate) fn strat(&self) -> Strategy { - self.strategy.unwrap_or(Strategy::Fast) - } -} - -/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum DictMode { - NoDict, - ExtDict, - DictMatchState, -} - -/// `ZSTD_dictTableLoadMethod_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum DictTableLoadMethod { - Fast, - Full, -} - -/// `ZSTD_tableFillPurpose_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum TableFillPurpose { - ForCCtx, - ForCDict, -} - -// ── entropy tables ──────────────────────────────────────────────────────── - -/// `ZSTD_hufCTables_t` -#[derive(Clone)] -pub(crate) struct HufCTables { - pub(crate) c_table: HufCTable, - pub(crate) repeat_mode: HufRepeat, -} - -/// `ZSTD_fseCTables_t` -#[derive(Clone)] -pub(crate) struct FseCTables { - pub(crate) offcode_c_table: FseCTable, - pub(crate) matchlength_c_table: FseCTable, - pub(crate) litlength_c_table: FseCTable, - pub(crate) offcode_repeat_mode: FseRepeat, - pub(crate) matchlength_repeat_mode: FseRepeat, - pub(crate) litlength_repeat_mode: FseRepeat, -} - -/// `ZSTD_entropyCTables_t` -#[derive(Clone)] -pub(crate) struct EntropyCTables { - pub(crate) huf: HufCTables, - pub(crate) fse: FseCTables, -} - -/// `ZSTD_compressedBlockState_t` -#[derive(Clone)] -pub(crate) struct CompressedBlockState { - pub(crate) entropy: EntropyCTables, - pub(crate) rep: [u32; ZSTD_REP_NUM], -} - -impl CompressedBlockState { - /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. - pub(crate) fn new() -> Self { - Self { - entropy: EntropyCTables { - huf: HufCTables { - c_table: HufCTable::new(), - repeat_mode: HufRepeat::None, - }, - fse: FseCTables { - offcode_c_table: FseCTable::new(), - matchlength_c_table: FseCTable::new(), - litlength_c_table: FseCTable::new(), - offcode_repeat_mode: FseRepeat::None, - matchlength_repeat_mode: FseRepeat::None, - litlength_repeat_mode: FseRepeat::None, - }, - }, - rep: REP_START_VALUE, - } - } - - /// `ZSTD_reset_compressedBlockState` - pub(crate) fn reset(&mut self) { - self.rep = REP_START_VALUE; - self.entropy.huf.repeat_mode = HufRepeat::None; - self.entropy.fse.offcode_repeat_mode = FseRepeat::None; - self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; - self.entropy.fse.litlength_repeat_mode = FseRepeat::None; - } -} - -/// `ZSTD_hufCTablesMetadata_t` -pub(crate) struct HufCTablesMetadata { - pub(crate) h_type: SymbolEncodingType, - pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], - pub(crate) huf_des_size: usize, -} - -/// `ZSTD_fseCTablesMetadata_t` -pub(crate) struct FseCTablesMetadata { - pub(crate) ll_type: SymbolEncodingType, - pub(crate) of_type: SymbolEncodingType, - pub(crate) ml_type: SymbolEncodingType, - pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], - pub(crate) fse_tables_size: usize, - /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. - pub(crate) last_count_size: usize, -} - -/// `ZSTD_entropyCTablesMetadata_t` -pub(crate) struct EntropyCTablesMetadata { - pub(crate) huf_metadata: HufCTablesMetadata, - pub(crate) fse_metadata: FseCTablesMetadata, -} - -impl EntropyCTablesMetadata { - pub(crate) const fn new() -> Self { - Self { - huf_metadata: HufCTablesMetadata { - h_type: SymbolEncodingType::Basic, - huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], - huf_des_size: 0, - }, - fse_metadata: FseCTablesMetadata { - ll_type: SymbolEncodingType::Basic, - of_type: SymbolEncodingType::Basic, - ml_type: SymbolEncodingType::Basic, - fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], - fse_tables_size: 0, - last_count_size: 0, - }, - } - } -} - -// ── sequences ───────────────────────────────────────────────────────────── - -/// `SeqDef` -#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] -pub(crate) struct SeqDef { - /// A repeat code (1 to 3), or an offset plus 3. - pub(crate) off_base: u32, - pub(crate) lit_length: u16, - /// The match length minus [`MINMATCH`]. - pub(crate) ml_base: u16, -} - -/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum LongLengthType { - None, - LiteralLength, - MatchLength, -} - -/// `SeqStore_t`: the sequences and the literals of a block. -pub(crate) struct SeqStore { - /// Never grows beyond the capacity it is given by [`Self::reserve`]. - pub(crate) sequences: Vec, - /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. - pub(crate) lit_buffer: Vec, - /// How much of `lit_buffer` is literals. - pub(crate) lit_len: usize, - pub(crate) ll_code: Vec, - pub(crate) ml_code: Vec, - pub(crate) of_code: Vec, - pub(crate) max_nb_seq: usize, - pub(crate) max_nb_lit: usize, - pub(crate) long_length_type: LongLengthType, - /// Which sequence has the long length. - pub(crate) long_length_pos: u32, -} - -/// `ZSTD_SequenceLength` -#[derive(Clone, Copy)] -pub(crate) struct SequenceLength { - pub(crate) lit_length: u32, - pub(crate) match_length: u32, -} - -impl SeqStore { - pub(crate) const fn new() -> Self { - Self { - sequences: Vec::new(), - lit_buffer: Vec::new(), - lit_len: 0, - ll_code: Vec::new(), - ml_code: Vec::new(), - of_code: Vec::new(), - max_nb_seq: 0, - max_nb_lit: 0, - long_length_type: LongLengthType::None, - long_length_pos: 0, - } - } - - /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. - pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { - fn grow(vec: &mut Vec, len: usize) -> Result<()> { - vec.clear(); - vec.try_reserve_exact(len) - .map_err(|_| ErrorCode::MemoryAllocation)?; - vec.resize(len, T::default()); - Ok(()) - } - grow(&mut self.sequences, max_nb_seq)?; - grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; - grow(&mut self.ll_code, max_nb_seq)?; - grow(&mut self.ml_code, max_nb_seq)?; - grow(&mut self.of_code, max_nb_seq)?; - self.max_nb_seq = max_nb_seq; - self.max_nb_lit = max_nb_lit; - self.reset(); - Ok(()) - } - - /// `ZSTD_resetSeqStore` - #[inline] - pub(crate) fn reset(&mut self) { - self.lit_len = 0; - self.sequences.clear(); - self.long_length_type = LongLengthType::None; - } - - #[inline] - pub(crate) fn literals(&self) -> &[u8] { - self.lit_buffer.get(..self.lit_len).unwrap_or_default() - } - - /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. - #[inline(always)] - pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { - debug_assert!(self.sequences.len() < self.max_nb_seq); - if lit_length > 0xFFFF { - self.long_length_type = LongLengthType::LiteralLength; - self.long_length_pos = self.sequences.len() as u32; - } - let ml_base = match_length.wrapping_sub(MINMATCH as usize); - if ml_base > 0xFFFF { - self.long_length_type = LongLengthType::MatchLength; - self.long_length_pos = self.sequences.len() as u32; - } - self.sequences.push(SeqDef { - off_base, - lit_length: lit_length as u16, - ml_base: ml_base as u16, - }); - } - - /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. - #[inline(always)] - pub(crate) fn store_seq( - &mut self, - lit_length: usize, - buf: &[u8], - literals: usize, - off_base: u32, - match_length: usize, - ) -> Result<()> { - match ( - buf.array_at::<16>(literals), - self.lit_buffer.array_at_mut::<16>(self.lit_len), - ) { - (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, - _ => self - .lit_buffer - .write_at(self.lit_len, buf.span(literals, lit_length)?)?, - } - self.lit_len += lit_length; - self.store_seq_only(lit_length, off_base, match_length); - Ok(()) - } - - /// Appends the literals that follow the last sequence of a block. - #[inline] - pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { - self.lit_buffer.write_at(self.lit_len, literals)?; - self.lit_len += literals.len(); - Ok(()) - } -} - -pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; -pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; -pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; - -/// `OFFSET_TO_OFFBASE`: `offset` is not 0. -#[inline(always)] -pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { - offset.wrapping_add(ZSTD_REP_NUM as u32) -} - -/// `OFFBASE_IS_OFFSET` -#[inline(always)] -pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { - off_base > ZSTD_REP_NUM as u32 -} - -/// `OFFBASE_IS_REPCODE` -#[inline(always)] -pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { - 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 -} - -/// `OFFBASE_TO_OFFSET` -#[inline(always)] -pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { - off_base.wrapping_sub(ZSTD_REP_NUM as u32) -} - -/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. -#[inline(always)] -pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { - let [rep0, rep1, rep2] = *rep; - if offbase_is_offset(off_base) { - *rep = [offbase_to_offset(off_base), rep0, rep1]; - } else { - match off_base - 1 + u32::from(ll0) { - 0 => {} - 1 => *rep = [rep1, rep0, rep2], - 2 => *rep = [rep2, rep0, rep1], - _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], - } - } -} - -/// `ZSTD_newRep` -#[inline(always)] -pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { - let mut new_reps = *rep; - update_rep(&mut new_reps, off_base, ll0); - new_reps -} - -/// `ZSTD_LLcode` -#[inline(always)] -pub(crate) fn ll_code(lit_length: u32) -> u32 { - #[rustfmt::skip] - const LL_CODE: [u8; 64] = [ - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, - ]; - const LL_DELTA_CODE: u32 = 19; - match LL_CODE.get(lit_length as usize) { - Some(&code) => u32::from(code), - None => highbit32(lit_length) + LL_DELTA_CODE, - } -} - -/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. -#[inline(always)] -pub(crate) fn ml_code(ml_base: u32) -> u32 { - #[rustfmt::skip] - const ML_CODE: [u8; 128] = [ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - ]; - const ML_DELTA_CODE: u32 = 36; - match ML_CODE.get(ml_base as usize) { - Some(&code) => u32::from(code), - None => highbit32(ml_base) + ML_DELTA_CODE, - } -} - -/// `ZSTD_noCompressBlock` -pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { - let c_block_header24 = - u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); - if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { - return Err(ErrorCode::DstSizeTooSmall); - } - write_le24(dst, 0, c_block_header24)?; - dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; - Ok(ZSTD_BLOCKHEADERSIZE + src.len()) -} - -/// `ZSTD_rleCompressBlock` -pub(crate) fn rle_compress_block( - dst: &mut [u8], - byte: u8, - src_size: usize, - last_block: bool, -) -> Result { - let c_block_header = - u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); - if dst.len() < 4 { - return Err(ErrorCode::DstSizeTooSmall); - } - write_le24(dst, 0, c_block_header)?; - dst.put(3, byte)?; - Ok(4) -} - -/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. -#[inline] -pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { - let minlog = if strat >= Strategy::BtUltra { - strat as u32 - 1 - } else { - 6 - }; - (src_size >> minlog) + 2 -} - -// ── matching ────────────────────────────────────────────────────────────── - -/// How many bytes `a` and `b` have in common at their start. -#[inline(always)] -pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { - let (a_words, _) = a.as_chunks::<8>(); - let (b_words, _) = b.as_chunks::<8>(); - let mut len = 0usize; - for (a_word, b_word) in a_words.iter().zip(b_words) { - let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); - if diff != 0 { - return len + (diff.trailing_zeros() >> 3) as usize; - } - len += 8; - } - let a_tail = a.get(len..).unwrap_or_default(); - let b_tail = b.get(len..).unwrap_or_default(); - len + a_tail - .iter() - .zip(b_tail) - .take_while(|(a_byte, b_byte)| a_byte == b_byte) - .count() -} - -/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. -#[inline(always)] -pub(crate) fn count( - buf: &[u8], - p_in: usize, - match_buf: &[u8], - p_match: usize, - p_in_limit: usize, -) -> Result { - Ok(common_prefix( - buf.sub(p_in..p_in_limit)?, - match_buf.sub(p_match..)?, - )) -} - -/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at -/// `i_start` in `buf`. -#[inline(always)] -pub(crate) fn count_2segments( - buf: &[u8], - ip: usize, - match_buf: &[u8], - match_pos: usize, - i_end: usize, - m_end: usize, - i_start: usize, -) -> Result { - let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); - if match_pos + match_length != m_end { - return Ok(match_length); - } - Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) -} - -const PRIME_3_BYTES: u32 = 506_832_829; -const PRIME_4_BYTES: u32 = 2_654_435_761; -const PRIME_5_BYTES: u64 = 889_523_592_379; -const PRIME_6_BYTES: u64 = 227_718_039_650_203; -const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; -pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; - -/// `ZSTD_hash3` -#[inline(always)] -const fn hash3(u: u32, h: u32, s: u32) -> u32 { - (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash4(u: u32, h: u32, s: u32) -> u32 { - ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash5(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash6(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash7(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -/// `ZSTD_hash8` -#[inline(always)] -pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { - (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} - -/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. -#[inline(always)] -pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { - Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) -} - -/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, -/// else 8. -#[inline(always)] -pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { - hash_ptr_salted(buf, pos, h_bits, mls, 0) -} - -/// `ZSTD_hashPtrSalted` -#[inline(always)] -pub(crate) fn hash_ptr_salted( - buf: &[u8], - pos: usize, - h_bits: u32, - mls: u32, - hash_salt: u64, -) -> Result { - Ok(match mls { - 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), - 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), - 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), - 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), - _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, - }) -} - -// ── the window ──────────────────────────────────────────────────────────── - -/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are -/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the -/// others, and an address is turned into a position in the buffer only to read. -pub(crate) type Addr = usize; - -/// The buffers that a window can have a segment in. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum BufId { - /// Where an empty window points. - None = 0, - /// The context's own buffer of input. - InBuff = 1, - /// What the caller gave to the call in progress. - User = 2, - /// The content of the dictionary, or the prefix. - Dict = 3, -} - -impl BufId { - /// The address of the first byte of the buffer. - #[inline(always)] - pub(crate) const fn start(self) -> Addr { - (self as usize + 1) << 44 - } -} - -/// The buffers themselves, for as long as a call lasts. -#[derive(Clone, Copy)] -pub(crate) struct Buffers<'a> { - pub(crate) in_buff: &'a [u8], - pub(crate) user: &'a [u8], - pub(crate) dict: &'a [u8], -} - -impl<'a> Buffers<'a> { - #[inline] - pub(crate) fn get(&self, id: BufId) -> &'a [u8] { - match id { - BufId::None => &[], - BufId::InBuff => self.in_buff, - BufId::User => self.user, - BufId::Dict => self.dict, - } - } -} - -/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. -pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; - -/// The index past which indices are brought down. -pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; -/// The most input that can be indexed at once. -pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; - -/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the -/// index is less than `dict_limit`. -#[derive(Clone, Copy, Debug)] -pub(crate) struct Window { - /// Where input has to continue to be contiguous. - pub(crate) next_src: Addr, - pub(crate) base: Addr, - pub(crate) dict_base: Addr, - /// The buffer of the segment that starts at `dict_limit`. - pub(crate) base_buf: BufId, - /// The buffer of the segment from `low_limit` to `dict_limit`. - pub(crate) dict_buf: BufId, - pub(crate) dict_limit: u32, - /// Below this, there is nothing valid. - pub(crate) low_limit: u32, - pub(crate) nb_overflow_corrections: u32, -} - -impl Window { - /// `ZSTD_window_init` - pub(crate) const fn new() -> Self { - let base = BufId::None.start(); - Self { - next_src: base + ZSTD_WINDOW_START_INDEX as usize, - base, - dict_base: base, - base_buf: BufId::None, - dict_buf: BufId::None, - dict_limit: ZSTD_WINDOW_START_INDEX, - low_limit: ZSTD_WINDOW_START_INDEX, - nb_overflow_corrections: 0, - } - } - - /// The index of `addr`, which is in the current segment or at its end. - #[inline(always)] - pub(crate) const fn index_of(&self, addr: Addr) -> u32 { - addr.wrapping_sub(self.base) as u32 - } - - /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. - pub(crate) fn clear(&mut self) { - let end = self.index_of(self.next_src); - self.low_limit = end; - self.dict_limit = end; - } - - /// `ZSTD_window_isEmpty` - pub(crate) fn is_empty(&self) -> bool { - self.dict_limit == ZSTD_WINDOW_START_INDEX - && self.low_limit == ZSTD_WINDOW_START_INDEX - && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize - } - - /// `ZSTD_window_hasExtDict` - #[inline] - pub(crate) fn has_ext_dict(&self) -> bool { - self.low_limit < self.dict_limit - } - - /// `ZSTD_window_needOverflowCorrection` - #[inline] - pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { - self.index_of(src_end) > ZSTD_CURRENT_MAX - } - - /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns - /// what to subtract from every index that is stored. - pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { - let cycle_size = 1u32 << cycle_log; - let cycle_mask = cycle_size - 1; - let curr = self.index_of(src); - let current_cycle = curr & cycle_mask; - // Indices must not fall below the start index. - let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { - cycle_size.max(ZSTD_WINDOW_START_INDEX) - } else { - 0 - }; - let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); - let correction = curr.wrapping_sub(new_current); - - self.base = self.base.wrapping_add(correction as usize); - self.dict_base = self.dict_base.wrapping_add(correction as usize); - self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { - ZSTD_WINDOW_START_INDEX - } else { - self.low_limit - correction - }; - self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { - ZSTD_WINDOW_START_INDEX - } else { - self.dict_limit - correction - }; - self.nb_overflow_corrections += 1; - correction - } - - /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the - /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. - pub(crate) fn enforce_max_dist( - &mut self, - block_end: Addr, - max_dist: u32, - loaded_dict_end: u32, - ) -> bool { - let block_end_idx = self.index_of(block_end); - if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { - let new_low_limit = block_end_idx - max_dist; - if self.low_limit < new_low_limit { - self.low_limit = new_low_limit; - } - if self.dict_limit < self.low_limit { - self.dict_limit = self.low_limit; - } - return true; - } - false - } - - /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not - /// right before the current segment, and is to be forgotten. - pub(crate) fn dict_is_invalid( - &self, - block_end: Addr, - max_dist: u32, - loaded_dict_end: u32, - ) -> bool { - let block_end_idx = self.index_of(block_end); - block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit - } - - /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not - /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. - pub(crate) fn update( - &mut self, - src: Addr, - src_buf: BufId, - src_size: usize, - force_non_contiguous: bool, - ) -> bool { - let mut contiguous = true; - if src_size == 0 { - return contiguous; - } - if src != self.next_src || force_non_contiguous { - let distance_from_base = self.next_src.wrapping_sub(self.base); - self.low_limit = self.dict_limit; - self.dict_limit = distance_from_base as u32; - self.dict_base = self.base; - self.dict_buf = self.base_buf; - self.base = src.wrapping_sub(distance_from_base); - self.base_buf = src_buf; - if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { - // Too small to be of use. - self.low_limit = self.dict_limit; - } - contiguous = false; - } - self.next_src = src + src_size; - // Where the input overlaps the older segment, it has overwritten it. - if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) - & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) - { - let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); - self.low_limit = if high_input_idx > self.dict_limit as usize { - self.dict_limit - } else { - high_input_idx as u32 - }; - } - contiguous - } -} - -/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes -/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. -#[inline(always)] -pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { - prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 -} - -pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; -pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; - -/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. -#[inline(always)] -pub(crate) fn write_tagged_index( - hash_table: &mut [u32], - hash_and_tag: usize, - index: u32, -) -> Result<()> { - let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; - hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) -} - -/// `ZSTD_comparePackedTags` -#[inline(always)] -pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { - (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) - == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) -} - -// ── the state of the match finders ──────────────────────────────────────── - -/// `ZSTD_match_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct Match { - /// As [`SeqDef::off_base`]. - pub(crate) off: u32, - pub(crate) len: u32, -} - -/// `rawSeq`: a sequence found by long distance matching. -#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] -pub(crate) struct RawSeq { - pub(crate) offset: u32, - pub(crate) lit_length: u32, - pub(crate) match_length: u32, -} - -/// `RawSeqStore_t` -#[derive(Clone, Default)] -pub(crate) struct RawSeqStore { - /// `capacity` of them, of which `size` are in use. - pub(crate) seq: Vec, - /// The sequence that is being read. - pub(crate) pos: usize, - /// How many of its bytes, literals first, are read. - pub(crate) pos_in_sequence: usize, - pub(crate) size: usize, - pub(crate) capacity: usize, -} - -/// `ZSTD_optimal_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct Optimal { - /// The cost of getting to this position. - pub(crate) price: i32, - pub(crate) off: u32, - pub(crate) mlen: u32, - pub(crate) litlen: u32, - pub(crate) rep: [u32; ZSTD_REP_NUM], -} - -/// `ZSTD_OptPrice_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum OptPrice { - Dynamic, - Predef, -} - -/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. -pub(crate) struct OptState { - pub(crate) lit_freq: Vec, - pub(crate) lit_length_freq: Vec, - pub(crate) match_length_freq: Vec, - pub(crate) off_code_freq: Vec, - pub(crate) match_table: Vec, - pub(crate) price_table: Vec, - pub(crate) lit_sum: u32, - pub(crate) lit_length_sum: u32, - pub(crate) match_length_sum: u32, - pub(crate) off_code_sum: u32, - pub(crate) lit_sum_base_price: u32, - pub(crate) lit_length_sum_base_price: u32, - pub(crate) match_length_sum_base_price: u32, - pub(crate) off_code_sum_base_price: u32, - pub(crate) price_type: OptPrice, - pub(crate) literal_compression_mode: ParamSwitch, -} - -impl OptState { - pub(crate) const fn new() -> Self { - Self { - lit_freq: Vec::new(), - lit_length_freq: Vec::new(), - match_length_freq: Vec::new(), - off_code_freq: Vec::new(), - match_table: Vec::new(), - price_table: Vec::new(), - lit_sum: 0, - lit_length_sum: 0, - match_length_sum: 0, - off_code_sum: 0, - lit_sum_base_price: 0, - lit_length_sum_base_price: 0, - match_length_sum_base_price: 0, - off_code_sum_base_price: 0, - price_type: OptPrice::Dynamic, - literal_compression_mode: ParamSwitch::Auto, - } - } -} - -pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; - -/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures -/// own, are in [`BlockSrc`]. -pub(crate) struct MatchState { - pub(crate) window: Window, - /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. - pub(crate) loaded_dict_end: u32, - /// The index from which the tables are still to be filled. - pub(crate) next_to_update: u32, - /// 0 if there is no table of 3-byte hashes. - pub(crate) hash_log3: u32, - pub(crate) row_hash_log: u32, - /// For the row-based match finder: a byte of hash for each entry of `hash_table`. - pub(crate) tag_table: Vec, - pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], - /// Makes what a previous frame left in `tag_table` meaningless. - pub(crate) hash_salt: u64, - pub(crate) hash_salt_entropy: u32, - pub(crate) hash_table: Vec, - pub(crate) hash_table3: Vec, - pub(crate) chain_table: Vec, - pub(crate) force_non_contiguous: bool, - pub(crate) opt: OptState, - /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. - pub(crate) has_dict_match_state: bool, - pub(crate) c_params: CompressionParameters, - pub(crate) lazy_skipping: bool, -} - -impl MatchState { - pub(crate) const fn new() -> Self { - Self { - window: Window::new(), - loaded_dict_end: 0, - next_to_update: 0, - hash_log3: 0, - row_hash_log: 0, - tag_table: Vec::new(), - hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], - hash_salt: 0, - hash_salt_entropy: 0, - hash_table: Vec::new(), - hash_table3: Vec::new(), - chain_table: Vec::new(), - force_non_contiguous: false, - opt: OptState::new(), - has_dict_match_state: false, - c_params: CompressionParameters { - window_log: 0, - chain_log: 0, - hash_log: 0, - search_log: 0, - min_match: 0, - target_length: 0, - strategy: None, - }, - lazy_skipping: false, - } - } - - /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. - pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { - if self.window.has_ext_dict() { - return DictMode::ExtDict; - } - if dms.is_some() { - DictMode::DictMatchState - } else { - DictMode::NoDict - } - } - - /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. - #[inline(always)] - pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { - let max_distance = 1u32 << window_log; - let lowest_valid = self.window.low_limit; - let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { - curr - max_distance - } else { - lowest_valid - }; - // A dictionary is valid as a whole until it is invalidated as a whole. - if self.loaded_dict_end != 0 { - lowest_valid - } else { - within_window - } - } - - /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. - #[inline(always)] - pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { - let max_distance = 1u32 << window_log; - let lowest_valid = self.window.dict_limit; - let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { - curr - max_distance - } else { - lowest_valid - }; - if self.loaded_dict_end != 0 { - lowest_valid - } else { - within_window - } - } -} - -/// The match state of a dictionary, and where its indices lead. -#[derive(Clone, Copy)] -pub(crate) struct DictMatchState<'a> { - pub(crate) ms: &'a MatchState, - /// The buffer its window is in, which is one segment. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, - /// `window.nextSrc`, as a position in `buf`. - pub(crate) end: usize, -} - -/// The current segment of a window, for the functions that index a dictionary. -#[derive(Clone, Copy)] -pub(crate) struct Prefix<'a> { - /// The buffer that the segment is in. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, -} - -impl Prefix<'_> { - /// The index of the position `pos` of `buf`. - #[inline(always)] - pub(crate) const fn index_of(&self, pos: usize) -> u32 { - pos.wrapping_sub(self.base) as u32 - } - - /// The position in `buf` of `index`. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } -} - -/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. -#[derive(Clone, Copy)] -pub(crate) struct BlockSrc<'a> { - /// The buffer that the current segment of the window is in, and so the block. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, - /// The buffer that the older segment of the window is in. - pub(crate) dict_buf: &'a [u8], - /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. - pub(crate) dict_base: usize, - /// Where the block starts in `buf`. - pub(crate) start: usize, - /// Where it ends. - pub(crate) end: usize, - pub(crate) dms: Option>, - /// `opt.symbolCosts`: the tables of the previous block. - pub(crate) symbol_costs: &'a EntropyCTables, - /// `ldmSeqStore`: matches that the optimal parser is to consider. - pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, -} - -impl<'a> BlockSrc<'a> { - #[inline(always)] - pub(crate) const fn prefix(&self) -> Prefix<'a> { - Prefix { - buf: self.buf, - base: self.base, - } - } - - /// The index of the position `pos` of `buf`. - #[inline(always)] - pub(crate) const fn index_of(&self, pos: usize) -> u32 { - pos.wrapping_sub(self.base) as u32 - } - - /// The position in `buf` of `index`, which is in the current segment. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } - - /// The position in `dict_buf` of `index`, which is in the older segment. - #[inline(always)] - pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { - self.dict_base.wrapping_add(index as usize) - } -} - -impl DictMatchState<'_> { - /// The position in `buf` of `index`. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } -} - -// ── long distance matching ──────────────────────────────────────────────── - -/// `ldmEntry_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct LdmEntry { - pub(crate) offset: u32, - pub(crate) checksum: u32, -} - -/// `ldmParams_t` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) struct LdmParams { - pub(crate) enable_ldm: ParamSwitch, - pub(crate) hash_log: u32, - pub(crate) bucket_size_log: u32, - pub(crate) min_match_length: u32, - pub(crate) hash_rate_log: u32, - pub(crate) window_log: u32, -} - -/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. -pub(crate) struct LdmState { - pub(crate) window: Window, - pub(crate) hash_table: Vec, - pub(crate) loaded_dict_end: u32, - /// The next entry to replace in each bucket. - pub(crate) bucket_offsets: Vec, -} - -impl LdmState { - pub(crate) const fn new() -> Self { - Self { - window: Window::new(), - hash_table: Vec::new(), - loaded_dict_end: 0, - bucket_offsets: Vec::new(), - } - } -} diff --git a/src/zstd/wip/work/entropy/zstd/compress/lazy.rs b/src/zstd/wip/work/entropy/zstd/compress/lazy.rs deleted file mode 100644 index 9f653fd684f3..000000000000 --- a/src/zstd/wip/work/entropy/zstd/compress/lazy.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. - -use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _ip: usize, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_row_update`: the same, in the rows of the row-based match finder. -pub(crate) fn row_update(_ms: &mut MatchState, _prefix: Prefix<'_>, _ip: usize) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: -/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. -pub(crate) fn compress_block_lazy( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, - _strategy: Strategy, - _use_row: bool, - _dict_mode: DictMode, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/entropy/zstd/compress/ldm.rs b/src/zstd/wip/work/entropy/zstd/compress/ldm.rs deleted file mode 100644 index 2177fd945e10..000000000000 --- a/src/zstd/wip/work/entropy/zstd/compress/ldm.rs +++ /dev/null @@ -1 +0,0 @@ -//! `zstd_ldm.c`: long distance matching. diff --git a/src/zstd/wip/work/entropy/zstd/compress/mod.rs b/src/zstd/wip/work/entropy/zstd/compress/mod.rs deleted file mode 100644 index 13ddecdf82e4..000000000000 --- a/src/zstd/wip/work/entropy/zstd/compress/mod.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! `lib/compress` - -pub(crate) mod double_fast; -pub(crate) mod fast; -pub(crate) mod fse; -pub(crate) mod hist; -pub(crate) mod huf; -pub(crate) mod internal; -pub(crate) mod lazy; -pub(crate) mod ldm; -pub(crate) mod opt; -pub(crate) mod presplit; - -use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; - -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; - -/// `ZSTD_rowMatchFinderSupported` -pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { - strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 -} - -/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to -/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies -/// that support it look at. Returns how many bytes at the end of the block are in no sequence. -pub(crate) fn compress_block( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row_match_finder: bool, - dict_mode: DictMode, -) -> Result { - match strategy { - Strategy::Fast => match dict_mode { - DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), - DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), - DictMode::DictMatchState => { - fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) - } - }, - Strategy::DFast => match dict_mode { - DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), - DictMode::ExtDict => { - double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) - } - }, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { - lazy::compress_block_lazy( - ms, - seq_store, - rep, - src, - strategy, - use_row_match_finder && row_match_finder_supported(strategy), - dict_mode, - ) - } - Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { - opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) - } - } -} diff --git a/src/zstd/wip/work/entropy/zstd/compress/opt.rs b/src/zstd/wip/work/entropy/zstd/compress/opt.rs deleted file mode 100644 index d055a44cf982..000000000000 --- a/src/zstd/wip/work/entropy/zstd/compress/opt.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. - -use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. -/// Matches are not compared past the position `iend`. -pub(crate) fn update_tree( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _ip: usize, - _iend: usize, -) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` -/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. -pub(crate) fn compress_block_opt( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, - _strategy: Strategy, - _dict_mode: DictMode, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/entropy/zstd/compress/presplit.rs b/src/zstd/wip/work/entropy/zstd/compress/presplit.rs deleted file mode 100644 index 8b778e2989f9..000000000000 --- a/src/zstd/wip/work/entropy/zstd/compress/presplit.rs +++ /dev/null @@ -1 +0,0 @@ -//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. diff --git a/src/zstd/wip/work/fast/STATUS.md b/src/zstd/wip/work/fast/STATUS.md new file mode 100644 index 000000000000..a02c065a8a2d --- /dev/null +++ b/src/zstd/wip/work/fast/STATUS.md @@ -0,0 +1,35 @@ +# fast / dfast port: status (12:27 UTC) + +## Deliverable +- `/root/zstd-ref/work/fast/zstd/compress/fast.rs` +- `/root/zstd-ref/work/fast/zstd/compress/double_fast.rs` + +Complete, `cargo fmt --check` clean, no clippy finding in either file. `double_fast.rs` uses four `pub(super)` helpers +of `fast.rs` (`FAST_HASH_FILL_STEP`, `Located`, `count_backwards`, `restore_rep`): take both files together. + +## IMPORTANT: the copy of double_fast.rs in the real repo is a deliberately broken mutant +The copy taken into `/root/repos/oven-sh/bun/src/zstd/compress/double_fast.rs` was taken while my mutation test was +rewriting the file in place. It has `match_index_s >= prefix_lowest_index` (line 423) where the right code, as in C, is +`match_index_s > prefix_lowest_index`. That is the dfast + attached CDict mismatch of the end-to-end test. Copy both +files again from the paths above. (`fast.rs` in the repo only differs by `let buf = src.buf;`, see below.) + +## Verified against C (all pass) +Harness: `cd /root/zstd-ref/work/fast/harness && RUSTFLAGS="--cap-lints=warn" cargo build --release` +(NOT `--profile bench`, which writes the same `target/release` without overflow checks). +- `zstd_harness seqs 1,2 <0|1>`: template test, fixed (several blocks, param ids, rep[2], short last + block). 330,000 iterations for each of dict 0 and 1 on the first version, and 320,000 each on the final files. +- `zstd_harness stream `: mine (`c/shim.c`, `src/stream.rs`, `TestMs` in `zstd/testing.rs`). Calls the C + functions directly on hand-made `ZSTD_MatchState_t`: all 6 block compressors and both fill functions with all 4 + (purpose, load method) pairs; attached dictionaries (dictMatchState), loaded dictionaries (contiguous or not), chunks + that are contiguous / elsewhere / overwrite the same buffer (ring buffer), blocks of any size from 1 byte, start + indices up to 3.6e9, repeat offsets at the limits. After every call compares sequences, literals, rep, return value, + long length and both tables. 220,000+ iterations on the final files (about 80,000 dictMatchState blocks per strategy). +- `zstd_harness frames 1,2 <0|1|2>`: copy of the harness of work/lazy (real CDict machinery). + 100,000 iterations of mode 2, 40,000 of mode 1. +- Line coverage of the two files by `stream`: everything but the 4 `end < 8` early returns of the fill functions. +- 45 hand-made mutants (boundary conditions in every function): all caught by `stream`. + +## Open +- Speed (bench profile, 23 MB of Rust source, `zstd_harness bench `): Rust takes 1.4x (level 1), 1.27x + (level 2), 1.2x (levels 3, 4) the time of C; dictMatchState 1.15-1.35x. See the hand-back for where the time goes. +- Nothing open for correctness. diff --git a/src/zstd/wip/work/fast/zstd/compress/double_fast.rs b/src/zstd/wip/work/fast/zstd/compress/double_fast.rs deleted file mode 100644 index 741c372c3cb6..000000000000 --- a/src/zstd/wip/work/fast/zstd/compress/double_fast.rs +++ /dev/null @@ -1,785 +0,0 @@ -//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. - -use super::fast::{FAST_HASH_FILL_STEP, Located, count_backwards, restore_rep}; -use super::internal::{ - BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, - compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, - write_tagged_index, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, read_le32, read_le64}; - -/// `ZSTD_fillDoubleHashTableForCDict` -fn fill_double_hash_table_for_cdict( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits_l = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let h_bits_s = ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; - let mls = ms.c_params.min_match; - let hash_large = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - // The positions in between only go in the table of long hashes, where there is nothing yet. - while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { - let curr = prefix.index_of(ip); - for i in 0..FAST_HASH_FILL_STEP { - let sm_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; - let lg_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; - if i == 0 { - write_tagged_index(hash_small, sm_hash_and_tag, curr + i)?; - } - if i == 0 || hash_large.at(lg_hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { - write_tagged_index(hash_large, lg_hash_and_tag, curr + i)?; - } - if dtlm == DictTableLoadMethod::Fast { - break; - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillDoubleHashTableForCCtx` -fn fill_double_hash_table_for_cctx( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let mls = ms.c_params.min_match; - let hash_large = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - // The positions in between only go in the table of long hashes, where there is nothing yet. - while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { - let curr = prefix.index_of(ip); - for i in 0..FAST_HASH_FILL_STEP { - let sm_hash = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; - let lg_hash = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; - if i == 0 { - hash_small.put(sm_hash, curr + i)?; - } - if i == 0 || hash_large.at(lg_hash)? == 0 { - hash_large.put(lg_hash, curr + i)?; - } - if dtlm == DictTableLoadMethod::Fast { - break; - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_double_hash_table( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, - tfp: TableFillPurpose, -) -> Result<()> { - match tfp { - TableFillPurpose::ForCDict => fill_double_hash_table_for_cdict(ms, prefix, end, dtlm), - TableFillPurpose::ForCCtx => fill_double_hash_table_for_cctx(ms, prefix, end, dtlm), - } -} - -/// `ZSTD_compressBlock_doubleFast_noDict_generic` -fn compress_block_double_fast_no_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; - - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let window_log = ms.c_params.window_log; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, window_log); - let prefix_lowest = src.pos_of(prefix_lowest_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip = istart + usize::from(istart == prefix_lowest); - let [mut offset_1, mut offset_2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let current = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(current, window_log); - let max_rep = current.wrapping_sub(window_low); - if offset_2 > max_rep { - offset_saved2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved1 = offset_1; - offset_1 = 0; - } - } - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - - // One turn per match. - 'start: loop { - let mut step = 1usize; - let mut next_step = ip + K_STEP_INCR; - let mut ip1 = ip + step; - if ip1 > ilimit { - break 'start; - } - - let mut hl0 = hash_ptr(buf, ip, h_bits_l, 8)?; - let mut idxl0 = hash_long.at(hl0)?; - let mut curr; - - let m_length = 'match_stored: { - // One turn per position. Ends with the offset and the length of a match, which `ip` is moved to the start - // of. - let (offset, m_length, hl1) = loop { - let hs0 = hash_ptr(buf, ip, h_bits_s, MLS)?; - let idxs0 = hash_small.at(hs0)?; - curr = src.index_of(ip); - hash_long.put(hl0, curr)?; - hash_small.put(hs0, curr)?; - - if (offset_1 > 0) - & (read_le32(buf, (ip + 1).wrapping_sub(offset_1 as usize))? - == read_le32(buf, ip + 1)?) - { - let m_length = count( - buf, - ip + 1 + 4, - buf, - (ip + 1 + 4).wrapping_sub(offset_1 as usize), - iend, - )? + 4; - ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - break 'match_stored m_length; - } - - let hl1 = hash_ptr(buf, ip1, h_bits_l, 8)?; - - let matchl0 = src.pos_of(idxl0); - if idxl0 >= prefix_lowest_index && read_le64(buf, matchl0)? == read_le64(buf, ip)? { - let m_length = count(buf, ip + 8, buf, matchl0 + 8, iend)? + 8; - let offset = ip.wrapping_sub(matchl0) as u32; - let back = count_backwards(buf, ip, anchor, buf, matchl0, prefix_lowest); - ip -= back; - break (offset, m_length + back, hl1); - } - - let idxl1 = hash_long.at(hl1)?; - - let mut matchs0 = src.pos_of(idxs0); - if idxs0 >= prefix_lowest_index && read_le32(buf, matchs0)? == read_le32(buf, ip)? { - // A short match: a longer one at the next position is preferred. - let mut m_length = count(buf, ip + 4, buf, matchs0 + 4, iend)? + 4; - let mut offset = ip.wrapping_sub(matchs0) as u32; - let matchl1 = src.pos_of(idxl1); - if idxl1 > prefix_lowest_index - && read_le64(buf, matchl1)? == read_le64(buf, ip1)? - { - let l1len = count(buf, ip1 + 8, buf, matchl1 + 8, iend)? + 8; - if l1len > m_length { - ip = ip1; - m_length = l1len; - offset = ip.wrapping_sub(matchl1) as u32; - matchs0 = matchl1; - } - } - let back = count_backwards(buf, ip, anchor, buf, matchs0, prefix_lowest); - ip -= back; - break (offset, m_length + back, hl1); - } - - if ip1 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - ip = ip1; - ip1 += step; - - hl0 = hl1; - idxl0 = idxl1; - if ip1 > ilimit { - break 'start; - } - }; - - offset_2 = offset_1; - offset_1 = offset; - // With a larger step, ip1 may be past the end of the match. - if step < 4 { - hash_long.put(hl1, src.index_of(ip1))?; - } - seq_store.store_seq( - ip - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - m_length - }; - - ip += m_length; - anchor = ip; - - if ip <= ilimit { - let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; - - while ip <= ilimit - && (offset_2 > 0) - & (read_le32(buf, ip)? == read_le32(buf, ip.wrapping_sub(offset_2 as usize))?) - { - let r_length = count( - buf, - ip + 4, - buf, - (ip + 4).wrapping_sub(offset_2 as usize), - iend, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, src.index_of(ip))?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, src.index_of(ip))?; - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; - ip += r_length; - anchor = ip; - } - } - } - - restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` -fn compress_block_double_fast_dict_match_state_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, ms.c_params.window_log); - let prefix_lowest = src.pos_of(prefix_lowest_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - let [mut offset_1, mut offset_2, _] = *rep; - - let dms = src.dms.as_ref().ok_or(BUG)?; - let dict_hash_long = dms.ms.hash_table.as_slice(); - let dict_hash_small = dms.ms.chain_table.as_slice(); - let dict_start_index = dms.ms.window.dict_limit; - let dict_start = dms.pos_of(dict_start_index); - let dict_end = dms.end; - // What to subtract from an index below prefix_lowest_index to get the index in the dictionary. - let dict_index_delta = prefix_lowest_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); - let dict_h_bits_l = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let dict_h_bits_s = dms.ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; - let dict_and_prefix_length = istart - .wrapping_sub(prefix_lowest) - .wrapping_add(dict_end.wrapping_sub(dict_start)) as u32; - - let locate = |index: u32| { - if index < prefix_lowest_index { - Located { - buf: dms.buf, - pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_lowest, - end: iend, - } - } - }; - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - - let mut anchor = istart; - let mut ip = istart + usize::from(dict_and_prefix_length == 0); - - 'search: while ip < ilimit { - let h2 = hash_ptr(buf, ip, h_bits_l, 8)?; - let h = hash_ptr(buf, ip, h_bits_s, MLS)?; - let dict_hash_and_tag_l = hash_ptr(buf, ip, dict_h_bits_l, 8)?; - let dict_hash_and_tag_s = hash_ptr(buf, ip, dict_h_bits_s, MLS)?; - let dict_match_index_and_tag_l = - dict_hash_long.at(dict_hash_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_match_index_and_tag_s = - dict_hash_small.at(dict_hash_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_tags_match_l = - compare_packed_tags(dict_match_index_and_tag_l as usize, dict_hash_and_tag_l); - let dict_tags_match_s = - compare_packed_tags(dict_match_index_and_tag_s as usize, dict_hash_and_tag_s); - let curr = src.index_of(ip); - let match_index_l = hash_long.at(h2)?; - let mut match_index_s = hash_small.at(h)?; - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); - hash_long.put(h2, curr)?; - hash_small.put(h, curr)?; - - let m_length = 'match_stored: { - if index_overlap_check(prefix_lowest_index, rep_index) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? - { - let m_length = count_2segments( - buf, - ip + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - iend, - rep_match.end, - prefix_lowest, - )? + 4; - ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - break 'match_stored m_length; - } - - // The offset and the length of a match, which `ip` is moved to the start of. - let (offset, m_length) = 'match_found: { - let match_long = src.pos_of(match_index_l); - if match_index_l >= prefix_lowest_index - && read_le64(buf, match_long)? == read_le64(buf, ip)? - { - let m_length = count(buf, ip + 8, buf, match_long + 8, iend)? + 8; - let offset = ip.wrapping_sub(match_long) as u32; - let back = count_backwards(buf, ip, anchor, buf, match_long, prefix_lowest); - ip -= back; - break 'match_found (offset, m_length + back); - } else if dict_tags_match_l { - let dict_match_index_l = - dict_match_index_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match_l = dms.pos_of(dict_match_index_l); - if dict_match_index_l > dict_start_index - && read_le64(dms.buf, dict_match_l)? == read_le64(buf, ip)? - { - let m_length = count_2segments( - buf, - ip + 8, - dms.buf, - dict_match_l + 8, - iend, - dict_end, - prefix_lowest, - )? + 8; - let offset = curr - .wrapping_sub(dict_match_index_l) - .wrapping_sub(dict_index_delta); - let back = - count_backwards(buf, ip, anchor, dms.buf, dict_match_l, dict_start); - ip -= back; - break 'match_found (offset, m_length + back); - } - } - - let short_found = if match_index_s > prefix_lowest_index { - read_le32(buf, src.pos_of(match_index_s))? == read_le32(buf, ip)? - } else if dict_tags_match_s { - let dict_match_index_s = - dict_match_index_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS; - match_index_s = dict_match_index_s.wrapping_add(dict_index_delta); - dict_match_index_s > dict_start_index - && read_le32(dms.buf, dms.pos_of(dict_match_index_s))? - == read_le32(buf, ip)? - } else { - false - }; - if !short_found { - ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; - continue 'search; - } - - // A short match: a long one at the next position is preferred. - let hl3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; - let dict_hash_and_tag_l3 = hash_ptr(buf, ip + 1, dict_h_bits_l, 8)?; - let match_index_l3 = hash_long.at(hl3)?; - let dict_match_index_and_tag_l3 = - dict_hash_long.at(dict_hash_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_tags_match_l3 = - compare_packed_tags(dict_match_index_and_tag_l3 as usize, dict_hash_and_tag_l3); - let match_l3 = src.pos_of(match_index_l3); - hash_long.put(hl3, curr + 1)?; - - if match_index_l3 >= prefix_lowest_index - && read_le64(buf, match_l3)? == read_le64(buf, ip + 1)? - { - let m_length = count(buf, ip + 9, buf, match_l3 + 8, iend)? + 8; - ip += 1; - let offset = ip.wrapping_sub(match_l3) as u32; - let back = count_backwards(buf, ip, anchor, buf, match_l3, prefix_lowest); - ip -= back; - break 'match_found (offset, m_length + back); - } else if dict_tags_match_l3 { - let dict_match_index_l3 = - dict_match_index_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match_l3 = dms.pos_of(dict_match_index_l3); - if dict_match_index_l3 > dict_start_index - && read_le64(dms.buf, dict_match_l3)? == read_le64(buf, ip + 1)? - { - let m_length = count_2segments( - buf, - ip + 1 + 8, - dms.buf, - dict_match_l3 + 8, - iend, - dict_end, - prefix_lowest, - )? + 8; - ip += 1; - let offset = curr - .wrapping_add(1) - .wrapping_sub(dict_match_index_l3) - .wrapping_sub(dict_index_delta); - let back = - count_backwards(buf, ip, anchor, dms.buf, dict_match_l3, dict_start); - ip -= back; - break 'match_found (offset, m_length + back); - } - } - - let match_s = locate(match_index_s); - let m_length = count_2segments( - buf, - ip + 4, - match_s.buf, - match_s.pos + 4, - iend, - match_s.end, - prefix_lowest, - )? + 4; - let offset = curr.wrapping_sub(match_index_s); - let back = - count_backwards(buf, ip, anchor, match_s.buf, match_s.pos, match_s.start); - ip -= back; - (offset, m_length + back) - }; - - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - m_length - }; - - ip += m_length; - anchor = ip; - - if ip <= ilimit { - let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; - - while ip <= ilimit { - let current2 = src.index_of(ip); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !(index_overlap_check(prefix_lowest_index, rep_index2) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_lowest, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; - ip += rep_length2; - anchor = ip; - } - } - } - - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_doubleFast` -pub(crate) fn compress_block_double_fast( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_double_fast_no_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_no_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_no_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_no_dict_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_doubleFast_dictMatchState` -pub(crate) fn compress_block_double_fast_dict_match_state( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_double_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_doubleFast_extDict_generic` -fn compress_block_double_fast_ext_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); - let dict_start_index = low_limit; - let dict_limit = ms.window.dict_limit; - let prefix_start_index = dict_limit.max(low_limit); - let prefix_start = src.pos_of(prefix_start_index); - let dict_start = src.dict_pos_of(dict_start_index); - let dict_end = src.dict_pos_of(prefix_start_index); - let [mut offset_1, mut offset_2, _] = *rep; - - // Nothing of the older segment is within reach. - if prefix_start_index == dict_start_index { - return compress_block_double_fast(ms, seq_store, rep, src); - } - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: src.dict_buf, - pos: src.dict_pos_of(index), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - - let mut anchor = istart; - let mut ip = istart; - - while ip < ilimit { - let h_small = hash_ptr(buf, ip, h_bits_s, MLS)?; - let match_index = hash_small.at(h_small)?; - let match_s = locate(match_index); - - let h_long = hash_ptr(buf, ip, h_bits_l, 8)?; - let match_long_index = hash_long.at(h_long)?; - let match_long = locate(match_long_index); - - let curr = src.index_of(ip); - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); - hash_long.put(h_long, curr)?; - hash_small.put(h_small, curr)?; - - let m_length; - if (index_overlap_check(prefix_start_index, rep_index) - & (offset_1 <= curr.wrapping_add(1).wrapping_sub(dict_start_index))) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? - { - m_length = count_2segments( - buf, - ip + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - iend, - rep_match.end, - prefix_start, - )? + 4; - ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - } else { - // The match, how much of it is known, and its offset. `ip` is moved to where it is for. - let (found, known, offset) = if match_long_index > dict_start_index - && read_le64(match_long.buf, match_long.pos)? == read_le64(buf, ip)? - { - (match_long, 8, curr.wrapping_sub(match_long_index)) - } else if match_index > dict_start_index - && read_le32(match_s.buf, match_s.pos)? == read_le32(buf, ip)? - { - // A short match: a long one at the next position is preferred. - let h3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; - let match_index3 = hash_long.at(h3)?; - let match3 = locate(match_index3); - hash_long.put(h3, curr + 1)?; - if match_index3 > dict_start_index - && read_le64(match3.buf, match3.pos)? == read_le64(buf, ip + 1)? - { - ip += 1; - (match3, 8, curr.wrapping_add(1).wrapping_sub(match_index3)) - } else { - (match_s, 4, curr.wrapping_sub(match_index)) - } - } else { - ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; - continue; - }; - let forwards = count_2segments( - buf, - ip + known, - found.buf, - found.pos + known, - iend, - found.end, - prefix_start, - )? + known; - let back = count_backwards(buf, ip, anchor, found.buf, found.pos, found.start); - ip -= back; - m_length = forwards + back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - } - - ip += m_length; - anchor = ip; - - if ip <= ilimit { - let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; - - while ip <= ilimit { - let current2 = src.index_of(ip); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !((index_overlap_check(prefix_start_index, rep_index2) - & (offset_2 <= current2.wrapping_sub(dict_start_index))) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_start, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; - ip += rep_length2; - anchor = ip; - } - } - } - - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_doubleFast_extDict` -pub(crate) fn compress_block_double_fast_ext_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_double_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), - } -} diff --git a/src/zstd/wip/work/fast/zstd/compress/fast.rs b/src/zstd/wip/work/fast/zstd/compress/fast.rs deleted file mode 100644 index f543cad9ae16..000000000000 --- a/src/zstd/wip/work/fast/zstd/compress/fast.rs +++ /dev/null @@ -1,829 +0,0 @@ -//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. - -use super::internal::{ - BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, - compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, - write_tagged_index, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, read_le32}; - -pub(super) const FAST_HASH_FILL_STEP: u32 = 3; - -/// Where an index leads, when it can be in one of two segments. -#[derive(Clone, Copy)] -pub(super) struct Located<'a> { - /// The buffer that the segment is in. - pub(super) buf: &'a [u8], - /// The position of the index in `buf`. - pub(super) pos: usize, - /// Where the segment starts in `buf`. - pub(super) start: usize, - /// Where it ends. - pub(super) end: usize, -} - -/// How many bytes before `ip`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. -#[inline(always)] -pub(super) fn count_backwards( - buf: &[u8], - ip: usize, - anchor: usize, - match_buf: &[u8], - match_pos: usize, - match_low: usize, -) -> usize { - let before_ip = buf.get(anchor..ip).unwrap_or_default(); - let before_match = match_buf.get(match_low..match_pos).unwrap_or_default(); - before_ip - .iter() - .rev() - .zip(before_match.iter().rev()) - .take_while(|(a, b)| a == b) - .count() -} - -/// The repeat offsets to leave at the end of a block, at the start of which those that reached too far were saved and -/// set to 0. -#[inline(always)] -pub(super) fn restore_rep( - rep: &mut [u32; ZSTD_REP_NUM], - offset_1: u32, - offset_2: u32, - offset_saved1: u32, - offset_saved2: u32, -) { - // If offset_1 started invalid and became valid, offset_saved1 is what rotates into the second place. - let offset_saved2 = if offset_saved1 != 0 && offset_1 != 0 { - offset_saved1 - } else { - offset_saved2 - }; - let [rep0, rep1, _] = rep; - *rep0 = if offset_1 != 0 { - offset_1 - } else { - offset_saved1 - }; - *rep1 = if offset_2 != 0 { - offset_2 - } else { - offset_saved2 - }; -} - -/// `ZSTD_fillHashTableForCDict` -fn fill_hash_table_for_cdict( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let mls = ms.c_params.min_match; - let hash_table = ms.hash_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { - let curr = prefix.index_of(ip); - write_tagged_index(hash_table, hash_ptr(buf, ip, h_bits, mls)?, curr)?; - if dtlm == DictTableLoadMethod::Full { - // The positions in between only go where there is nothing yet. - for p in 1..FAST_HASH_FILL_STEP { - let hash_and_tag = hash_ptr(buf, ip + p as usize, h_bits, mls)?; - if hash_table.at(hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { - write_tagged_index(hash_table, hash_and_tag, curr + p)?; - } - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillHashTableForCCtx` -fn fill_hash_table_for_cctx( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits = ms.c_params.hash_log; - let mls = ms.c_params.min_match; - let hash_table = ms.hash_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { - let curr = prefix.index_of(ip); - hash_table.put(hash_ptr(buf, ip, h_bits, mls)?, curr)?; - if dtlm == DictTableLoadMethod::Full { - // The positions in between only go where there is nothing yet. - for p in 1..FAST_HASH_FILL_STEP { - let hash = hash_ptr(buf, ip + p as usize, h_bits, mls)?; - if hash_table.at(hash)? == 0 { - hash_table.put(hash, curr + p)?; - } - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_hash_table( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, - tfp: TableFillPurpose, -) -> Result<()> { - match tfp { - TableFillPurpose::ForCDict => fill_hash_table_for_cdict(ms, prefix, end, dtlm), - TableFillPurpose::ForCCtx => fill_hash_table_for_cctx(ms, prefix, end, dtlm), - } -} - -/// `ZSTD_match4Found_branch`, and `ZSTD_match4Found_cmov`, which only differs in how it avoids reading at an index that -/// is not valid. -#[inline(always)] -fn match4_found( - buf: &[u8], - current: usize, - match_pos: usize, - match_idx: u32, - idx_low_limit: u32, -) -> Result { - Ok(match_idx >= idx_low_limit && read_le32(buf, current)? == read_le32(buf, match_pos)?) -} - -/// `ZSTD_compressBlock_fast_noDict_generic` -/// -/// Each turn of the inner loop looks at two positions, `ip0` and `ip1`, for a match, and at `ip2` for a repeat. The -/// hash of a position is computed, and the table is read, one position ahead of the comparison. -fn compress_block_fast_no_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); - - let hlog = ms.c_params.hash_log; - let window_log = ms.c_params.window_log; - let target_length = ms.c_params.target_length; - // The least step is 2, as two positions are looked at per turn. - let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_start_index = ms.get_lowest_prefix_index(end_index, window_log); - let prefix_start = src.pos_of(prefix_start_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip0 = istart + usize::from(istart == prefix_start); - let [mut rep_offset1, mut rep_offset2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let curr = src.index_of(ip0); - let window_low = ms.get_lowest_prefix_index(curr, window_log); - let max_rep = curr.wrapping_sub(window_low); - if rep_offset2 > max_rep { - offset_saved2 = rep_offset2; - rep_offset2 = 0; - } - if rep_offset1 > max_rep { - offset_saved1 = rep_offset1; - rep_offset1 = 0; - } - } - let hash_table = ms.hash_table.as_mut_slice(); - - 'start: loop { - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - - let mut ip1 = ip0 + 1; - let mut ip2 = ip0 + step; - let mut ip3 = ip2 + 1; - if ip3 >= ilimit { - break 'start; - } - - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; - let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; - let mut match_idx = hash_table.at(hash0)?; - let mut current0; - - // Ends with the start and the length so far of a repeat match, or with `None` for a match at `match_idx`. - let rep_match = loop { - let rval = read_le32(buf, ip2.wrapping_sub(rep_offset1 as usize))?; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if (read_le32(buf, ip2)? == rval) & (rep_offset1 > 0) { - ip0 = ip2; - let match0 = ip0.wrapping_sub(rep_offset1 as usize); - let back = usize::from(buf.at(ip0 - 1)? == buf.at(match0.wrapping_sub(1))?); - ip0 -= back; - // The hash of ip1 is there, and ip1 is before the end of the match. - hash_table.put(hash1, src.index_of(ip1))?; - break Some((match0.wrapping_sub(back), 4 + back)); - } - - if match4_found( - buf, - ip0, - src.pos_of(match_idx), - match_idx, - prefix_start_index, - )? { - hash_table.put(hash1, src.index_of(ip1))?; - break None; - } - - match_idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip3; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if match4_found( - buf, - ip0, - src.pos_of(match_idx), - match_idx, - prefix_start_index, - )? { - // ip1 is certain to be before the end of the match only if the step is small. - if step <= 4 { - hash_table.put(hash1, src.index_of(ip1))?; - } - break None; - } - - match_idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip0 + step; - ip3 = ip1 + step; - - if ip2 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - if ip3 >= ilimit { - break 'start; - } - }; - - let (offcode, match0, mut m_length) = match rep_match { - Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), - None => { - let match0 = src.pos_of(match_idx); - rep_offset2 = rep_offset1; - rep_offset1 = ip0.wrapping_sub(match0) as u32; - let back = count_backwards(buf, ip0, anchor, buf, match0, prefix_start); - ip0 -= back; - (offset_to_offbase(rep_offset1), match0 - back, 4 + back) - } - }; - - m_length += count(buf, ip0 + m_length, buf, match0 + m_length, iend)?; - seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; - ip0 += m_length; - anchor = ip0; - - if ip0 <= ilimit { - hash_table.put( - hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, - current0 + 2, - )?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; - - if rep_offset2 > 0 { - while ip0 <= ilimit - && read_le32(buf, ip0)? - == read_le32(buf, ip0.wrapping_sub(rep_offset2 as usize))? - { - let r_length = count( - buf, - ip0 + 4, - buf, - (ip0 + 4).wrapping_sub(rep_offset2 as usize), - iend, - )? + 4; - core::mem::swap(&mut rep_offset1, &mut rep_offset2); - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; - ip0 += r_length; - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; - anchor = ip0; - } - } - } - } - - restore_rep(rep, rep_offset1, rep_offset2, offset_saved1, offset_saved2); - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_fast` -pub(crate) fn compress_block_fast( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_fast_no_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_no_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_no_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_no_dict_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_fast_dictMatchState_generic` -fn compress_block_fast_dict_match_state_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; - - let hlog = ms.c_params.hash_log; - let target_length = ms.c_params.target_length; - let step_size = (target_length + u32::from(target_length == 0)) as usize; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let prefix_start_index = ms.window.dict_limit; - let prefix_start = src.pos_of(prefix_start_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - let [mut offset_1, mut offset_2, _] = *rep; - - let dms = src.dms.as_ref().ok_or(BUG)?; - let dict_hash_table = dms.ms.hash_table.as_slice(); - let dict_start_index = dms.ms.window.dict_limit; - let dict_start = dms.pos_of(dict_start_index); - let dict_end = dms.end; - // What to subtract from an index below prefix_start_index to get the index in the dictionary. - let dict_index_delta = prefix_start_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); - let dict_and_prefix_length = istart - .wrapping_sub(prefix_start) - .wrapping_add(dict_end) - .wrapping_sub(dict_start) as u32; - let dict_h_bits = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: dms.buf, - pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; - let hash_table = ms.hash_table.as_mut_slice(); - - let mut anchor = istart; - let mut ip0 = istart + usize::from(dict_and_prefix_length == 0); - let mut ip1 = istart + step_size; - - 'search: while ip1 <= ilimit { - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; - - let dict_hash_and_tag0 = hash_ptr(buf, ip0, dict_h_bits, MLS)?; - let mut dict_match_index_and_tag = - dict_hash_table.at(dict_hash_and_tag0 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let mut dict_tags_match = - compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag0); - - let mut match_index = hash_table.at(hash0)?; - let mut curr = src.index_of(ip0); - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - - let m_length = loop { - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); - let hash1 = hash_ptr(buf, ip1, hlog, MLS)?; - let dict_hash_and_tag1 = hash_ptr(buf, ip1, dict_h_bits, MLS)?; - hash_table.put(hash0, curr)?; - - if index_overlap_check(prefix_start_index, rep_index) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip0 + 1)? - { - let m_length = count_2segments( - buf, - ip0 + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - iend, - rep_match.end, - prefix_start, - )? + 4; - ip0 += 1; - seq_store.store_seq(ip0 - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - break m_length; - } - - if dict_tags_match { - let dict_match_index = dict_match_index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match = dms.pos_of(dict_match_index); - // A match in the dictionary is only taken if there is no candidate in the prefix. - if dict_match_index > dict_start_index - && read_le32(dms.buf, dict_match)? == read_le32(buf, ip0)? - && match_index <= prefix_start_index - { - let offset = curr - .wrapping_sub(dict_match_index) - .wrapping_sub(dict_index_delta); - let mut m_length = count_2segments( - buf, - ip0 + 4, - dms.buf, - dict_match + 4, - iend, - dict_end, - prefix_start, - )? + 4; - let back = count_backwards(buf, ip0, anchor, dms.buf, dict_match, dict_start); - ip0 -= back; - m_length += back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip0 - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - break m_length; - } - } - - let match_pos = src.pos_of(match_index); - if match4_found(buf, ip0, match_pos, match_index, prefix_start_index)? { - let offset = ip0.wrapping_sub(match_pos) as u32; - let mut m_length = count(buf, ip0 + 4, buf, match_pos + 4, iend)? + 4; - let back = count_backwards(buf, ip0, anchor, buf, match_pos, prefix_start); - ip0 -= back; - m_length += back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip0 - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - break m_length; - } - - dict_match_index_and_tag = - dict_hash_table.at(dict_hash_and_tag1 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - dict_tags_match = - compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag1); - match_index = hash_table.at(hash1)?; - - if ip1 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - ip0 = ip1; - ip1 += step; - if ip1 > ilimit { - break 'search; - } - - curr = src.index_of(ip0); - hash0 = hash1; - }; - - ip0 += m_length; - anchor = ip0; - - if ip0 <= ilimit { - hash_table.put(hash_ptr(buf, src.pos_of(curr + 2), hlog, MLS)?, curr + 2)?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; - - while ip0 <= ilimit { - let current2 = src.index_of(ip0); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !(index_overlap_check(prefix_start_index, rep_index2) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip0 + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_start, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, current2)?; - ip0 += rep_length2; - anchor = ip0; - } - } - - ip1 = ip0 + step_size; - } - - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_fast_dictMatchState` -pub(crate) fn compress_block_fast_dict_match_state( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_fast_extDict_generic` -fn compress_block_fast_ext_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); - - let hlog = ms.c_params.hash_log; - let target_length = ms.c_params.target_length; - let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); - let dict_start_index = low_limit; - let dict_start = src.dict_pos_of(dict_start_index); - let dict_limit = ms.window.dict_limit; - let prefix_start_index = dict_limit.max(low_limit); - let prefix_start = src.pos_of(prefix_start_index); - let dict_end = src.dict_pos_of(prefix_start_index); - - // Nothing of the older segment is within reach. - if prefix_start_index == dict_start_index { - return compress_block_fast(ms, seq_store, rep, src); - } - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip0 = istart; - let [mut offset_1, mut offset_2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let curr = src.index_of(ip0); - let max_rep = curr.wrapping_sub(dict_start_index); - if offset_2 >= max_rep { - offset_saved2 = offset_2; - offset_2 = 0; - } - if offset_1 >= max_rep { - offset_saved1 = offset_1; - offset_1 = 0; - } - } - - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: src.dict_buf, - pos: src.dict_pos_of(index), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; - // Whether there is a match for the position `ip` at `idx`, which is from the table. - let match_found = |ip: usize, idx: u32| -> Result { - let candidate = locate(idx); - Ok(idx >= dict_start_index - && read_le32(candidate.buf, candidate.pos)? == read_le32(buf, ip)?) - }; - let hash_table = ms.hash_table.as_mut_slice(); - - 'start: loop { - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - - let mut ip1 = ip0 + 1; - let mut ip2 = ip0 + step; - let mut ip3 = ip2 + 1; - if ip3 >= ilimit { - break 'start; - } - - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; - let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; - let mut idx = hash_table.at(hash0)?; - let mut current0; - - // Ends with the start and the length so far of a repeat match, or with `None` for a match at `idx`. - let rep_match = loop { - let current2 = src.index_of(ip2); - let rep_index = current2.wrapping_sub(offset_1); - let mut rep_match = locate(rep_index); - // The 4 bytes have to be in one segment, with a byte before them. - let rep_found = ((prefix_start_index.wrapping_sub(rep_index) >= 4) & (offset_1 > 0)) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip2)?; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if rep_found { - ip0 = ip2; - let back = usize::from( - buf.at(ip0 - 1)? == rep_match.buf.at(rep_match.pos.wrapping_sub(1))?, - ); - ip0 -= back; - rep_match.pos = rep_match.pos.wrapping_sub(back); - break Some((rep_match, 4 + back)); - } - - if match_found(ip0, idx)? { - break None; - } - - idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip3; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if match_found(ip0, idx)? { - break None; - } - - idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip0 + step; - ip3 = ip1 + step; - - if ip2 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - if ip3 >= ilimit { - break 'start; - } - }; - - let (offcode, match0, mut m_length) = match rep_match { - Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), - None => { - let offset = current0.wrapping_sub(idx); - let mut match0 = locate(idx); - offset_2 = offset_1; - offset_1 = offset; - let back = count_backwards(buf, ip0, anchor, match0.buf, match0.pos, match0.start); - ip0 -= back; - match0.pos -= back; - (offset_to_offbase(offset), match0, 4 + back) - } - }; - - m_length += count_2segments( - buf, - ip0 + m_length, - match0.buf, - match0.pos + m_length, - iend, - match0.end, - prefix_start, - )?; - seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; - ip0 += m_length; - anchor = ip0; - - if ip1 < ip0 { - hash_table.put(hash1, src.index_of(ip1))?; - } - - if ip0 <= ilimit { - hash_table.put( - hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, - current0 + 2, - )?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; - - while ip0 <= ilimit { - let rep_index2 = src.index_of(ip0).wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !((index_overlap_check(prefix_start_index, rep_index2) & (offset_2 > 0)) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip0 + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_start, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; - ip0 += rep_length2; - anchor = ip0; - } - } - } - - restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_fast_extDict` -pub(crate) fn compress_block_fast_ext_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), - } -} diff --git a/src/zstd/wip/work/fast/zstd/compress/fse.rs b/src/zstd/wip/work/fast/zstd/compress/fse.rs deleted file mode 100644 index 1b8f7cbf3173..000000000000 --- a/src/zstd/wip/work/fast/zstd/compress/fse.rs +++ /dev/null @@ -1,70 +0,0 @@ -//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. - -use crate::common::{MAX_SEQ, ML_FSE_LOG}; -use crate::util::SliceExt; - -/// `FSE_repeat` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum FseRepeat { - /// The table cannot be used again. - None, - /// It can, if it has a code for every symbol to encode. - Check, - /// It can. - Valid, -} - -/// `FSE_symbolCompressionTransform` -#[derive(Clone, Copy, Default)] -pub(crate) struct FseSymbolTransform { - pub(crate) delta_find_state: i32, - pub(crate) delta_nb_bits: u32, -} - -/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. -#[derive(Clone)] -pub(crate) struct FseCTable { - pub(crate) table_log: u32, - pub(crate) max_symbol_value: u32, - pub(crate) state_table: [u16; 1 << ML_FSE_LOG], - pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], -} - -impl FseCTable { - pub(crate) const fn new() -> Self { - Self { - table_log: 0, - max_symbol_value: 0, - state_table: [0; 1 << ML_FSE_LOG], - symbol_tt: [FseSymbolTransform { - delta_find_state: 0, - delta_nb_bits: 0, - }; MAX_SEQ as usize + 1], - } - } - - /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log - /// if the table has no code for it. - #[inline(always)] - pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { - let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); - transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 - } - - /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. - /// The table has a code for it. - #[inline(always)] - pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { - let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); - let min_nb_bits = transform.delta_nb_bits >> 16; - let threshold = (min_nb_bits + 1) << 16; - let table_size = 1u32 << self.table_log; - let delta_from_threshold = - threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); - // A linear interpolation, which is very approximate. - let normalized_delta_from_threshold = - (delta_from_threshold << accuracy_log) >> self.table_log; - let bit_multiplier = 1u32 << accuracy_log; - ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) - } -} diff --git a/src/zstd/wip/work/fast/zstd/compress/hist.rs b/src/zstd/wip/work/fast/zstd/compress/hist.rs deleted file mode 100644 index becf42516867..000000000000 --- a/src/zstd/wip/work/fast/zstd/compress/hist.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! `hist.c`: how many times each byte value occurs. - -use crate::error::Result; -use crate::util::SliceExt; - -/// `HIST_add`: `count` has room for every value in `src`. -pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { - for &byte in src { - *count.at_mut(usize::from(byte))? += 1; - } - Ok(()) -} - -/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. -/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. -pub(crate) fn count_simple( - count: &mut [u32], - max_symbol_value: &mut u32, - src: &[u8], -) -> Result { - let mut max_symbol = *max_symbol_value as usize; - count.sub_mut(..=max_symbol)?.fill(0); - if src.is_empty() { - *max_symbol_value = 0; - return Ok(0); - } - add(count, src)?; - while count.at(max_symbol)? == 0 { - max_symbol -= 1; - } - *max_symbol_value = max_symbol as u32; - Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) -} diff --git a/src/zstd/wip/work/fast/zstd/compress/huf.rs b/src/zstd/wip/work/fast/zstd/compress/huf.rs deleted file mode 100644 index 78373a89ebf0..000000000000 --- a/src/zstd/wip/work/fast/zstd/compress/huf.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! `huf_compress.c`: Huffman coding of literals. - -use crate::common::HUF_SYMBOLVALUE_MAX; -use crate::util::SliceExt; - -/// `HUF_repeat` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum HufRepeat { - /// The table cannot be used again. - None, - /// It can, if it has a code for every symbol to encode. - Check, - /// It can. - Valid, -} - -/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the -/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in -/// its highest bits. -#[derive(Clone)] -pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); - -impl HufCTable { - pub(crate) const fn new() -> Self { - Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) - } - - /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. - #[inline] - pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { - let [header, elements @ ..] = &self.0; - let max_symbol_value = (header >> 8) as u8; - if symbol_value > u32::from(max_symbol_value) { - return 0; - } - (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 - } -} diff --git a/src/zstd/wip/work/fast/zstd/compress/internal.rs b/src/zstd/wip/work/fast/zstd/compress/internal.rs deleted file mode 100644 index 51157ea6c1ce..000000000000 --- a/src/zstd/wip/work/fast/zstd/compress/internal.rs +++ /dev/null @@ -1,1212 +0,0 @@ -//! `zstd_compress_internal.h`: what the parts of the compressor share. - -use super::fse::{FseCTable, FseRepeat}; -use super::huf::{HufCTable, HufRepeat}; -use crate::common::{ - BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, - ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, -}; -use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; - -pub(crate) const K_SEARCH_STRENGTH: u32 = 8; -pub(crate) const HASH_READ_SIZE: usize = 8; -/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. -pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; - -pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; -pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; - -pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; -pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; - -/// `ZSTD_strategy` -#[repr(u8)] -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] -pub(crate) enum Strategy { - Fast = 1, - DFast = 2, - Greedy = 3, - Lazy = 4, - Lazy2 = 5, - BtLazy2 = 6, - BtOpt = 7, - BtUltra = 8, - BtUltra2 = 9, -} - -impl Strategy { - pub(crate) const fn from_int(value: i32) -> Option { - Some(match value { - 1 => Self::Fast, - 2 => Self::DFast, - 3 => Self::Greedy, - 4 => Self::Lazy, - 5 => Self::Lazy2, - 6 => Self::BtLazy2, - 7 => Self::BtOpt, - 8 => Self::BtUltra, - 9 => Self::BtUltra2, - _ => return None, - }) - } -} - -/// `ZSTD_ParamSwitch_e` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum ParamSwitch { - Auto = 0, - Enable = 1, - Disable = 2, -} - -/// `ZSTD_compressionParameters` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) struct CompressionParameters { - pub(crate) window_log: u32, - pub(crate) chain_log: u32, - pub(crate) hash_log: u32, - pub(crate) search_log: u32, - pub(crate) min_match: u32, - pub(crate) target_length: u32, - /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. - pub(crate) strategy: Option, -} - -impl CompressionParameters { - /// The strategy of parameters that are ready to be applied. - #[inline(always)] - pub(crate) fn strat(&self) -> Strategy { - self.strategy.unwrap_or(Strategy::Fast) - } -} - -/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum DictMode { - NoDict, - ExtDict, - DictMatchState, -} - -/// `ZSTD_dictTableLoadMethod_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum DictTableLoadMethod { - Fast, - Full, -} - -/// `ZSTD_tableFillPurpose_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum TableFillPurpose { - ForCCtx, - ForCDict, -} - -// ── entropy tables ──────────────────────────────────────────────────────── - -/// `ZSTD_hufCTables_t` -#[derive(Clone)] -pub(crate) struct HufCTables { - pub(crate) c_table: HufCTable, - pub(crate) repeat_mode: HufRepeat, -} - -/// `ZSTD_fseCTables_t` -#[derive(Clone)] -pub(crate) struct FseCTables { - pub(crate) offcode_c_table: FseCTable, - pub(crate) matchlength_c_table: FseCTable, - pub(crate) litlength_c_table: FseCTable, - pub(crate) offcode_repeat_mode: FseRepeat, - pub(crate) matchlength_repeat_mode: FseRepeat, - pub(crate) litlength_repeat_mode: FseRepeat, -} - -/// `ZSTD_entropyCTables_t` -#[derive(Clone)] -pub(crate) struct EntropyCTables { - pub(crate) huf: HufCTables, - pub(crate) fse: FseCTables, -} - -/// `ZSTD_compressedBlockState_t` -#[derive(Clone)] -pub(crate) struct CompressedBlockState { - pub(crate) entropy: EntropyCTables, - pub(crate) rep: [u32; ZSTD_REP_NUM], -} - -impl CompressedBlockState { - /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. - pub(crate) fn new() -> Self { - Self { - entropy: EntropyCTables { - huf: HufCTables { - c_table: HufCTable::new(), - repeat_mode: HufRepeat::None, - }, - fse: FseCTables { - offcode_c_table: FseCTable::new(), - matchlength_c_table: FseCTable::new(), - litlength_c_table: FseCTable::new(), - offcode_repeat_mode: FseRepeat::None, - matchlength_repeat_mode: FseRepeat::None, - litlength_repeat_mode: FseRepeat::None, - }, - }, - rep: REP_START_VALUE, - } - } - - /// `ZSTD_reset_compressedBlockState` - pub(crate) fn reset(&mut self) { - self.rep = REP_START_VALUE; - self.entropy.huf.repeat_mode = HufRepeat::None; - self.entropy.fse.offcode_repeat_mode = FseRepeat::None; - self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; - self.entropy.fse.litlength_repeat_mode = FseRepeat::None; - } -} - -/// `ZSTD_hufCTablesMetadata_t` -pub(crate) struct HufCTablesMetadata { - pub(crate) h_type: SymbolEncodingType, - pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], - pub(crate) huf_des_size: usize, -} - -/// `ZSTD_fseCTablesMetadata_t` -pub(crate) struct FseCTablesMetadata { - pub(crate) ll_type: SymbolEncodingType, - pub(crate) of_type: SymbolEncodingType, - pub(crate) ml_type: SymbolEncodingType, - pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], - pub(crate) fse_tables_size: usize, - /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. - pub(crate) last_count_size: usize, -} - -/// `ZSTD_entropyCTablesMetadata_t` -pub(crate) struct EntropyCTablesMetadata { - pub(crate) huf_metadata: HufCTablesMetadata, - pub(crate) fse_metadata: FseCTablesMetadata, -} - -impl EntropyCTablesMetadata { - pub(crate) const fn new() -> Self { - Self { - huf_metadata: HufCTablesMetadata { - h_type: SymbolEncodingType::Basic, - huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], - huf_des_size: 0, - }, - fse_metadata: FseCTablesMetadata { - ll_type: SymbolEncodingType::Basic, - of_type: SymbolEncodingType::Basic, - ml_type: SymbolEncodingType::Basic, - fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], - fse_tables_size: 0, - last_count_size: 0, - }, - } - } -} - -// ── sequences ───────────────────────────────────────────────────────────── - -/// `SeqDef` -#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] -pub(crate) struct SeqDef { - /// A repeat code (1 to 3), or an offset plus 3. - pub(crate) off_base: u32, - pub(crate) lit_length: u16, - /// The match length minus [`MINMATCH`]. - pub(crate) ml_base: u16, -} - -/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum LongLengthType { - None, - LiteralLength, - MatchLength, -} - -/// `SeqStore_t`: the sequences and the literals of a block. -pub(crate) struct SeqStore { - /// Never grows beyond the capacity it is given by [`Self::reserve`]. - pub(crate) sequences: Vec, - /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. - pub(crate) lit_buffer: Vec, - /// How much of `lit_buffer` is literals. - pub(crate) lit_len: usize, - pub(crate) ll_code: Vec, - pub(crate) ml_code: Vec, - pub(crate) of_code: Vec, - pub(crate) max_nb_seq: usize, - pub(crate) max_nb_lit: usize, - pub(crate) long_length_type: LongLengthType, - /// Which sequence has the long length. - pub(crate) long_length_pos: u32, -} - -/// `ZSTD_SequenceLength` -#[derive(Clone, Copy)] -pub(crate) struct SequenceLength { - pub(crate) lit_length: u32, - pub(crate) match_length: u32, -} - -impl SeqStore { - pub(crate) const fn new() -> Self { - Self { - sequences: Vec::new(), - lit_buffer: Vec::new(), - lit_len: 0, - ll_code: Vec::new(), - ml_code: Vec::new(), - of_code: Vec::new(), - max_nb_seq: 0, - max_nb_lit: 0, - long_length_type: LongLengthType::None, - long_length_pos: 0, - } - } - - /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. - pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { - fn grow(vec: &mut Vec, len: usize) -> Result<()> { - vec.clear(); - vec.try_reserve_exact(len) - .map_err(|_| ErrorCode::MemoryAllocation)?; - vec.resize(len, T::default()); - Ok(()) - } - grow(&mut self.sequences, max_nb_seq)?; - grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; - grow(&mut self.ll_code, max_nb_seq)?; - grow(&mut self.ml_code, max_nb_seq)?; - grow(&mut self.of_code, max_nb_seq)?; - self.max_nb_seq = max_nb_seq; - self.max_nb_lit = max_nb_lit; - self.reset(); - Ok(()) - } - - /// `ZSTD_resetSeqStore` - #[inline] - pub(crate) fn reset(&mut self) { - self.lit_len = 0; - self.sequences.clear(); - self.long_length_type = LongLengthType::None; - } - - #[inline] - pub(crate) fn literals(&self) -> &[u8] { - self.lit_buffer.get(..self.lit_len).unwrap_or_default() - } - - /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. - #[inline(always)] - pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { - debug_assert!(self.sequences.len() < self.max_nb_seq); - if lit_length > 0xFFFF { - self.long_length_type = LongLengthType::LiteralLength; - self.long_length_pos = self.sequences.len() as u32; - } - let ml_base = match_length.wrapping_sub(MINMATCH as usize); - if ml_base > 0xFFFF { - self.long_length_type = LongLengthType::MatchLength; - self.long_length_pos = self.sequences.len() as u32; - } - self.sequences.push(SeqDef { - off_base, - lit_length: lit_length as u16, - ml_base: ml_base as u16, - }); - } - - /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. - #[inline(always)] - pub(crate) fn store_seq( - &mut self, - lit_length: usize, - buf: &[u8], - literals: usize, - off_base: u32, - match_length: usize, - ) -> Result<()> { - match ( - buf.array_at::<16>(literals), - self.lit_buffer.array_at_mut::<16>(self.lit_len), - ) { - (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, - _ => self - .lit_buffer - .write_at(self.lit_len, buf.span(literals, lit_length)?)?, - } - self.lit_len += lit_length; - self.store_seq_only(lit_length, off_base, match_length); - Ok(()) - } - - /// Appends the literals that follow the last sequence of a block. - #[inline] - pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { - self.lit_buffer.write_at(self.lit_len, literals)?; - self.lit_len += literals.len(); - Ok(()) - } -} - -pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; -pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; -pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; - -/// `OFFSET_TO_OFFBASE`: `offset` is not 0. -#[inline(always)] -pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { - offset.wrapping_add(ZSTD_REP_NUM as u32) -} - -/// `OFFBASE_IS_OFFSET` -#[inline(always)] -pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { - off_base > ZSTD_REP_NUM as u32 -} - -/// `OFFBASE_IS_REPCODE` -#[inline(always)] -pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { - 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 -} - -/// `OFFBASE_TO_OFFSET` -#[inline(always)] -pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { - off_base.wrapping_sub(ZSTD_REP_NUM as u32) -} - -/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. -#[inline(always)] -pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { - let [rep0, rep1, rep2] = *rep; - if offbase_is_offset(off_base) { - *rep = [offbase_to_offset(off_base), rep0, rep1]; - } else { - match off_base - 1 + u32::from(ll0) { - 0 => {} - 1 => *rep = [rep1, rep0, rep2], - 2 => *rep = [rep2, rep0, rep1], - _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], - } - } -} - -/// `ZSTD_newRep` -#[inline(always)] -pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { - let mut new_reps = *rep; - update_rep(&mut new_reps, off_base, ll0); - new_reps -} - -/// `ZSTD_LLcode` -#[inline(always)] -pub(crate) fn ll_code(lit_length: u32) -> u32 { - #[rustfmt::skip] - const LL_CODE: [u8; 64] = [ - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, - ]; - const LL_DELTA_CODE: u32 = 19; - match LL_CODE.get(lit_length as usize) { - Some(&code) => u32::from(code), - None => highbit32(lit_length) + LL_DELTA_CODE, - } -} - -/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. -#[inline(always)] -pub(crate) fn ml_code(ml_base: u32) -> u32 { - #[rustfmt::skip] - const ML_CODE: [u8; 128] = [ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - ]; - const ML_DELTA_CODE: u32 = 36; - match ML_CODE.get(ml_base as usize) { - Some(&code) => u32::from(code), - None => highbit32(ml_base) + ML_DELTA_CODE, - } -} - -/// `ZSTD_noCompressBlock` -pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { - let c_block_header24 = - u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); - if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { - return Err(ErrorCode::DstSizeTooSmall); - } - write_le24(dst, 0, c_block_header24)?; - dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; - Ok(ZSTD_BLOCKHEADERSIZE + src.len()) -} - -/// `ZSTD_rleCompressBlock` -pub(crate) fn rle_compress_block( - dst: &mut [u8], - byte: u8, - src_size: usize, - last_block: bool, -) -> Result { - let c_block_header = - u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); - if dst.len() < 4 { - return Err(ErrorCode::DstSizeTooSmall); - } - write_le24(dst, 0, c_block_header)?; - dst.put(3, byte)?; - Ok(4) -} - -/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. -#[inline] -pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { - let minlog = if strat >= Strategy::BtUltra { - strat as u32 - 1 - } else { - 6 - }; - (src_size >> minlog) + 2 -} - -// ── matching ────────────────────────────────────────────────────────────── - -/// How many bytes `a` and `b` have in common at their start. -#[inline(always)] -pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { - let (a_words, _) = a.as_chunks::<8>(); - let (b_words, _) = b.as_chunks::<8>(); - let mut len = 0usize; - for (a_word, b_word) in a_words.iter().zip(b_words) { - let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); - if diff != 0 { - return len + (diff.trailing_zeros() >> 3) as usize; - } - len += 8; - } - let a_tail = a.get(len..).unwrap_or_default(); - let b_tail = b.get(len..).unwrap_or_default(); - len + a_tail - .iter() - .zip(b_tail) - .take_while(|(a_byte, b_byte)| a_byte == b_byte) - .count() -} - -/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. -#[inline(always)] -pub(crate) fn count( - buf: &[u8], - p_in: usize, - match_buf: &[u8], - p_match: usize, - p_in_limit: usize, -) -> Result { - Ok(common_prefix( - buf.sub(p_in..p_in_limit)?, - match_buf.sub(p_match..)?, - )) -} - -/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at -/// `i_start` in `buf`. -#[inline(always)] -pub(crate) fn count_2segments( - buf: &[u8], - ip: usize, - match_buf: &[u8], - match_pos: usize, - i_end: usize, - m_end: usize, - i_start: usize, -) -> Result { - let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); - if match_pos + match_length != m_end { - return Ok(match_length); - } - Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) -} - -const PRIME_3_BYTES: u32 = 506_832_829; -const PRIME_4_BYTES: u32 = 2_654_435_761; -const PRIME_5_BYTES: u64 = 889_523_592_379; -const PRIME_6_BYTES: u64 = 227_718_039_650_203; -const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; -pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; - -/// `ZSTD_hash3` -#[inline(always)] -const fn hash3(u: u32, h: u32, s: u32) -> u32 { - (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash4(u: u32, h: u32, s: u32) -> u32 { - ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash5(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash6(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash7(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -/// `ZSTD_hash8` -#[inline(always)] -pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { - (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} - -/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. -#[inline(always)] -pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { - Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) -} - -/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, -/// else 8. -#[inline(always)] -pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { - hash_ptr_salted(buf, pos, h_bits, mls, 0) -} - -/// `ZSTD_hashPtrSalted` -#[inline(always)] -pub(crate) fn hash_ptr_salted( - buf: &[u8], - pos: usize, - h_bits: u32, - mls: u32, - hash_salt: u64, -) -> Result { - Ok(match mls { - 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), - 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), - 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), - 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), - _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, - }) -} - -// ── the window ──────────────────────────────────────────────────────────── - -/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are -/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the -/// others, and an address is turned into a position in the buffer only to read. -pub(crate) type Addr = usize; - -/// The buffers that a window can have a segment in. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum BufId { - /// Where an empty window points. - None = 0, - /// The context's own buffer of input. - InBuff = 1, - /// What the caller gave to the call in progress. - User = 2, - /// The content of the dictionary, or the prefix. - Dict = 3, -} - -impl BufId { - /// The address of the first byte of the buffer. - #[inline(always)] - pub(crate) const fn start(self) -> Addr { - (self as usize + 1) << 44 - } -} - -/// The buffers themselves, for as long as a call lasts. -#[derive(Clone, Copy)] -pub(crate) struct Buffers<'a> { - pub(crate) in_buff: &'a [u8], - pub(crate) user: &'a [u8], - pub(crate) dict: &'a [u8], -} - -impl<'a> Buffers<'a> { - #[inline] - pub(crate) fn get(&self, id: BufId) -> &'a [u8] { - match id { - BufId::None => &[], - BufId::InBuff => self.in_buff, - BufId::User => self.user, - BufId::Dict => self.dict, - } - } -} - -/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. -pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; - -/// The index past which indices are brought down. -pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; -/// The most input that can be indexed at once. -pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; - -/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the -/// index is less than `dict_limit`. -#[derive(Clone, Copy, Debug)] -pub(crate) struct Window { - /// Where input has to continue to be contiguous. - pub(crate) next_src: Addr, - pub(crate) base: Addr, - pub(crate) dict_base: Addr, - /// The buffer of the segment that starts at `dict_limit`. - pub(crate) base_buf: BufId, - /// The buffer of the segment from `low_limit` to `dict_limit`. - pub(crate) dict_buf: BufId, - pub(crate) dict_limit: u32, - /// Below this, there is nothing valid. - pub(crate) low_limit: u32, - pub(crate) nb_overflow_corrections: u32, -} - -impl Window { - /// `ZSTD_window_init` - pub(crate) const fn new() -> Self { - let base = BufId::None.start(); - Self { - next_src: base + ZSTD_WINDOW_START_INDEX as usize, - base, - dict_base: base, - base_buf: BufId::None, - dict_buf: BufId::None, - dict_limit: ZSTD_WINDOW_START_INDEX, - low_limit: ZSTD_WINDOW_START_INDEX, - nb_overflow_corrections: 0, - } - } - - /// The index of `addr`, which is in the current segment or at its end. - #[inline(always)] - pub(crate) const fn index_of(&self, addr: Addr) -> u32 { - addr.wrapping_sub(self.base) as u32 - } - - /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. - pub(crate) fn clear(&mut self) { - let end = self.index_of(self.next_src); - self.low_limit = end; - self.dict_limit = end; - } - - /// `ZSTD_window_isEmpty` - pub(crate) fn is_empty(&self) -> bool { - self.dict_limit == ZSTD_WINDOW_START_INDEX - && self.low_limit == ZSTD_WINDOW_START_INDEX - && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize - } - - /// `ZSTD_window_hasExtDict` - #[inline] - pub(crate) fn has_ext_dict(&self) -> bool { - self.low_limit < self.dict_limit - } - - /// `ZSTD_window_needOverflowCorrection` - #[inline] - pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { - self.index_of(src_end) > ZSTD_CURRENT_MAX - } - - /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns - /// what to subtract from every index that is stored. - pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { - let cycle_size = 1u32 << cycle_log; - let cycle_mask = cycle_size - 1; - let curr = self.index_of(src); - let current_cycle = curr & cycle_mask; - // Indices must not fall below the start index. - let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { - cycle_size.max(ZSTD_WINDOW_START_INDEX) - } else { - 0 - }; - let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); - let correction = curr.wrapping_sub(new_current); - - self.base = self.base.wrapping_add(correction as usize); - self.dict_base = self.dict_base.wrapping_add(correction as usize); - self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { - ZSTD_WINDOW_START_INDEX - } else { - self.low_limit - correction - }; - self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { - ZSTD_WINDOW_START_INDEX - } else { - self.dict_limit - correction - }; - self.nb_overflow_corrections += 1; - correction - } - - /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the - /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. - pub(crate) fn enforce_max_dist( - &mut self, - block_end: Addr, - max_dist: u32, - loaded_dict_end: u32, - ) -> bool { - let block_end_idx = self.index_of(block_end); - if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { - let new_low_limit = block_end_idx - max_dist; - if self.low_limit < new_low_limit { - self.low_limit = new_low_limit; - } - if self.dict_limit < self.low_limit { - self.dict_limit = self.low_limit; - } - return true; - } - false - } - - /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not - /// right before the current segment, and is to be forgotten. - pub(crate) fn dict_is_invalid( - &self, - block_end: Addr, - max_dist: u32, - loaded_dict_end: u32, - ) -> bool { - let block_end_idx = self.index_of(block_end); - block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit - } - - /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not - /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. - pub(crate) fn update( - &mut self, - src: Addr, - src_buf: BufId, - src_size: usize, - force_non_contiguous: bool, - ) -> bool { - let mut contiguous = true; - if src_size == 0 { - return contiguous; - } - if src != self.next_src || force_non_contiguous { - let distance_from_base = self.next_src.wrapping_sub(self.base); - self.low_limit = self.dict_limit; - self.dict_limit = distance_from_base as u32; - self.dict_base = self.base; - self.dict_buf = self.base_buf; - self.base = src.wrapping_sub(distance_from_base); - self.base_buf = src_buf; - if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { - // Too small to be of use. - self.low_limit = self.dict_limit; - } - contiguous = false; - } - self.next_src = src + src_size; - // Where the input overlaps the older segment, it has overwritten it. - if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) - & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) - { - let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); - self.low_limit = if high_input_idx > self.dict_limit as usize { - self.dict_limit - } else { - high_input_idx as u32 - }; - } - contiguous - } -} - -/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes -/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. -#[inline(always)] -pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { - prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 -} - -pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; -pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; - -/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. -#[inline(always)] -pub(crate) fn write_tagged_index( - hash_table: &mut [u32], - hash_and_tag: usize, - index: u32, -) -> Result<()> { - let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; - hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) -} - -/// `ZSTD_comparePackedTags` -#[inline(always)] -pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { - (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) - == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) -} - -// ── the state of the match finders ──────────────────────────────────────── - -/// `ZSTD_match_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct Match { - /// As [`SeqDef::off_base`]. - pub(crate) off: u32, - pub(crate) len: u32, -} - -/// `rawSeq`: a sequence found by long distance matching. -#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] -pub(crate) struct RawSeq { - pub(crate) offset: u32, - pub(crate) lit_length: u32, - pub(crate) match_length: u32, -} - -/// `RawSeqStore_t` -#[derive(Clone, Default)] -pub(crate) struct RawSeqStore { - /// `capacity` of them, of which `size` are in use. - pub(crate) seq: Vec, - /// The sequence that is being read. - pub(crate) pos: usize, - /// How many of its bytes, literals first, are read. - pub(crate) pos_in_sequence: usize, - pub(crate) size: usize, - pub(crate) capacity: usize, -} - -/// `ZSTD_optimal_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct Optimal { - /// The cost of getting to this position. - pub(crate) price: i32, - pub(crate) off: u32, - pub(crate) mlen: u32, - pub(crate) litlen: u32, - pub(crate) rep: [u32; ZSTD_REP_NUM], -} - -/// `ZSTD_OptPrice_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum OptPrice { - Dynamic, - Predef, -} - -/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. -pub(crate) struct OptState { - pub(crate) lit_freq: Vec, - pub(crate) lit_length_freq: Vec, - pub(crate) match_length_freq: Vec, - pub(crate) off_code_freq: Vec, - pub(crate) match_table: Vec, - pub(crate) price_table: Vec, - pub(crate) lit_sum: u32, - pub(crate) lit_length_sum: u32, - pub(crate) match_length_sum: u32, - pub(crate) off_code_sum: u32, - pub(crate) lit_sum_base_price: u32, - pub(crate) lit_length_sum_base_price: u32, - pub(crate) match_length_sum_base_price: u32, - pub(crate) off_code_sum_base_price: u32, - pub(crate) price_type: OptPrice, - pub(crate) literal_compression_mode: ParamSwitch, -} - -impl OptState { - pub(crate) const fn new() -> Self { - Self { - lit_freq: Vec::new(), - lit_length_freq: Vec::new(), - match_length_freq: Vec::new(), - off_code_freq: Vec::new(), - match_table: Vec::new(), - price_table: Vec::new(), - lit_sum: 0, - lit_length_sum: 0, - match_length_sum: 0, - off_code_sum: 0, - lit_sum_base_price: 0, - lit_length_sum_base_price: 0, - match_length_sum_base_price: 0, - off_code_sum_base_price: 0, - price_type: OptPrice::Dynamic, - literal_compression_mode: ParamSwitch::Auto, - } - } -} - -pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; - -/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures -/// own, are in [`BlockSrc`]. -pub(crate) struct MatchState { - pub(crate) window: Window, - /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. - pub(crate) loaded_dict_end: u32, - /// The index from which the tables are still to be filled. - pub(crate) next_to_update: u32, - /// 0 if there is no table of 3-byte hashes. - pub(crate) hash_log3: u32, - pub(crate) row_hash_log: u32, - /// For the row-based match finder: a byte of hash for each entry of `hash_table`. - pub(crate) tag_table: Vec, - pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], - /// Makes what a previous frame left in `tag_table` meaningless. - pub(crate) hash_salt: u64, - pub(crate) hash_salt_entropy: u32, - pub(crate) hash_table: Vec, - pub(crate) hash_table3: Vec, - pub(crate) chain_table: Vec, - pub(crate) force_non_contiguous: bool, - pub(crate) opt: OptState, - /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. - pub(crate) has_dict_match_state: bool, - pub(crate) c_params: CompressionParameters, - pub(crate) lazy_skipping: bool, -} - -impl MatchState { - pub(crate) const fn new() -> Self { - Self { - window: Window::new(), - loaded_dict_end: 0, - next_to_update: 0, - hash_log3: 0, - row_hash_log: 0, - tag_table: Vec::new(), - hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], - hash_salt: 0, - hash_salt_entropy: 0, - hash_table: Vec::new(), - hash_table3: Vec::new(), - chain_table: Vec::new(), - force_non_contiguous: false, - opt: OptState::new(), - has_dict_match_state: false, - c_params: CompressionParameters { - window_log: 0, - chain_log: 0, - hash_log: 0, - search_log: 0, - min_match: 0, - target_length: 0, - strategy: None, - }, - lazy_skipping: false, - } - } - - /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. - pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { - if self.window.has_ext_dict() { - return DictMode::ExtDict; - } - if dms.is_some() { - DictMode::DictMatchState - } else { - DictMode::NoDict - } - } - - /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. - #[inline(always)] - pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { - let max_distance = 1u32 << window_log; - let lowest_valid = self.window.low_limit; - let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { - curr - max_distance - } else { - lowest_valid - }; - // A dictionary is valid as a whole until it is invalidated as a whole. - if self.loaded_dict_end != 0 { - lowest_valid - } else { - within_window - } - } - - /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. - #[inline(always)] - pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { - let max_distance = 1u32 << window_log; - let lowest_valid = self.window.dict_limit; - let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { - curr - max_distance - } else { - lowest_valid - }; - if self.loaded_dict_end != 0 { - lowest_valid - } else { - within_window - } - } -} - -/// The match state of a dictionary, and where its indices lead. -#[derive(Clone, Copy)] -pub(crate) struct DictMatchState<'a> { - pub(crate) ms: &'a MatchState, - /// The buffer its window is in, which is one segment. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, - /// `window.nextSrc`, as a position in `buf`. - pub(crate) end: usize, -} - -/// The current segment of a window, for the functions that index a dictionary. -#[derive(Clone, Copy)] -pub(crate) struct Prefix<'a> { - /// The buffer that the segment is in. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, -} - -impl Prefix<'_> { - /// The index of the position `pos` of `buf`. - #[inline(always)] - pub(crate) const fn index_of(&self, pos: usize) -> u32 { - pos.wrapping_sub(self.base) as u32 - } - - /// The position in `buf` of `index`. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } -} - -/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. -#[derive(Clone, Copy)] -pub(crate) struct BlockSrc<'a> { - /// The buffer that the current segment of the window is in, and so the block. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, - /// The buffer that the older segment of the window is in. - pub(crate) dict_buf: &'a [u8], - /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. - pub(crate) dict_base: usize, - /// Where the block starts in `buf`. - pub(crate) start: usize, - /// Where it ends. - pub(crate) end: usize, - pub(crate) dms: Option>, - /// `opt.symbolCosts`: the tables of the previous block. - pub(crate) symbol_costs: &'a EntropyCTables, - /// `ldmSeqStore`: matches that the optimal parser is to consider. - pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, -} - -impl<'a> BlockSrc<'a> { - #[inline(always)] - pub(crate) const fn prefix(&self) -> Prefix<'a> { - Prefix { - buf: self.buf, - base: self.base, - } - } - - /// The index of the position `pos` of `buf`. - #[inline(always)] - pub(crate) const fn index_of(&self, pos: usize) -> u32 { - pos.wrapping_sub(self.base) as u32 - } - - /// The position in `buf` of `index`, which is in the current segment. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } - - /// The position in `dict_buf` of `index`, which is in the older segment. - #[inline(always)] - pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { - self.dict_base.wrapping_add(index as usize) - } -} - -impl DictMatchState<'_> { - /// The position in `buf` of `index`. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } -} - -// ── long distance matching ──────────────────────────────────────────────── - -/// `ldmEntry_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct LdmEntry { - pub(crate) offset: u32, - pub(crate) checksum: u32, -} - -/// `ldmParams_t` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) struct LdmParams { - pub(crate) enable_ldm: ParamSwitch, - pub(crate) hash_log: u32, - pub(crate) bucket_size_log: u32, - pub(crate) min_match_length: u32, - pub(crate) hash_rate_log: u32, - pub(crate) window_log: u32, -} - -/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. -pub(crate) struct LdmState { - pub(crate) window: Window, - pub(crate) hash_table: Vec, - pub(crate) loaded_dict_end: u32, - /// The next entry to replace in each bucket. - pub(crate) bucket_offsets: Vec, -} - -impl LdmState { - pub(crate) const fn new() -> Self { - Self { - window: Window::new(), - hash_table: Vec::new(), - loaded_dict_end: 0, - bucket_offsets: Vec::new(), - } - } -} diff --git a/src/zstd/wip/work/fast/zstd/compress/lazy.rs b/src/zstd/wip/work/fast/zstd/compress/lazy.rs deleted file mode 100644 index 9f653fd684f3..000000000000 --- a/src/zstd/wip/work/fast/zstd/compress/lazy.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. - -use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _ip: usize, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_row_update`: the same, in the rows of the row-based match finder. -pub(crate) fn row_update(_ms: &mut MatchState, _prefix: Prefix<'_>, _ip: usize) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: -/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. -pub(crate) fn compress_block_lazy( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, - _strategy: Strategy, - _use_row: bool, - _dict_mode: DictMode, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/fast/zstd/compress/ldm.rs b/src/zstd/wip/work/fast/zstd/compress/ldm.rs deleted file mode 100644 index 2177fd945e10..000000000000 --- a/src/zstd/wip/work/fast/zstd/compress/ldm.rs +++ /dev/null @@ -1 +0,0 @@ -//! `zstd_ldm.c`: long distance matching. diff --git a/src/zstd/wip/work/fast/zstd/compress/mod.rs b/src/zstd/wip/work/fast/zstd/compress/mod.rs deleted file mode 100644 index 13ddecdf82e4..000000000000 --- a/src/zstd/wip/work/fast/zstd/compress/mod.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! `lib/compress` - -pub(crate) mod double_fast; -pub(crate) mod fast; -pub(crate) mod fse; -pub(crate) mod hist; -pub(crate) mod huf; -pub(crate) mod internal; -pub(crate) mod lazy; -pub(crate) mod ldm; -pub(crate) mod opt; -pub(crate) mod presplit; - -use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; - -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; - -/// `ZSTD_rowMatchFinderSupported` -pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { - strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 -} - -/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to -/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies -/// that support it look at. Returns how many bytes at the end of the block are in no sequence. -pub(crate) fn compress_block( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row_match_finder: bool, - dict_mode: DictMode, -) -> Result { - match strategy { - Strategy::Fast => match dict_mode { - DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), - DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), - DictMode::DictMatchState => { - fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) - } - }, - Strategy::DFast => match dict_mode { - DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), - DictMode::ExtDict => { - double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) - } - }, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { - lazy::compress_block_lazy( - ms, - seq_store, - rep, - src, - strategy, - use_row_match_finder && row_match_finder_supported(strategy), - dict_mode, - ) - } - Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { - opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) - } - } -} diff --git a/src/zstd/wip/work/fast/zstd/compress/opt.rs b/src/zstd/wip/work/fast/zstd/compress/opt.rs deleted file mode 100644 index d055a44cf982..000000000000 --- a/src/zstd/wip/work/fast/zstd/compress/opt.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. - -use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. -/// Matches are not compared past the position `iend`. -pub(crate) fn update_tree( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _ip: usize, - _iend: usize, -) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` -/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. -pub(crate) fn compress_block_opt( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, - _strategy: Strategy, - _dict_mode: DictMode, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/fast/zstd/compress/presplit.rs b/src/zstd/wip/work/fast/zstd/compress/presplit.rs deleted file mode 100644 index 8b778e2989f9..000000000000 --- a/src/zstd/wip/work/fast/zstd/compress/presplit.rs +++ /dev/null @@ -1 +0,0 @@ -//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. diff --git a/src/zstd/wip/work/lazy/STATUS.md b/src/zstd/wip/work/lazy/STATUS.md new file mode 100644 index 000000000000..6884f955acc4 --- /dev/null +++ b/src/zstd/wip/work/lazy/STATUS.md @@ -0,0 +1,53 @@ +# lazy (zstd_lazy.c -> compress/lazy.rs): status + +## Deliverable +`/root/zstd-ref/work/lazy/zstd/compress/lazy.rs` (same file saved as `/root/zstd-ref/work/lazy/lazy.v2.rs`). +Needs `#![feature(hint_prefetch)]` in lib.rs (safe `core::hint::prefetch_read`). `cargo fmt --check` ok, `cargo clippy`: no +finding in lazy.rs. Public API of the stub unchanged, plus `pub(crate) const ZSTD_ROW_HASH_TAG_BITS` (from zstd_lazy.h; params/cctx +need it for `maxRowHashLog`). Private `prefetch_l1` at the top of the file, to be moved to util.rs. +`lazy.v1.rs` = the same without any prefetch (builds without the feature). + +## Structure +- `ZSTD_compressBlock_lazy_generic` and `..._extDict_generic` are ONE fn `compress_block_lazy_generic` + (`#[inline(never)]`, 21 instances). The only differences in C are kept: the initial `ip += ...`, the saved rep offsets (noDict) and the + lazy-skipping threshold (`step > 8` with step one less in extDict). +- The 4 copies of the repcode check are `rep_length` (0 = no match). `DictSegment` = what an index below dictLimit leads to + (older segment, or the dms with `base - dictIndexDelta`). +- Search fns are `#[inline(never)]` const-generic: `hc_find_best_match`, `bt_find_best_match`, + `row_find_best_match`; `search_max` switches on mls/rowLog at each call as C does. +- `dummy32` is done by aliasing the two slots (`smaller_ptr = larger_ptr`) so the final clear writes only the other one. +- If `iend < 8` (or 16 with rows) as a position in the buffer, `ilimit` would be negative: early return (no search, no hashing, rep unchanged). + +## Tests (all against C, sequences AND final match state) +Harness: `/root/zstd-ref/work/lazy/harness` (+ `shim.c`, reads cctx internals), driver `/root/zstd-ref/work/lazy/zstd/testing.rs` (`frame`). +Scratch-only: `zstd/compress/opt.rs::update_tree` is a throw-away noDict port, only to load dictionaries for btlazy2. + cd /root/zstd-ref/work/lazy/harness && RUSTFLAGS="--cap-lints=warn" cargo build --release + ./target/release/zstd_harness seqs 3,4,5,6 +mode 0 no dict, 1 prefix (extDict), 2 attached CDict (dictMatchState), 3 through ZSTD_compressStream2's wrapping input buffer +(extDict with dict_buf == buf), mixed with none/prefix/CDict. Frames are multi-block (up to 400 KB, windowLog 10..22). +Compared after each frame: every sequence (ll, ml, repcode/offset), hash/chain/tag tables, hashCache, nextToUpdate, hashSaltEntropy, +lazySkipping, lowLimit, dictLimit, loadedDictEnd, rep, whether the dms is still attached. + ./target/release/zstd_harness bench <0|1|2> (build with --profile bench for real numbers) + +## Verified (no mismatch, no error, no overflow panic; harness has overflow checks + debug assertions on) +- First version (no prefetch, plain mask loop), complete runs: mode 0: 320,000 frames; mode 1: 320,000; mode 2: 320,000 (134,306 with a CDict attached). +- Final file (prefetch, mask by chunks of 16), runs stopped by a time limit, not by a failure (logs/final_*.log): mode 0: >=152,000; mode 1: >=140,000; + mode 2: >=306,000 (2 seeds); mode 3: >=284,000 (2 seeds). Plus 3,000 to 8,000 of each mode on other seeds. +- Line coverage (llvm-cov) over modes 0-3: every line of lazy.rs is hit except `?` error paths, `_ => Err(BUG)` and the never-false `off_base != 0`. + `insert_dubt1` with `curr < dict_limit` and the extDict "offset beyond the window" test are only reached by mode 3. + +## Not tested +- Index overflow correction (ZSTD_window_correctOverflow / reduceIndex, DUBT mark preservation): not in this file, not exercised. +- Reusing a context for several frames (hash salt != first value, stale tag table). hashSaltEntropy is compared, the salted second frame is not run. +- CDict copied rather than attached; ldm + lazy (`ZSTD_ldm_blockCompress` calling these); aarch64 only inspected as asm, not run. + +## Speed (7 MB of source text, sequences only, Rust as % of C; logs/bench0.txt) +greedy HC 115, greedy row 108, lazy HC 96, lazy row 88, lazy2 HC 100, lazy2 row L10 77-82, L12 75-78, btlazy2 97-98. +Without prefetch the row finder was 97/78/62/59. +Open: the rest of the row gap. Known causes seen in the asm of `row_find_best_match`: +1. the first 16 tag bytes are assembled from pieces (byte 0 is also loaded as the scalar `head` in an earlier block, and SLP reuses it) instead of + one movdqu: about 15 extra shuffles per search, on aarch64 too. In isolation the fn compiles to movdqu+pcmpeqb+pmovmskb. + Tried without success: `.max(1)` on nb_attempts; loading the chunk as u128 (slower). +2. `match_buffer` is zeroed on each call (64 to 256 bytes); 3. two bounds checks per 4-byte read in the candidate loop (`read_le32(ip + ml - 3)` could + be kept in a variable that changes with `ml`); 4. the rare "skip > 384 positions" path is inlined and costs registers (could be `#[cold]`). +Tried and rejected: prefetch emulated by real loads (`black_box`: +5% at best; early filtering load of the candidates' first 4 bytes: slower). diff --git a/src/zstd/wip/work/lazy/zstd/compress/double_fast.rs b/src/zstd/wip/work/lazy/zstd/compress/double_fast.rs deleted file mode 100644 index 65d401675d1b..000000000000 --- a/src/zstd/wip/work/lazy/zstd/compress/double_fast.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. - -use super::internal::{ - BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_double_hash_table( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _end: usize, - _dtlm: DictTableLoadMethod, - _tfp: TableFillPurpose, -) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast` -pub(crate) fn compress_block_double_fast( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast_dictMatchState` -pub(crate) fn compress_block_double_fast_dict_match_state( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast_extDict` -pub(crate) fn compress_block_double_fast_ext_dict( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/lazy/zstd/compress/fast.rs b/src/zstd/wip/work/lazy/zstd/compress/fast.rs deleted file mode 100644 index 1f4788af28b4..000000000000 --- a/src/zstd/wip/work/lazy/zstd/compress/fast.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. - -use super::internal::{ - BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_hash_table( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _end: usize, - _dtlm: DictTableLoadMethod, - _tfp: TableFillPurpose, -) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast` -pub(crate) fn compress_block_fast( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast_dictMatchState` -pub(crate) fn compress_block_fast_dict_match_state( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast_extDict` -pub(crate) fn compress_block_fast_ext_dict( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/lazy/zstd/compress/fse.rs b/src/zstd/wip/work/lazy/zstd/compress/fse.rs deleted file mode 100644 index 1b8f7cbf3173..000000000000 --- a/src/zstd/wip/work/lazy/zstd/compress/fse.rs +++ /dev/null @@ -1,70 +0,0 @@ -//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. - -use crate::common::{MAX_SEQ, ML_FSE_LOG}; -use crate::util::SliceExt; - -/// `FSE_repeat` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum FseRepeat { - /// The table cannot be used again. - None, - /// It can, if it has a code for every symbol to encode. - Check, - /// It can. - Valid, -} - -/// `FSE_symbolCompressionTransform` -#[derive(Clone, Copy, Default)] -pub(crate) struct FseSymbolTransform { - pub(crate) delta_find_state: i32, - pub(crate) delta_nb_bits: u32, -} - -/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. -#[derive(Clone)] -pub(crate) struct FseCTable { - pub(crate) table_log: u32, - pub(crate) max_symbol_value: u32, - pub(crate) state_table: [u16; 1 << ML_FSE_LOG], - pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], -} - -impl FseCTable { - pub(crate) const fn new() -> Self { - Self { - table_log: 0, - max_symbol_value: 0, - state_table: [0; 1 << ML_FSE_LOG], - symbol_tt: [FseSymbolTransform { - delta_find_state: 0, - delta_nb_bits: 0, - }; MAX_SEQ as usize + 1], - } - } - - /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log - /// if the table has no code for it. - #[inline(always)] - pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { - let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); - transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 - } - - /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. - /// The table has a code for it. - #[inline(always)] - pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { - let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); - let min_nb_bits = transform.delta_nb_bits >> 16; - let threshold = (min_nb_bits + 1) << 16; - let table_size = 1u32 << self.table_log; - let delta_from_threshold = - threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); - // A linear interpolation, which is very approximate. - let normalized_delta_from_threshold = - (delta_from_threshold << accuracy_log) >> self.table_log; - let bit_multiplier = 1u32 << accuracy_log; - ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) - } -} diff --git a/src/zstd/wip/work/lazy/zstd/compress/hist.rs b/src/zstd/wip/work/lazy/zstd/compress/hist.rs deleted file mode 100644 index becf42516867..000000000000 --- a/src/zstd/wip/work/lazy/zstd/compress/hist.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! `hist.c`: how many times each byte value occurs. - -use crate::error::Result; -use crate::util::SliceExt; - -/// `HIST_add`: `count` has room for every value in `src`. -pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { - for &byte in src { - *count.at_mut(usize::from(byte))? += 1; - } - Ok(()) -} - -/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. -/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. -pub(crate) fn count_simple( - count: &mut [u32], - max_symbol_value: &mut u32, - src: &[u8], -) -> Result { - let mut max_symbol = *max_symbol_value as usize; - count.sub_mut(..=max_symbol)?.fill(0); - if src.is_empty() { - *max_symbol_value = 0; - return Ok(0); - } - add(count, src)?; - while count.at(max_symbol)? == 0 { - max_symbol -= 1; - } - *max_symbol_value = max_symbol as u32; - Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) -} diff --git a/src/zstd/wip/work/lazy/zstd/compress/huf.rs b/src/zstd/wip/work/lazy/zstd/compress/huf.rs deleted file mode 100644 index 78373a89ebf0..000000000000 --- a/src/zstd/wip/work/lazy/zstd/compress/huf.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! `huf_compress.c`: Huffman coding of literals. - -use crate::common::HUF_SYMBOLVALUE_MAX; -use crate::util::SliceExt; - -/// `HUF_repeat` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum HufRepeat { - /// The table cannot be used again. - None, - /// It can, if it has a code for every symbol to encode. - Check, - /// It can. - Valid, -} - -/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the -/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in -/// its highest bits. -#[derive(Clone)] -pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); - -impl HufCTable { - pub(crate) const fn new() -> Self { - Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) - } - - /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. - #[inline] - pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { - let [header, elements @ ..] = &self.0; - let max_symbol_value = (header >> 8) as u8; - if symbol_value > u32::from(max_symbol_value) { - return 0; - } - (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 - } -} diff --git a/src/zstd/wip/work/lazy/zstd/compress/internal.rs b/src/zstd/wip/work/lazy/zstd/compress/internal.rs deleted file mode 100644 index 51157ea6c1ce..000000000000 --- a/src/zstd/wip/work/lazy/zstd/compress/internal.rs +++ /dev/null @@ -1,1212 +0,0 @@ -//! `zstd_compress_internal.h`: what the parts of the compressor share. - -use super::fse::{FseCTable, FseRepeat}; -use super::huf::{HufCTable, HufRepeat}; -use crate::common::{ - BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, - ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, -}; -use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; - -pub(crate) const K_SEARCH_STRENGTH: u32 = 8; -pub(crate) const HASH_READ_SIZE: usize = 8; -/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. -pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; - -pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; -pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; - -pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; -pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; - -/// `ZSTD_strategy` -#[repr(u8)] -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] -pub(crate) enum Strategy { - Fast = 1, - DFast = 2, - Greedy = 3, - Lazy = 4, - Lazy2 = 5, - BtLazy2 = 6, - BtOpt = 7, - BtUltra = 8, - BtUltra2 = 9, -} - -impl Strategy { - pub(crate) const fn from_int(value: i32) -> Option { - Some(match value { - 1 => Self::Fast, - 2 => Self::DFast, - 3 => Self::Greedy, - 4 => Self::Lazy, - 5 => Self::Lazy2, - 6 => Self::BtLazy2, - 7 => Self::BtOpt, - 8 => Self::BtUltra, - 9 => Self::BtUltra2, - _ => return None, - }) - } -} - -/// `ZSTD_ParamSwitch_e` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum ParamSwitch { - Auto = 0, - Enable = 1, - Disable = 2, -} - -/// `ZSTD_compressionParameters` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) struct CompressionParameters { - pub(crate) window_log: u32, - pub(crate) chain_log: u32, - pub(crate) hash_log: u32, - pub(crate) search_log: u32, - pub(crate) min_match: u32, - pub(crate) target_length: u32, - /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. - pub(crate) strategy: Option, -} - -impl CompressionParameters { - /// The strategy of parameters that are ready to be applied. - #[inline(always)] - pub(crate) fn strat(&self) -> Strategy { - self.strategy.unwrap_or(Strategy::Fast) - } -} - -/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum DictMode { - NoDict, - ExtDict, - DictMatchState, -} - -/// `ZSTD_dictTableLoadMethod_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum DictTableLoadMethod { - Fast, - Full, -} - -/// `ZSTD_tableFillPurpose_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum TableFillPurpose { - ForCCtx, - ForCDict, -} - -// ── entropy tables ──────────────────────────────────────────────────────── - -/// `ZSTD_hufCTables_t` -#[derive(Clone)] -pub(crate) struct HufCTables { - pub(crate) c_table: HufCTable, - pub(crate) repeat_mode: HufRepeat, -} - -/// `ZSTD_fseCTables_t` -#[derive(Clone)] -pub(crate) struct FseCTables { - pub(crate) offcode_c_table: FseCTable, - pub(crate) matchlength_c_table: FseCTable, - pub(crate) litlength_c_table: FseCTable, - pub(crate) offcode_repeat_mode: FseRepeat, - pub(crate) matchlength_repeat_mode: FseRepeat, - pub(crate) litlength_repeat_mode: FseRepeat, -} - -/// `ZSTD_entropyCTables_t` -#[derive(Clone)] -pub(crate) struct EntropyCTables { - pub(crate) huf: HufCTables, - pub(crate) fse: FseCTables, -} - -/// `ZSTD_compressedBlockState_t` -#[derive(Clone)] -pub(crate) struct CompressedBlockState { - pub(crate) entropy: EntropyCTables, - pub(crate) rep: [u32; ZSTD_REP_NUM], -} - -impl CompressedBlockState { - /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. - pub(crate) fn new() -> Self { - Self { - entropy: EntropyCTables { - huf: HufCTables { - c_table: HufCTable::new(), - repeat_mode: HufRepeat::None, - }, - fse: FseCTables { - offcode_c_table: FseCTable::new(), - matchlength_c_table: FseCTable::new(), - litlength_c_table: FseCTable::new(), - offcode_repeat_mode: FseRepeat::None, - matchlength_repeat_mode: FseRepeat::None, - litlength_repeat_mode: FseRepeat::None, - }, - }, - rep: REP_START_VALUE, - } - } - - /// `ZSTD_reset_compressedBlockState` - pub(crate) fn reset(&mut self) { - self.rep = REP_START_VALUE; - self.entropy.huf.repeat_mode = HufRepeat::None; - self.entropy.fse.offcode_repeat_mode = FseRepeat::None; - self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; - self.entropy.fse.litlength_repeat_mode = FseRepeat::None; - } -} - -/// `ZSTD_hufCTablesMetadata_t` -pub(crate) struct HufCTablesMetadata { - pub(crate) h_type: SymbolEncodingType, - pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], - pub(crate) huf_des_size: usize, -} - -/// `ZSTD_fseCTablesMetadata_t` -pub(crate) struct FseCTablesMetadata { - pub(crate) ll_type: SymbolEncodingType, - pub(crate) of_type: SymbolEncodingType, - pub(crate) ml_type: SymbolEncodingType, - pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], - pub(crate) fse_tables_size: usize, - /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. - pub(crate) last_count_size: usize, -} - -/// `ZSTD_entropyCTablesMetadata_t` -pub(crate) struct EntropyCTablesMetadata { - pub(crate) huf_metadata: HufCTablesMetadata, - pub(crate) fse_metadata: FseCTablesMetadata, -} - -impl EntropyCTablesMetadata { - pub(crate) const fn new() -> Self { - Self { - huf_metadata: HufCTablesMetadata { - h_type: SymbolEncodingType::Basic, - huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], - huf_des_size: 0, - }, - fse_metadata: FseCTablesMetadata { - ll_type: SymbolEncodingType::Basic, - of_type: SymbolEncodingType::Basic, - ml_type: SymbolEncodingType::Basic, - fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], - fse_tables_size: 0, - last_count_size: 0, - }, - } - } -} - -// ── sequences ───────────────────────────────────────────────────────────── - -/// `SeqDef` -#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] -pub(crate) struct SeqDef { - /// A repeat code (1 to 3), or an offset plus 3. - pub(crate) off_base: u32, - pub(crate) lit_length: u16, - /// The match length minus [`MINMATCH`]. - pub(crate) ml_base: u16, -} - -/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum LongLengthType { - None, - LiteralLength, - MatchLength, -} - -/// `SeqStore_t`: the sequences and the literals of a block. -pub(crate) struct SeqStore { - /// Never grows beyond the capacity it is given by [`Self::reserve`]. - pub(crate) sequences: Vec, - /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. - pub(crate) lit_buffer: Vec, - /// How much of `lit_buffer` is literals. - pub(crate) lit_len: usize, - pub(crate) ll_code: Vec, - pub(crate) ml_code: Vec, - pub(crate) of_code: Vec, - pub(crate) max_nb_seq: usize, - pub(crate) max_nb_lit: usize, - pub(crate) long_length_type: LongLengthType, - /// Which sequence has the long length. - pub(crate) long_length_pos: u32, -} - -/// `ZSTD_SequenceLength` -#[derive(Clone, Copy)] -pub(crate) struct SequenceLength { - pub(crate) lit_length: u32, - pub(crate) match_length: u32, -} - -impl SeqStore { - pub(crate) const fn new() -> Self { - Self { - sequences: Vec::new(), - lit_buffer: Vec::new(), - lit_len: 0, - ll_code: Vec::new(), - ml_code: Vec::new(), - of_code: Vec::new(), - max_nb_seq: 0, - max_nb_lit: 0, - long_length_type: LongLengthType::None, - long_length_pos: 0, - } - } - - /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. - pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { - fn grow(vec: &mut Vec, len: usize) -> Result<()> { - vec.clear(); - vec.try_reserve_exact(len) - .map_err(|_| ErrorCode::MemoryAllocation)?; - vec.resize(len, T::default()); - Ok(()) - } - grow(&mut self.sequences, max_nb_seq)?; - grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; - grow(&mut self.ll_code, max_nb_seq)?; - grow(&mut self.ml_code, max_nb_seq)?; - grow(&mut self.of_code, max_nb_seq)?; - self.max_nb_seq = max_nb_seq; - self.max_nb_lit = max_nb_lit; - self.reset(); - Ok(()) - } - - /// `ZSTD_resetSeqStore` - #[inline] - pub(crate) fn reset(&mut self) { - self.lit_len = 0; - self.sequences.clear(); - self.long_length_type = LongLengthType::None; - } - - #[inline] - pub(crate) fn literals(&self) -> &[u8] { - self.lit_buffer.get(..self.lit_len).unwrap_or_default() - } - - /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. - #[inline(always)] - pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { - debug_assert!(self.sequences.len() < self.max_nb_seq); - if lit_length > 0xFFFF { - self.long_length_type = LongLengthType::LiteralLength; - self.long_length_pos = self.sequences.len() as u32; - } - let ml_base = match_length.wrapping_sub(MINMATCH as usize); - if ml_base > 0xFFFF { - self.long_length_type = LongLengthType::MatchLength; - self.long_length_pos = self.sequences.len() as u32; - } - self.sequences.push(SeqDef { - off_base, - lit_length: lit_length as u16, - ml_base: ml_base as u16, - }); - } - - /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. - #[inline(always)] - pub(crate) fn store_seq( - &mut self, - lit_length: usize, - buf: &[u8], - literals: usize, - off_base: u32, - match_length: usize, - ) -> Result<()> { - match ( - buf.array_at::<16>(literals), - self.lit_buffer.array_at_mut::<16>(self.lit_len), - ) { - (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, - _ => self - .lit_buffer - .write_at(self.lit_len, buf.span(literals, lit_length)?)?, - } - self.lit_len += lit_length; - self.store_seq_only(lit_length, off_base, match_length); - Ok(()) - } - - /// Appends the literals that follow the last sequence of a block. - #[inline] - pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { - self.lit_buffer.write_at(self.lit_len, literals)?; - self.lit_len += literals.len(); - Ok(()) - } -} - -pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; -pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; -pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; - -/// `OFFSET_TO_OFFBASE`: `offset` is not 0. -#[inline(always)] -pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { - offset.wrapping_add(ZSTD_REP_NUM as u32) -} - -/// `OFFBASE_IS_OFFSET` -#[inline(always)] -pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { - off_base > ZSTD_REP_NUM as u32 -} - -/// `OFFBASE_IS_REPCODE` -#[inline(always)] -pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { - 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 -} - -/// `OFFBASE_TO_OFFSET` -#[inline(always)] -pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { - off_base.wrapping_sub(ZSTD_REP_NUM as u32) -} - -/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. -#[inline(always)] -pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { - let [rep0, rep1, rep2] = *rep; - if offbase_is_offset(off_base) { - *rep = [offbase_to_offset(off_base), rep0, rep1]; - } else { - match off_base - 1 + u32::from(ll0) { - 0 => {} - 1 => *rep = [rep1, rep0, rep2], - 2 => *rep = [rep2, rep0, rep1], - _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], - } - } -} - -/// `ZSTD_newRep` -#[inline(always)] -pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { - let mut new_reps = *rep; - update_rep(&mut new_reps, off_base, ll0); - new_reps -} - -/// `ZSTD_LLcode` -#[inline(always)] -pub(crate) fn ll_code(lit_length: u32) -> u32 { - #[rustfmt::skip] - const LL_CODE: [u8; 64] = [ - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, - ]; - const LL_DELTA_CODE: u32 = 19; - match LL_CODE.get(lit_length as usize) { - Some(&code) => u32::from(code), - None => highbit32(lit_length) + LL_DELTA_CODE, - } -} - -/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. -#[inline(always)] -pub(crate) fn ml_code(ml_base: u32) -> u32 { - #[rustfmt::skip] - const ML_CODE: [u8; 128] = [ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - ]; - const ML_DELTA_CODE: u32 = 36; - match ML_CODE.get(ml_base as usize) { - Some(&code) => u32::from(code), - None => highbit32(ml_base) + ML_DELTA_CODE, - } -} - -/// `ZSTD_noCompressBlock` -pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { - let c_block_header24 = - u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); - if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { - return Err(ErrorCode::DstSizeTooSmall); - } - write_le24(dst, 0, c_block_header24)?; - dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; - Ok(ZSTD_BLOCKHEADERSIZE + src.len()) -} - -/// `ZSTD_rleCompressBlock` -pub(crate) fn rle_compress_block( - dst: &mut [u8], - byte: u8, - src_size: usize, - last_block: bool, -) -> Result { - let c_block_header = - u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); - if dst.len() < 4 { - return Err(ErrorCode::DstSizeTooSmall); - } - write_le24(dst, 0, c_block_header)?; - dst.put(3, byte)?; - Ok(4) -} - -/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. -#[inline] -pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { - let minlog = if strat >= Strategy::BtUltra { - strat as u32 - 1 - } else { - 6 - }; - (src_size >> minlog) + 2 -} - -// ── matching ────────────────────────────────────────────────────────────── - -/// How many bytes `a` and `b` have in common at their start. -#[inline(always)] -pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { - let (a_words, _) = a.as_chunks::<8>(); - let (b_words, _) = b.as_chunks::<8>(); - let mut len = 0usize; - for (a_word, b_word) in a_words.iter().zip(b_words) { - let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); - if diff != 0 { - return len + (diff.trailing_zeros() >> 3) as usize; - } - len += 8; - } - let a_tail = a.get(len..).unwrap_or_default(); - let b_tail = b.get(len..).unwrap_or_default(); - len + a_tail - .iter() - .zip(b_tail) - .take_while(|(a_byte, b_byte)| a_byte == b_byte) - .count() -} - -/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. -#[inline(always)] -pub(crate) fn count( - buf: &[u8], - p_in: usize, - match_buf: &[u8], - p_match: usize, - p_in_limit: usize, -) -> Result { - Ok(common_prefix( - buf.sub(p_in..p_in_limit)?, - match_buf.sub(p_match..)?, - )) -} - -/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at -/// `i_start` in `buf`. -#[inline(always)] -pub(crate) fn count_2segments( - buf: &[u8], - ip: usize, - match_buf: &[u8], - match_pos: usize, - i_end: usize, - m_end: usize, - i_start: usize, -) -> Result { - let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); - if match_pos + match_length != m_end { - return Ok(match_length); - } - Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) -} - -const PRIME_3_BYTES: u32 = 506_832_829; -const PRIME_4_BYTES: u32 = 2_654_435_761; -const PRIME_5_BYTES: u64 = 889_523_592_379; -const PRIME_6_BYTES: u64 = 227_718_039_650_203; -const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; -pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; - -/// `ZSTD_hash3` -#[inline(always)] -const fn hash3(u: u32, h: u32, s: u32) -> u32 { - (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash4(u: u32, h: u32, s: u32) -> u32 { - ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash5(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash6(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash7(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -/// `ZSTD_hash8` -#[inline(always)] -pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { - (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} - -/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. -#[inline(always)] -pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { - Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) -} - -/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, -/// else 8. -#[inline(always)] -pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { - hash_ptr_salted(buf, pos, h_bits, mls, 0) -} - -/// `ZSTD_hashPtrSalted` -#[inline(always)] -pub(crate) fn hash_ptr_salted( - buf: &[u8], - pos: usize, - h_bits: u32, - mls: u32, - hash_salt: u64, -) -> Result { - Ok(match mls { - 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), - 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), - 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), - 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), - _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, - }) -} - -// ── the window ──────────────────────────────────────────────────────────── - -/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are -/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the -/// others, and an address is turned into a position in the buffer only to read. -pub(crate) type Addr = usize; - -/// The buffers that a window can have a segment in. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum BufId { - /// Where an empty window points. - None = 0, - /// The context's own buffer of input. - InBuff = 1, - /// What the caller gave to the call in progress. - User = 2, - /// The content of the dictionary, or the prefix. - Dict = 3, -} - -impl BufId { - /// The address of the first byte of the buffer. - #[inline(always)] - pub(crate) const fn start(self) -> Addr { - (self as usize + 1) << 44 - } -} - -/// The buffers themselves, for as long as a call lasts. -#[derive(Clone, Copy)] -pub(crate) struct Buffers<'a> { - pub(crate) in_buff: &'a [u8], - pub(crate) user: &'a [u8], - pub(crate) dict: &'a [u8], -} - -impl<'a> Buffers<'a> { - #[inline] - pub(crate) fn get(&self, id: BufId) -> &'a [u8] { - match id { - BufId::None => &[], - BufId::InBuff => self.in_buff, - BufId::User => self.user, - BufId::Dict => self.dict, - } - } -} - -/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. -pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; - -/// The index past which indices are brought down. -pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; -/// The most input that can be indexed at once. -pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; - -/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the -/// index is less than `dict_limit`. -#[derive(Clone, Copy, Debug)] -pub(crate) struct Window { - /// Where input has to continue to be contiguous. - pub(crate) next_src: Addr, - pub(crate) base: Addr, - pub(crate) dict_base: Addr, - /// The buffer of the segment that starts at `dict_limit`. - pub(crate) base_buf: BufId, - /// The buffer of the segment from `low_limit` to `dict_limit`. - pub(crate) dict_buf: BufId, - pub(crate) dict_limit: u32, - /// Below this, there is nothing valid. - pub(crate) low_limit: u32, - pub(crate) nb_overflow_corrections: u32, -} - -impl Window { - /// `ZSTD_window_init` - pub(crate) const fn new() -> Self { - let base = BufId::None.start(); - Self { - next_src: base + ZSTD_WINDOW_START_INDEX as usize, - base, - dict_base: base, - base_buf: BufId::None, - dict_buf: BufId::None, - dict_limit: ZSTD_WINDOW_START_INDEX, - low_limit: ZSTD_WINDOW_START_INDEX, - nb_overflow_corrections: 0, - } - } - - /// The index of `addr`, which is in the current segment or at its end. - #[inline(always)] - pub(crate) const fn index_of(&self, addr: Addr) -> u32 { - addr.wrapping_sub(self.base) as u32 - } - - /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. - pub(crate) fn clear(&mut self) { - let end = self.index_of(self.next_src); - self.low_limit = end; - self.dict_limit = end; - } - - /// `ZSTD_window_isEmpty` - pub(crate) fn is_empty(&self) -> bool { - self.dict_limit == ZSTD_WINDOW_START_INDEX - && self.low_limit == ZSTD_WINDOW_START_INDEX - && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize - } - - /// `ZSTD_window_hasExtDict` - #[inline] - pub(crate) fn has_ext_dict(&self) -> bool { - self.low_limit < self.dict_limit - } - - /// `ZSTD_window_needOverflowCorrection` - #[inline] - pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { - self.index_of(src_end) > ZSTD_CURRENT_MAX - } - - /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns - /// what to subtract from every index that is stored. - pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { - let cycle_size = 1u32 << cycle_log; - let cycle_mask = cycle_size - 1; - let curr = self.index_of(src); - let current_cycle = curr & cycle_mask; - // Indices must not fall below the start index. - let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { - cycle_size.max(ZSTD_WINDOW_START_INDEX) - } else { - 0 - }; - let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); - let correction = curr.wrapping_sub(new_current); - - self.base = self.base.wrapping_add(correction as usize); - self.dict_base = self.dict_base.wrapping_add(correction as usize); - self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { - ZSTD_WINDOW_START_INDEX - } else { - self.low_limit - correction - }; - self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { - ZSTD_WINDOW_START_INDEX - } else { - self.dict_limit - correction - }; - self.nb_overflow_corrections += 1; - correction - } - - /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the - /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. - pub(crate) fn enforce_max_dist( - &mut self, - block_end: Addr, - max_dist: u32, - loaded_dict_end: u32, - ) -> bool { - let block_end_idx = self.index_of(block_end); - if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { - let new_low_limit = block_end_idx - max_dist; - if self.low_limit < new_low_limit { - self.low_limit = new_low_limit; - } - if self.dict_limit < self.low_limit { - self.dict_limit = self.low_limit; - } - return true; - } - false - } - - /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not - /// right before the current segment, and is to be forgotten. - pub(crate) fn dict_is_invalid( - &self, - block_end: Addr, - max_dist: u32, - loaded_dict_end: u32, - ) -> bool { - let block_end_idx = self.index_of(block_end); - block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit - } - - /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not - /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. - pub(crate) fn update( - &mut self, - src: Addr, - src_buf: BufId, - src_size: usize, - force_non_contiguous: bool, - ) -> bool { - let mut contiguous = true; - if src_size == 0 { - return contiguous; - } - if src != self.next_src || force_non_contiguous { - let distance_from_base = self.next_src.wrapping_sub(self.base); - self.low_limit = self.dict_limit; - self.dict_limit = distance_from_base as u32; - self.dict_base = self.base; - self.dict_buf = self.base_buf; - self.base = src.wrapping_sub(distance_from_base); - self.base_buf = src_buf; - if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { - // Too small to be of use. - self.low_limit = self.dict_limit; - } - contiguous = false; - } - self.next_src = src + src_size; - // Where the input overlaps the older segment, it has overwritten it. - if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) - & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) - { - let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); - self.low_limit = if high_input_idx > self.dict_limit as usize { - self.dict_limit - } else { - high_input_idx as u32 - }; - } - contiguous - } -} - -/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes -/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. -#[inline(always)] -pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { - prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 -} - -pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; -pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; - -/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. -#[inline(always)] -pub(crate) fn write_tagged_index( - hash_table: &mut [u32], - hash_and_tag: usize, - index: u32, -) -> Result<()> { - let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; - hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) -} - -/// `ZSTD_comparePackedTags` -#[inline(always)] -pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { - (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) - == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) -} - -// ── the state of the match finders ──────────────────────────────────────── - -/// `ZSTD_match_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct Match { - /// As [`SeqDef::off_base`]. - pub(crate) off: u32, - pub(crate) len: u32, -} - -/// `rawSeq`: a sequence found by long distance matching. -#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] -pub(crate) struct RawSeq { - pub(crate) offset: u32, - pub(crate) lit_length: u32, - pub(crate) match_length: u32, -} - -/// `RawSeqStore_t` -#[derive(Clone, Default)] -pub(crate) struct RawSeqStore { - /// `capacity` of them, of which `size` are in use. - pub(crate) seq: Vec, - /// The sequence that is being read. - pub(crate) pos: usize, - /// How many of its bytes, literals first, are read. - pub(crate) pos_in_sequence: usize, - pub(crate) size: usize, - pub(crate) capacity: usize, -} - -/// `ZSTD_optimal_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct Optimal { - /// The cost of getting to this position. - pub(crate) price: i32, - pub(crate) off: u32, - pub(crate) mlen: u32, - pub(crate) litlen: u32, - pub(crate) rep: [u32; ZSTD_REP_NUM], -} - -/// `ZSTD_OptPrice_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum OptPrice { - Dynamic, - Predef, -} - -/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. -pub(crate) struct OptState { - pub(crate) lit_freq: Vec, - pub(crate) lit_length_freq: Vec, - pub(crate) match_length_freq: Vec, - pub(crate) off_code_freq: Vec, - pub(crate) match_table: Vec, - pub(crate) price_table: Vec, - pub(crate) lit_sum: u32, - pub(crate) lit_length_sum: u32, - pub(crate) match_length_sum: u32, - pub(crate) off_code_sum: u32, - pub(crate) lit_sum_base_price: u32, - pub(crate) lit_length_sum_base_price: u32, - pub(crate) match_length_sum_base_price: u32, - pub(crate) off_code_sum_base_price: u32, - pub(crate) price_type: OptPrice, - pub(crate) literal_compression_mode: ParamSwitch, -} - -impl OptState { - pub(crate) const fn new() -> Self { - Self { - lit_freq: Vec::new(), - lit_length_freq: Vec::new(), - match_length_freq: Vec::new(), - off_code_freq: Vec::new(), - match_table: Vec::new(), - price_table: Vec::new(), - lit_sum: 0, - lit_length_sum: 0, - match_length_sum: 0, - off_code_sum: 0, - lit_sum_base_price: 0, - lit_length_sum_base_price: 0, - match_length_sum_base_price: 0, - off_code_sum_base_price: 0, - price_type: OptPrice::Dynamic, - literal_compression_mode: ParamSwitch::Auto, - } - } -} - -pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; - -/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures -/// own, are in [`BlockSrc`]. -pub(crate) struct MatchState { - pub(crate) window: Window, - /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. - pub(crate) loaded_dict_end: u32, - /// The index from which the tables are still to be filled. - pub(crate) next_to_update: u32, - /// 0 if there is no table of 3-byte hashes. - pub(crate) hash_log3: u32, - pub(crate) row_hash_log: u32, - /// For the row-based match finder: a byte of hash for each entry of `hash_table`. - pub(crate) tag_table: Vec, - pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], - /// Makes what a previous frame left in `tag_table` meaningless. - pub(crate) hash_salt: u64, - pub(crate) hash_salt_entropy: u32, - pub(crate) hash_table: Vec, - pub(crate) hash_table3: Vec, - pub(crate) chain_table: Vec, - pub(crate) force_non_contiguous: bool, - pub(crate) opt: OptState, - /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. - pub(crate) has_dict_match_state: bool, - pub(crate) c_params: CompressionParameters, - pub(crate) lazy_skipping: bool, -} - -impl MatchState { - pub(crate) const fn new() -> Self { - Self { - window: Window::new(), - loaded_dict_end: 0, - next_to_update: 0, - hash_log3: 0, - row_hash_log: 0, - tag_table: Vec::new(), - hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], - hash_salt: 0, - hash_salt_entropy: 0, - hash_table: Vec::new(), - hash_table3: Vec::new(), - chain_table: Vec::new(), - force_non_contiguous: false, - opt: OptState::new(), - has_dict_match_state: false, - c_params: CompressionParameters { - window_log: 0, - chain_log: 0, - hash_log: 0, - search_log: 0, - min_match: 0, - target_length: 0, - strategy: None, - }, - lazy_skipping: false, - } - } - - /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. - pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { - if self.window.has_ext_dict() { - return DictMode::ExtDict; - } - if dms.is_some() { - DictMode::DictMatchState - } else { - DictMode::NoDict - } - } - - /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. - #[inline(always)] - pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { - let max_distance = 1u32 << window_log; - let lowest_valid = self.window.low_limit; - let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { - curr - max_distance - } else { - lowest_valid - }; - // A dictionary is valid as a whole until it is invalidated as a whole. - if self.loaded_dict_end != 0 { - lowest_valid - } else { - within_window - } - } - - /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. - #[inline(always)] - pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { - let max_distance = 1u32 << window_log; - let lowest_valid = self.window.dict_limit; - let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { - curr - max_distance - } else { - lowest_valid - }; - if self.loaded_dict_end != 0 { - lowest_valid - } else { - within_window - } - } -} - -/// The match state of a dictionary, and where its indices lead. -#[derive(Clone, Copy)] -pub(crate) struct DictMatchState<'a> { - pub(crate) ms: &'a MatchState, - /// The buffer its window is in, which is one segment. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, - /// `window.nextSrc`, as a position in `buf`. - pub(crate) end: usize, -} - -/// The current segment of a window, for the functions that index a dictionary. -#[derive(Clone, Copy)] -pub(crate) struct Prefix<'a> { - /// The buffer that the segment is in. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, -} - -impl Prefix<'_> { - /// The index of the position `pos` of `buf`. - #[inline(always)] - pub(crate) const fn index_of(&self, pos: usize) -> u32 { - pos.wrapping_sub(self.base) as u32 - } - - /// The position in `buf` of `index`. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } -} - -/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. -#[derive(Clone, Copy)] -pub(crate) struct BlockSrc<'a> { - /// The buffer that the current segment of the window is in, and so the block. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, - /// The buffer that the older segment of the window is in. - pub(crate) dict_buf: &'a [u8], - /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. - pub(crate) dict_base: usize, - /// Where the block starts in `buf`. - pub(crate) start: usize, - /// Where it ends. - pub(crate) end: usize, - pub(crate) dms: Option>, - /// `opt.symbolCosts`: the tables of the previous block. - pub(crate) symbol_costs: &'a EntropyCTables, - /// `ldmSeqStore`: matches that the optimal parser is to consider. - pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, -} - -impl<'a> BlockSrc<'a> { - #[inline(always)] - pub(crate) const fn prefix(&self) -> Prefix<'a> { - Prefix { - buf: self.buf, - base: self.base, - } - } - - /// The index of the position `pos` of `buf`. - #[inline(always)] - pub(crate) const fn index_of(&self, pos: usize) -> u32 { - pos.wrapping_sub(self.base) as u32 - } - - /// The position in `buf` of `index`, which is in the current segment. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } - - /// The position in `dict_buf` of `index`, which is in the older segment. - #[inline(always)] - pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { - self.dict_base.wrapping_add(index as usize) - } -} - -impl DictMatchState<'_> { - /// The position in `buf` of `index`. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } -} - -// ── long distance matching ──────────────────────────────────────────────── - -/// `ldmEntry_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct LdmEntry { - pub(crate) offset: u32, - pub(crate) checksum: u32, -} - -/// `ldmParams_t` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) struct LdmParams { - pub(crate) enable_ldm: ParamSwitch, - pub(crate) hash_log: u32, - pub(crate) bucket_size_log: u32, - pub(crate) min_match_length: u32, - pub(crate) hash_rate_log: u32, - pub(crate) window_log: u32, -} - -/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. -pub(crate) struct LdmState { - pub(crate) window: Window, - pub(crate) hash_table: Vec, - pub(crate) loaded_dict_end: u32, - /// The next entry to replace in each bucket. - pub(crate) bucket_offsets: Vec, -} - -impl LdmState { - pub(crate) const fn new() -> Self { - Self { - window: Window::new(), - hash_table: Vec::new(), - loaded_dict_end: 0, - bucket_offsets: Vec::new(), - } - } -} diff --git a/src/zstd/wip/work/lazy/zstd/compress/lazy.rs b/src/zstd/wip/work/lazy/zstd/compress/lazy.rs deleted file mode 100644 index 6ca0df1fe9fa..000000000000 --- a/src/zstd/wip/work/lazy/zstd/compress/lazy.rs +++ /dev/null @@ -1,1420 +0,0 @@ -//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. - -use core::hint::{Locality, prefetch_read}; - -use super::internal::{ - BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, - SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, - hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, - offset_to_offbase, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32}; - -const K_LAZY_SKIPPING_STEP: usize = 8; - -// `ZSTD_dictMode_e`, as an argument of a const generic. -const NO_DICT: u8 = DictMode::NoDict as u8; -const EXT_DICT: u8 = DictMode::ExtDict as u8; -const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; - -/// `PREFETCH_L1`: a hint that `slice[index]` is about to be read. `index` may be anything. -#[inline(always)] -fn prefetch_l1(slice: &[T], index: usize) { - prefetch_read(slice.as_ptr().wrapping_add(index), Locality::L1); -} - -/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. -#[inline(always)] -const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { - dms.end.wrapping_sub(dms.base) as u32 -} - -// ── binary tree search ──────────────────────────────────────────────────── - -/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. -#[inline(always)] -fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { - let hash_log = ms.c_params.hash_log; - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let bt = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); - *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; - } - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. -fn insert_dubt1( - ms: &mut MatchState, - src: &BlockSrc<'_>, - curr: u32, - input_end: usize, - mut nb_compares: u32, - bt_low: u32, -) -> Result<()> { - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let (ip_buf, ip, iend) = if curr >= dict_limit { - (src.buf, src.pos_of(curr), input_end) - } else { - (src.dict_buf, src.dict_pos_of(curr), dict_end) - }; - let window_valid = ms.window.low_limit; - let max_distance = 1u32 << ms.c_params.window_log; - let window_low = if curr.wrapping_sub(window_valid) > max_distance { - curr - max_distance - } else { - window_valid - }; - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - // The next sorted candidate. The other slot has the previous unsorted one. - let mut match_index = bt.at(smaller_ptr)?; - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let in_prefix = - DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; - // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = if in_prefix { - (src.buf, src.pos_of(match_index)) - } else { - (src.dict_buf, src.dict_pos_of(match_index)) - }; - - if in_prefix || curr < dict_limit { - // Both are in the same segment. - match_length += count( - ip_buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - match_length += count_2segments( - ip_buf, - ip + match_length, - match_buf, - match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if match_index as usize + match_length >= dict_limit as usize { - (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); - } - } - - if ip + match_length == iend { - // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. - break; - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0) -} - -/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. -fn dubt_find_better_dict_match( - ms: &MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mut best_length: usize, - mut nb_compares: u32, - mls: u32, -) -> Result { - let dms = src.dms.ok_or(BUG)?; - let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(h)?; - - let prefix_start = src.pos_of(ms.window.dict_limit); - let curr = src.index_of(ip); - let dict_high_limit = dms_end_index(&dms); - let dict_low_limit = dms.ms.window.low_limit; - let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); - - let dict_bt = dms.ms.chain_table.as_slice(); - let bt_log = dms.ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { - dict_low_limit - } else { - dict_high_limit - bt_mask - }; - - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - - while nb_compares != 0 && dict_match_index > dict_low_limit { - let next_ptr = 2 * (dict_match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let match_index = dict_match_index.wrapping_add(dict_index_delta); - match_length += count_2segments( - src.buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - iend, - dms.end, - prefix_start, - )?; - - if match_length > best_length { - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(offset_ptr.wrapping_add(1)) as i32 - { - best_length = match_length; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // No way to know which is smaller. - break; - } - } - - if dict_match_index <= bt_low { - // Beyond the tree. - break; - } - let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { - let match_pos = src.pos_of(match_index); - src.buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? - }; - if match_byte < src.buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dict_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dict_bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - Ok(best_length) -} - -/// `ZSTD_DUBT_findBestMatch` -#[inline(always)] -fn dubt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - off_base_ptr: &mut u32, - mls: u32, -) -> Result { - let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; - let mut match_index = ms.hash_table.at(h)?; - - let curr = src.index_of(ip); - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = curr.saturating_sub(bt_mask); - let unsort_limit = bt_low.max(window_low); - let mut nb_compares = 1u32 << ms.c_params.search_log; - let mut nb_candidates = nb_compares; - let mut previous_candidate = 0u32; - - // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. - let bt = ms.chain_table.as_mut_slice(); - let mut next_candidate = 2 * (match_index & bt_mask) as usize; - while match_index > unsort_limit - && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK - && nb_candidates > 1 - { - bt.put(next_candidate + 1, previous_candidate)?; - previous_candidate = match_index; - match_index = bt.at(next_candidate)?; - next_candidate = 2 * (match_index & bt_mask) as usize; - nb_candidates -= 1; - } - - // If the last candidate is still unsorted, cut the list there. - if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { - *bt.array_at_mut(next_candidate)? = [0, 0]; - } - - // Sort the candidates, the oldest first. - match_index = previous_candidate; - while match_index != 0 { - let next_candidate_idx = ms - .chain_table - .at(2 * (match_index & bt_mask) as usize + 1)?; - insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; - match_index = next_candidate_idx; - nb_candidates += 1; - } - - // Find the longest match. - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - let mut match_end_idx = curr + 8 + 1; - let mut best_length = 0usize; - - match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); - - if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - src.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - let dict_match_pos = src.dict_pos_of(match_index); - match_length += count_2segments( - src.buf, - ip + match_length, - src.dict_buf, - dict_match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - (match_buf, match_pos) = (src.dict_buf, dict_match_pos); - } - } - - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index + match_length as u32; - } - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(*off_base_ptr) as i32 - { - best_length = match_length; - *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // Equal: no way to know which is smaller. - if DICT_MODE == DICT_MATCH_STATE { - // Nothing in the dictionary can be longer. - nb_compares = 0; - } - break; - } - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0)?; - - if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { - best_length = dubt_find_better_dict_match( - ms, - src, - ip, - iend, - off_base_ptr, - best_length, - nb_compares, - mls, - )?; - } - - // Skip repetitive patterns. - ms.next_to_update = match_end_idx - 8; - Ok(best_length) -} - -/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. -/// `off_base_ptr` is written to if there is one. -#[inline(never)] -fn bt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - off_base_ptr: &mut u32, -) -> Result { - if ip < src.pos_of(ms.next_to_update) { - // In an area that is skipped. - return Ok(0); - } - update_dubt(ms, src.prefix(), ip, MLS)?; - dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) -} - -// ── hash chains ─────────────────────────────────────────────────────────── - -/// `ZSTD_insertAndFindFirstIndex_internal` -#[inline(always)] -fn insert_and_find_first_index_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - lazy_skipping: bool, -) -> Result { - let hash_log = ms.c_params.hash_log; - let chain_mask = (1u32 << ms.c_params.chain_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let chain_table = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); - chain_table.put((idx & chain_mask) as usize, next_in_chain)?; - // While skipping, not every position is inserted. - if lazy_skipping { - break; - } - } - - ms.next_to_update = target; - hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) -} - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) -} - -/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn hc_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let chain_size = 1u32 << ms.c_params.chain_log; - let chain_mask = chain_size - 1; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let min_chain = curr.saturating_sub(chain_size); - let mut nb_attempts = 1u32 << ms.c_params.search_log; - let mut ml = 4 - 1; - - let mut match_index = - insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; - let chain_table = ms.chain_table.as_slice(); - - while match_index >= low_limit && nb_attempts > 0 { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - - if match_index <= min_chain { - break; - } - match_index = chain_table.at((match_index & chain_mask) as usize)?; - nb_attempts -= 1; - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_chain_table = dms.ms.chain_table.as_slice(); - let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; - let dms_chain_mask = dms_chain_size - 1; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_size = dms_end_index(&dms); - let dms_index_delta = dict_limit.wrapping_sub(dms_size); - let dms_min_chain = dms_size.saturating_sub(dms_chain_size); - - let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; - match_index = dms.ms.hash_table.at(h)?; - - while match_index >= dms_lowest_index && nb_attempts > 0 { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - - if match_index <= dms_min_chain { - break; - } - match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; - nb_attempts -= 1; - } - } - - Ok(ml) -} - -// ── row-based match finder ──────────────────────────────────────────────── -// -// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits -// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag -// table is the "head": which entry was written last. - -pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; -const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; -const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; - -/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. -#[inline(always)] -fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { - let mut next = u32::from(*head).wrapping_sub(1) & row_mask; - if next == 0 { - next = row_mask; - } - *head = next as u8; - next -} - -/// `ZSTD_row_prefetch`: of the row that starts at `rel_row`. -#[inline(always)] -fn row_prefetch(hash_table: &[u32], tag_table: &[u8], rel_row: usize, row_log: u32) { - prefetch_l1(hash_table, rel_row); - if row_log >= 5 { - prefetch_l1(hash_table, rel_row + 16); - } - prefetch_l1(tag_table, rel_row); - if row_log == 6 { - prefetch_l1(tag_table, rel_row + 32); - } -} - -/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. -#[inline(always)] -fn row_fill_hash_cache( - ms: &mut MatchState, - prefix: Prefix<'_>, - row_log: u32, - mls: u32, - idx: u32, - i_limit: usize, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); - let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; - - for idx in idx..lim { - let hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - ms.hash_salt, - )? as u32; - let row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - row_prefetch(&ms.hash_table, &ms.tag_table, row, row_log); - let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; - ms.hash_cache.put(cached, hash)?; - } - Ok(()) -} - -/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes -/// further takes its place. -#[inline(always)] -fn row_next_cached_hash( - cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], - hash_table: &[u32], - tag_table: &[u8], - prefix: Prefix<'_>, - idx: u32, - hash_log: u32, - row_log: u32, - mls: u32, - hash_salt: u64, -) -> Result { - let new_hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32; - let row = ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - row_prefetch(hash_table, tag_table, row, row_log); - let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; - Ok(core::mem::replace(cached, new_hash)) -} - -/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. -#[inline(always)] -fn row_update_internal_impl( - ms: &mut MatchState, - prefix: Prefix<'_>, - update_start_idx: u32, - update_end_idx: u32, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let hash_salt = ms.hash_salt; - let hash_table = ms.hash_table.as_mut_slice(); - let tag_table = ms.tag_table.as_mut_slice(); - - for idx in update_start_idx..update_end_idx { - let hash = if use_cache { - row_next_cached_hash( - &mut ms.hash_cache, - hash_table, - tag_table, - prefix, - idx, - hash_log, - row_log, - mls, - hash_salt, - )? - } else { - hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32 - }; - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; - tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; - hash_table.put(rel_row + pos, idx)?; - } - Ok(()) -} - -/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is -/// when compressing, not all of them if they are many. -#[inline(always)] -fn row_update_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - const K_SKIP_THRESHOLD: u32 = 384; - const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; - const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; - - let mut idx = ms.next_to_update; - let target = prefix.index_of(ip); - - if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { - let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; - row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; - idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; - row_fill_hash_cache(ms, prefix, row_log, mls, idx, ip + 1)?; - } - row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the -/// row-based match finder. -pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { - let row_log = ms.c_params.search_log.clamp(4, 6); - let row_mask = (1u32 << row_log) - 1; - let mls = ms.c_params.min_match.min(6); - row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) -} - -/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. -#[inline(always)] -fn row_get_match_mask( - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - head: u32, -) -> u64 { - let mut matches = 0u64; - let (chunks, _) = tag_row.as_chunks::<16>(); - for (n, chunk) in chunks.iter().enumerate() { - let mut chunk_matches = 0u16; - let word = u128::from_le_bytes(*chunk); - for i in 0..16 { - chunk_matches |= u16::from((word >> (8 * i)) as u8 == tag) << i; - } - matches |= u64::from(chunk_matches) << (16 * n); - } - match ROW_ENTRIES { - 16 => u64::from((matches as u16).rotate_right(head)), - 32 => u64::from((matches as u32).rotate_right(head)), - _ => matches.rotate_right(head), - } -} - -/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, -/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than -/// `low_limit`. `prefetch` is called with each. Returns how many there are. -#[inline(always)] -fn row_collect_matches( - row: &[u32; ROW_ENTRIES], - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - low_limit: u32, - nb_attempts: &mut u32, - match_buffer: &mut [u32; ROW_ENTRIES], - prefetch: impl Fn(u32), -) -> Result { - let row_mask = ROW_ENTRIES as u32 - 1; - let head = u32::from(tag_row.at(0)?) & row_mask; - let mut num_matches = 0usize; - let mut matches = row_get_match_mask(tag_row, tag, head); - - while matches > 0 && *nb_attempts > 0 { - let match_pos = (head + matches.trailing_zeros()) & row_mask; - matches &= matches - 1; - if match_pos == 0 { - // The head, which happens to be the same as the tag. - continue; - } - let match_index = row.at(match_pos as usize)?; - if match_index < low_limit { - break; - } - prefetch(match_index); - match_buffer.put(num_matches, match_index)?; - num_matches += 1; - *nb_attempts -= 1; - } - Ok(num_matches) -} - -/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn row_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let hash_log = ms.row_hash_log; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let row_log = ROW_ENTRIES.trailing_zeros(); - let row_mask = ROW_ENTRIES as u32 - 1; - // There are no more searches than entries in a row. - let capped_search_log = ms.c_params.search_log.min(row_log); - let hash_salt = ms.hash_salt; - let mut nb_attempts = 1u32 << capped_search_log; - let mut ml = 4 - 1; - let mut match_buffer = [0u32; ROW_ENTRIES]; - - let mut dms_hash = 0u32; - let mut dms_rel_row = 0usize; - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; - dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; - dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - row_prefetch(&dms.ms.hash_table, &dms.ms.tag_table, dms_rel_row, row_log); - } - - // Update the tables up to, and not including, `ip`. - let hash = if ms.lazy_skipping { - // While skipping, not every position is inserted. - ms.next_to_update = curr; - hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 - } else { - let prefix = src.prefix(); - row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; - row_next_cached_hash( - &mut ms.hash_cache, - &ms.hash_table, - &ms.tag_table, - prefix, - curr, - hash_log, - row_log, - MLS, - hash_salt, - )? - }; - ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); - - { - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; - let row = ms.hash_table.array_at_mut::(rel_row)?; - let tag_row = ms.tag_table.array_at_mut::(rel_row)?; - let num_matches = row_collect_matches( - row, - tag_row, - tag, - low_limit, - &mut nb_attempts, - &mut match_buffer, - |match_index| { - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - prefetch_l1(buf, src.pos_of(match_index)); - } else { - prefetch_l1(src.dict_buf, src.dict_pos_of(match_index)); - } - }, - )?; - - // Insert `ip`. - let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; - tag_row.put(pos, tag)?; - row.put(pos, ms.next_to_update)?; - ms.next_to_update += 1; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - } - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); - - let num_matches = row_collect_matches( - dms.ms.hash_table.array_at::(dms_rel_row)?, - dms.ms.tag_table.array_at::(dms_rel_row)?, - (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, - dms_lowest_index, - &mut nb_attempts, - &mut match_buffer, - |match_index| prefetch_l1(dms.buf, dms.pos_of(match_index)), - )?; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - } - } - - Ok(ml) -} - -// ── the parsers ─────────────────────────────────────────────────────────── - -// `searchMethod_e`, as an argument of a const generic. -const SEARCH_HASH_CHAIN: u8 = 0; -const SEARCH_BINARY_TREE: u8 = 1; -const SEARCH_ROW_HASH: u8 = 2; - -/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. -#[inline(always)] -fn search_max( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mls: u32, - row_log: u32, -) -> Result { - match SEARCH_METHOD { - SEARCH_HASH_CHAIN => match mls { - 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - SEARCH_BINARY_TREE => match mls { - 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - _ => match (mls, row_log) { - (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - } -} - -/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match -/// state is searched. -#[derive(Clone, Copy)] -struct DictSegment<'a> { - buf: &'a [u8], - /// Its sum with an index of the window is a position in `buf`. It wraps around. - base: usize, - /// Where the segment starts in `buf`. - start: usize, - /// Where it ends. - end: usize, -} - -/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. -#[inline(always)] -fn rep_length( - ms: &MatchState, - src: &BlockSrc<'_>, - dict: &DictSegment<'_>, - ip: usize, - offset: u32, -) -> Result { - let buf = src.buf; - let iend = src.end; - if DICT_MODE == NO_DICT { - let rep_match = ip.wrapping_sub(offset as usize); - if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { - return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); - } - return Ok(0); - } - - let dict_limit = ms.window.dict_limit; - let curr = src.index_of(ip); - let rep_index = curr.wrapping_sub(offset); - if !index_overlap_check(dict_limit, rep_index) { - return Ok(0); - } - if DICT_MODE == EXT_DICT { - // The offset may lead to what is no longer in the window. - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - if offset > curr.wrapping_sub(window_low) { - return Ok(0); - } - } - let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { - let rep_match = dict.base.wrapping_add(rep_index as usize); - (dict.buf, rep_match, dict.end) - } else { - (buf, src.pos_of(rep_index), iend) - }; - if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { - return Ok(0); - } - let prefix_start = src.pos_of(dict_limit); - Ok(count_2segments( - buf, - ip + 4, - rep_buf, - rep_match + 4, - iend, - rep_end, - prefix_start, - )? + 4) -} - -/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions -/// further a better match is looked for, once there is one. -#[inline(never)] -fn compress_block_lazy_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let buf = src.buf; - let istart = src.start; - let mut ip = istart; - let mut anchor = istart; - let iend = src.end; - let prefix_lowest_index = ms.window.dict_limit; - let prefix_lowest = src.pos_of(prefix_lowest_index); - let mls = ms.c_params.min_match.clamp(4, 6); - let row_log = ms.c_params.search_log.clamp(4, 6); - - let [rep_0, rep_1, _] = rep; - let mut offset_1 = *rep_0; - let mut offset_2 = *rep_1; - let mut offset_saved_1 = 0u32; - let mut offset_saved_2 = 0u32; - - ms.lazy_skipping = false; - - let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { - 8 + ZSTD_ROW_HASH_CACHE_SIZE - } else { - 8 - }; - let Some(ilimit) = iend.checked_sub(margin) else { - // There is no position to search, nor to hash. - return Ok(iend - istart); - }; - - let dict = match DICT_MODE { - NO_DICT => DictSegment { - buf: &[], - base: 0, - start: 0, - end: 0, - }, - EXT_DICT => DictSegment { - buf: src.dict_buf, - base: src.dict_base, - start: src.dict_pos_of(ms.window.low_limit), - end: src.dict_pos_of(prefix_lowest_index), - }, - _ => { - let dms = src.dms.ok_or(BUG)?; - let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); - DictSegment { - buf: dms.buf, - base: dms.base.wrapping_sub(dict_index_delta as usize), - start: dms.pos_of(dms.ms.window.dict_limit), - end: dms.end, - } - } - }; - - if DICT_MODE == EXT_DICT { - ip += usize::from(ip == prefix_lowest); - } else { - let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); - ip += usize::from(dict_and_prefix_length == 0); - } - if DICT_MODE == NO_DICT { - let curr = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); - let max_rep = curr - window_low; - if offset_2 > max_rep { - offset_saved_2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved_1 = offset_1; - offset_1 = 0; - } - } - - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), row_log, mls, ms.next_to_update, ilimit)?; - } - - 'parse: while ip < ilimit { - let mut match_length; - let mut off_base = REPCODE1_TO_OFFBASE; - let mut start = ip + 1; - - 'found: { - // Check the repeat offset. - match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; - if DEPTH == 0 && match_length != 0 { - break 'found; - } - - // First search. - { - let mut off_base_found = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut off_base_found, - mls, - row_log, - )?; - if ml2 > match_length { - match_length = ml2; - start = ip; - off_base = off_base_found; - } - } - - if match_length < 4 { - // Jump faster over incompressible sections. - let step = (ip - anchor) >> K_SEARCH_STRENGTH; - ip += step + 1; - // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of - // no use out of the tables. As in the C code, it does not start at the same step in all modes. - ms.lazy_skipping = if DICT_MODE == EXT_DICT { - step > K_LAZY_SKIPPING_STEP - } else { - step + 1 > K_LAZY_SKIPPING_STEP - }; - continue 'parse; - } - - // Look for something better at the next positions. - if DEPTH >= 1 { - while ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 3) as i32; - let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - - // Nothing better: try one position further. - if DEPTH == 2 && ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 4) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - } - break; - } - } - - // Catch up: the match may start earlier. - if offbase_is_offset(off_base) { - let offset = offbase_to_offset(off_base); - let match_index = src.index_of(start).wrapping_sub(offset); - let (match_buf, mut match_pos, m_start) = - if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { - let match_pos = dict.base.wrapping_add(match_index as usize); - (dict.buf, match_pos, dict.start) - } else { - (buf, src.pos_of(match_index), prefix_lowest) - }; - while start > anchor - && match_pos > m_start - && buf.at(start - 1)? == match_buf.at(match_pos - 1)? - { - start -= 1; - match_pos -= 1; - match_length += 1; - } - offset_2 = offset_1; - offset_1 = offset; - } - } - - // Store the sequence. - seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; - ip = start + match_length; - anchor = ip; - - if ms.lazy_skipping { - // There was a match: every position is inserted again. - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), row_log, mls, ms.next_to_update, ilimit)?; - } - ms.lazy_skipping = false; - } - - // Check the second repeat offset right away. - while ip <= ilimit { - match_length = rep_length::(ms, src, &dict, ip, offset_2)?; - if match_length == 0 { - break; - } - (offset_1, offset_2) = (offset_2, offset_1); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; - ip += match_length; - anchor = ip; - } - } - - // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. - if offset_saved_1 != 0 && offset_1 != 0 { - offset_saved_2 = offset_saved_1; - } - - // Save the repeat offsets for the next block. - *rep_0 = if offset_1 != 0 { - offset_1 - } else { - offset_saved_1 - }; - *rep_1 = if offset_2 != 0 { - offset_2 - } else { - offset_saved_2 - }; - - // The size of the last literals. - Ok(iend - anchor) -} - -/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. -#[inline(always)] -fn compress_block_lazy_any_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - match dict_mode { - DictMode::NoDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::ExtDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - compress_block_lazy_generic::( - ms, seq_store, rep, src, - ) - } - } -} - -/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: -/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. -pub(crate) fn compress_block_lazy( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row: bool, - dict_mode: DictMode, -) -> Result { - match (strategy, use_row) { - (Strategy::Greedy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Greedy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( - ms, seq_store, rep, src, dict_mode, - ), - _ => Err(BUG), - } -} diff --git a/src/zstd/wip/work/lazy/zstd/compress/ldm.rs b/src/zstd/wip/work/lazy/zstd/compress/ldm.rs deleted file mode 100644 index 2177fd945e10..000000000000 --- a/src/zstd/wip/work/lazy/zstd/compress/ldm.rs +++ /dev/null @@ -1 +0,0 @@ -//! `zstd_ldm.c`: long distance matching. diff --git a/src/zstd/wip/work/lazy/zstd/compress/mod.rs b/src/zstd/wip/work/lazy/zstd/compress/mod.rs deleted file mode 100644 index 13ddecdf82e4..000000000000 --- a/src/zstd/wip/work/lazy/zstd/compress/mod.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! `lib/compress` - -pub(crate) mod double_fast; -pub(crate) mod fast; -pub(crate) mod fse; -pub(crate) mod hist; -pub(crate) mod huf; -pub(crate) mod internal; -pub(crate) mod lazy; -pub(crate) mod ldm; -pub(crate) mod opt; -pub(crate) mod presplit; - -use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; - -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; - -/// `ZSTD_rowMatchFinderSupported` -pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { - strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 -} - -/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to -/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies -/// that support it look at. Returns how many bytes at the end of the block are in no sequence. -pub(crate) fn compress_block( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row_match_finder: bool, - dict_mode: DictMode, -) -> Result { - match strategy { - Strategy::Fast => match dict_mode { - DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), - DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), - DictMode::DictMatchState => { - fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) - } - }, - Strategy::DFast => match dict_mode { - DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), - DictMode::ExtDict => { - double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) - } - }, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { - lazy::compress_block_lazy( - ms, - seq_store, - rep, - src, - strategy, - use_row_match_finder && row_match_finder_supported(strategy), - dict_mode, - ) - } - Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { - opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) - } - } -} diff --git a/src/zstd/wip/work/lazy/zstd/compress/opt.rs b/src/zstd/wip/work/lazy/zstd/compress/opt.rs deleted file mode 100644 index 584379ea28d3..000000000000 --- a/src/zstd/wip/work/lazy/zstd/compress/opt.rs +++ /dev/null @@ -1,96 +0,0 @@ -//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. - -use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. -/// Matches are not compared past the position `iend`. -pub(crate) fn update_tree( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - iend: usize, -) -> Result<()> { - // Test scaffolding only: noDict `ZSTD_updateTree_internal` + `ZSTD_insertBt1`. - use super::internal::{count, hash_ptr}; - use crate::util::SliceExt; - let mls = ms.c_params.min_match; - let target = prefix.index_of(ip); - let mut idx = ms.next_to_update; - while idx < target { - let ip = prefix.pos_of(idx); - let h = hash_ptr(prefix.buf, ip, ms.c_params.hash_log, mls)?; - let bt_mask = (1u32 << (ms.c_params.chain_log - 1)) - 1; - let mut match_index = ms.hash_table.at(h)?; - let (mut cls, mut cll) = (0usize, 0usize); - let curr = idx; - let bt_low = curr.saturating_sub(bt_mask); - let mut smaller = 2 * (curr & bt_mask) as usize; - let mut larger = smaller + 1; - let window_low = ms.get_lowest_match_index(target, ms.c_params.window_log); - let mut match_end_idx = curr + 8 + 1; - let mut best_length = 8usize; - let mut nb_compares = 1u32 << ms.c_params.search_log; - ms.hash_table.put(h, curr)?; - let bt = ms.chain_table.as_mut_slice(); - while nb_compares != 0 && match_index >= window_low { - let next = 2 * (match_index & bt_mask) as usize; - let mut ml = cls.min(cll); - let m = prefix.pos_of(match_index); - ml += count(prefix.buf, ip + ml, prefix.buf, m + ml, iend)?; - if ml > best_length { - best_length = ml; - if ml > (match_end_idx - match_index) as usize { - match_end_idx = match_index + ml as u32; - } - } - if ip + ml == iend { - break; - } - if prefix.buf.at(m + ml)? < prefix.buf.at(ip + ml)? { - bt.put(smaller, match_index)?; - cls = ml; - if match_index <= bt_low { - smaller = larger; - break; - } - smaller = next + 1; - match_index = bt.at(next + 1)?; - } else { - bt.put(larger, match_index)?; - cll = ml; - if match_index <= bt_low { - larger = smaller; - break; - } - larger = next; - match_index = bt.at(next)?; - } - nb_compares -= 1; - } - bt.put(smaller, 0)?; - bt.put(larger, 0)?; - let positions = if best_length > 384 { - 192.min(best_length as u32 - 384) - } else { - 0 - }; - idx += positions.max(match_end_idx - (curr + 8)); - } - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` -/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. -pub(crate) fn compress_block_opt( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, - _strategy: Strategy, - _dict_mode: DictMode, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/lazy/zstd/compress/presplit.rs b/src/zstd/wip/work/lazy/zstd/compress/presplit.rs deleted file mode 100644 index 8b778e2989f9..000000000000 --- a/src/zstd/wip/work/lazy/zstd/compress/presplit.rs +++ /dev/null @@ -1 +0,0 @@ -//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. diff --git a/src/zstd/wip/work/ldm/snap/double_fast.rs b/src/zstd/wip/work/ldm/snap/double_fast.rs deleted file mode 100644 index fbf2d4120cad..000000000000 --- a/src/zstd/wip/work/ldm/snap/double_fast.rs +++ /dev/null @@ -1,785 +0,0 @@ -//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. - -use super::fast::{FAST_HASH_FILL_STEP, Located, count_backwards, restore_rep}; -use super::internal::{ - BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, - compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, - write_tagged_index, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, read_le32, read_le64}; - -/// `ZSTD_fillDoubleHashTableForCDict` -fn fill_double_hash_table_for_cdict( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits_l = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let h_bits_s = ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; - let mls = ms.c_params.min_match; - let hash_large = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - // The positions in between only go in the table of long hashes, where there is nothing yet. - while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { - let curr = prefix.index_of(ip); - for i in 0..FAST_HASH_FILL_STEP { - let sm_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; - let lg_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; - if i == 0 { - write_tagged_index(hash_small, sm_hash_and_tag, curr + i)?; - } - if i == 0 || hash_large.at(lg_hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { - write_tagged_index(hash_large, lg_hash_and_tag, curr + i)?; - } - if dtlm == DictTableLoadMethod::Fast { - break; - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillDoubleHashTableForCCtx` -fn fill_double_hash_table_for_cctx( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let mls = ms.c_params.min_match; - let hash_large = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - // The positions in between only go in the table of long hashes, where there is nothing yet. - while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { - let curr = prefix.index_of(ip); - for i in 0..FAST_HASH_FILL_STEP { - let sm_hash = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; - let lg_hash = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; - if i == 0 { - hash_small.put(sm_hash, curr + i)?; - } - if i == 0 || hash_large.at(lg_hash)? == 0 { - hash_large.put(lg_hash, curr + i)?; - } - if dtlm == DictTableLoadMethod::Fast { - break; - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_double_hash_table( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, - tfp: TableFillPurpose, -) -> Result<()> { - match tfp { - TableFillPurpose::ForCDict => fill_double_hash_table_for_cdict(ms, prefix, end, dtlm), - TableFillPurpose::ForCCtx => fill_double_hash_table_for_cctx(ms, prefix, end, dtlm), - } -} - -/// `ZSTD_compressBlock_doubleFast_noDict_generic` -fn compress_block_double_fast_no_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; - - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let window_log = ms.c_params.window_log; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, window_log); - let prefix_lowest = src.pos_of(prefix_lowest_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip = istart + usize::from(istart == prefix_lowest); - let [mut offset_1, mut offset_2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let current = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(current, window_log); - let max_rep = current.wrapping_sub(window_low); - if offset_2 > max_rep { - offset_saved2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved1 = offset_1; - offset_1 = 0; - } - } - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - - // One turn per match. - 'start: loop { - let mut step = 1usize; - let mut next_step = ip + K_STEP_INCR; - let mut ip1 = ip + step; - if ip1 > ilimit { - break 'start; - } - - let mut hl0 = hash_ptr(buf, ip, h_bits_l, 8)?; - let mut idxl0 = hash_long.at(hl0)?; - let mut curr; - - let m_length = 'match_stored: { - // One turn per position. Ends with the offset and the length of a match, which `ip` is moved to the start - // of. - let (offset, m_length, hl1) = loop { - let hs0 = hash_ptr(buf, ip, h_bits_s, MLS)?; - let idxs0 = hash_small.at(hs0)?; - curr = src.index_of(ip); - hash_long.put(hl0, curr)?; - hash_small.put(hs0, curr)?; - - if (offset_1 > 0) - & (read_le32(buf, (ip + 1).wrapping_sub(offset_1 as usize))? - == read_le32(buf, ip + 1)?) - { - let m_length = count( - buf, - ip + 1 + 4, - buf, - (ip + 1 + 4).wrapping_sub(offset_1 as usize), - iend, - )? + 4; - ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - break 'match_stored m_length; - } - - let hl1 = hash_ptr(buf, ip1, h_bits_l, 8)?; - - let matchl0 = src.pos_of(idxl0); - if idxl0 >= prefix_lowest_index && read_le64(buf, matchl0)? == read_le64(buf, ip)? { - let m_length = count(buf, ip + 8, buf, matchl0 + 8, iend)? + 8; - let offset = ip.wrapping_sub(matchl0) as u32; - let back = count_backwards(buf, ip, anchor, buf, matchl0, prefix_lowest); - ip -= back; - break (offset, m_length + back, hl1); - } - - let idxl1 = hash_long.at(hl1)?; - - let mut matchs0 = src.pos_of(idxs0); - if idxs0 >= prefix_lowest_index && read_le32(buf, matchs0)? == read_le32(buf, ip)? { - // A short match: a longer one at the next position is preferred. - let mut m_length = count(buf, ip + 4, buf, matchs0 + 4, iend)? + 4; - let mut offset = ip.wrapping_sub(matchs0) as u32; - let matchl1 = src.pos_of(idxl1); - if idxl1 > prefix_lowest_index - && read_le64(buf, matchl1)? == read_le64(buf, ip1)? - { - let l1len = count(buf, ip1 + 8, buf, matchl1 + 8, iend)? + 8; - if l1len > m_length { - ip = ip1; - m_length = l1len; - offset = ip.wrapping_sub(matchl1) as u32; - matchs0 = matchl1; - } - } - let back = count_backwards(buf, ip, anchor, buf, matchs0, prefix_lowest); - ip -= back; - break (offset, m_length + back, hl1); - } - - if ip1 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - ip = ip1; - ip1 += step; - - hl0 = hl1; - idxl0 = idxl1; - if ip1 > ilimit { - break 'start; - } - }; - - offset_2 = offset_1; - offset_1 = offset; - // With a larger step, ip1 may be past the end of the match. - if step < 4 { - hash_long.put(hl1, src.index_of(ip1))?; - } - seq_store.store_seq( - ip - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - m_length - }; - - ip += m_length; - anchor = ip; - - if ip <= ilimit { - let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; - - while ip <= ilimit - && (offset_2 > 0) - & (read_le32(buf, ip)? == read_le32(buf, ip.wrapping_sub(offset_2 as usize))?) - { - let r_length = count( - buf, - ip + 4, - buf, - (ip + 4).wrapping_sub(offset_2 as usize), - iend, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, src.index_of(ip))?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, src.index_of(ip))?; - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; - ip += r_length; - anchor = ip; - } - } - } - - restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` -fn compress_block_double_fast_dict_match_state_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, ms.c_params.window_log); - let prefix_lowest = src.pos_of(prefix_lowest_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - let [mut offset_1, mut offset_2, _] = *rep; - - let dms = src.dms.as_ref().ok_or(BUG)?; - let dict_hash_long = dms.ms.hash_table.as_slice(); - let dict_hash_small = dms.ms.chain_table.as_slice(); - let dict_start_index = dms.ms.window.dict_limit; - let dict_start = dms.pos_of(dict_start_index); - let dict_end = dms.end; - // What to subtract from an index below prefix_lowest_index to get the index in the dictionary. - let dict_index_delta = prefix_lowest_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); - let dict_h_bits_l = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let dict_h_bits_s = dms.ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; - let dict_and_prefix_length = istart - .wrapping_sub(prefix_lowest) - .wrapping_add(dict_end.wrapping_sub(dict_start)) as u32; - - let locate = |index: u32| { - if index < prefix_lowest_index { - Located { - buf: dms.buf, - pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_lowest, - end: iend, - } - } - }; - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - - let mut anchor = istart; - let mut ip = istart + usize::from(dict_and_prefix_length == 0); - - 'search: while ip < ilimit { - let h2 = hash_ptr(buf, ip, h_bits_l, 8)?; - let h = hash_ptr(buf, ip, h_bits_s, MLS)?; - let dict_hash_and_tag_l = hash_ptr(buf, ip, dict_h_bits_l, 8)?; - let dict_hash_and_tag_s = hash_ptr(buf, ip, dict_h_bits_s, MLS)?; - let dict_match_index_and_tag_l = - dict_hash_long.at(dict_hash_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_match_index_and_tag_s = - dict_hash_small.at(dict_hash_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_tags_match_l = - compare_packed_tags(dict_match_index_and_tag_l as usize, dict_hash_and_tag_l); - let dict_tags_match_s = - compare_packed_tags(dict_match_index_and_tag_s as usize, dict_hash_and_tag_s); - let curr = src.index_of(ip); - let match_index_l = hash_long.at(h2)?; - let mut match_index_s = hash_small.at(h)?; - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); - hash_long.put(h2, curr)?; - hash_small.put(h, curr)?; - - let m_length = 'match_stored: { - if index_overlap_check(prefix_lowest_index, rep_index) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? - { - let m_length = count_2segments( - buf, - ip + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - iend, - rep_match.end, - prefix_lowest, - )? + 4; - ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - break 'match_stored m_length; - } - - // The offset and the length of a match, which `ip` is moved to the start of. - let (offset, m_length) = 'match_found: { - let match_long = src.pos_of(match_index_l); - if match_index_l >= prefix_lowest_index - && read_le64(buf, match_long)? == read_le64(buf, ip)? - { - let m_length = count(buf, ip + 8, buf, match_long + 8, iend)? + 8; - let offset = ip.wrapping_sub(match_long) as u32; - let back = count_backwards(buf, ip, anchor, buf, match_long, prefix_lowest); - ip -= back; - break 'match_found (offset, m_length + back); - } else if dict_tags_match_l { - let dict_match_index_l = - dict_match_index_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match_l = dms.pos_of(dict_match_index_l); - if dict_match_index_l > dict_start_index - && read_le64(dms.buf, dict_match_l)? == read_le64(buf, ip)? - { - let m_length = count_2segments( - buf, - ip + 8, - dms.buf, - dict_match_l + 8, - iend, - dict_end, - prefix_lowest, - )? + 8; - let offset = curr - .wrapping_sub(dict_match_index_l) - .wrapping_sub(dict_index_delta); - let back = - count_backwards(buf, ip, anchor, dms.buf, dict_match_l, dict_start); - ip -= back; - break 'match_found (offset, m_length + back); - } - } - - let short_found = if match_index_s > prefix_lowest_index { - read_le32(buf, src.pos_of(match_index_s))? == read_le32(buf, ip)? - } else if dict_tags_match_s { - let dict_match_index_s = - dict_match_index_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS; - match_index_s = dict_match_index_s.wrapping_add(dict_index_delta); - dict_match_index_s > dict_start_index - && read_le32(dms.buf, dms.pos_of(dict_match_index_s))? - == read_le32(buf, ip)? - } else { - false - }; - if !short_found { - ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; - continue 'search; - } - - // A short match: a long one at the next position is preferred. - let hl3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; - let dict_hash_and_tag_l3 = hash_ptr(buf, ip + 1, dict_h_bits_l, 8)?; - let match_index_l3 = hash_long.at(hl3)?; - let dict_match_index_and_tag_l3 = - dict_hash_long.at(dict_hash_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_tags_match_l3 = - compare_packed_tags(dict_match_index_and_tag_l3 as usize, dict_hash_and_tag_l3); - let match_l3 = src.pos_of(match_index_l3); - hash_long.put(hl3, curr + 1)?; - - if match_index_l3 >= prefix_lowest_index - && read_le64(buf, match_l3)? == read_le64(buf, ip + 1)? - { - let m_length = count(buf, ip + 9, buf, match_l3 + 8, iend)? + 8; - ip += 1; - let offset = ip.wrapping_sub(match_l3) as u32; - let back = count_backwards(buf, ip, anchor, buf, match_l3, prefix_lowest); - ip -= back; - break 'match_found (offset, m_length + back); - } else if dict_tags_match_l3 { - let dict_match_index_l3 = - dict_match_index_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match_l3 = dms.pos_of(dict_match_index_l3); - if dict_match_index_l3 > dict_start_index - && read_le64(dms.buf, dict_match_l3)? == read_le64(buf, ip + 1)? - { - let m_length = count_2segments( - buf, - ip + 1 + 8, - dms.buf, - dict_match_l3 + 8, - iend, - dict_end, - prefix_lowest, - )? + 8; - ip += 1; - let offset = curr - .wrapping_add(1) - .wrapping_sub(dict_match_index_l3) - .wrapping_sub(dict_index_delta); - let back = - count_backwards(buf, ip, anchor, dms.buf, dict_match_l3, dict_start); - ip -= back; - break 'match_found (offset, m_length + back); - } - } - - let match_s = locate(match_index_s); - let m_length = count_2segments( - buf, - ip + 4, - match_s.buf, - match_s.pos + 4, - iend, - match_s.end, - prefix_lowest, - )? + 4; - let offset = curr.wrapping_sub(match_index_s); - let back = - count_backwards(buf, ip, anchor, match_s.buf, match_s.pos, match_s.start); - ip -= back; - (offset, m_length + back) - }; - - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - m_length - }; - - ip += m_length; - anchor = ip; - - if ip <= ilimit { - let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; - - while ip <= ilimit { - let current2 = src.index_of(ip); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !(index_overlap_check(prefix_lowest_index, rep_index2) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_lowest, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; - ip += rep_length2; - anchor = ip; - } - } - } - - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_doubleFast` -pub(crate) fn compress_block_double_fast( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_double_fast_no_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_no_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_no_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_no_dict_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_doubleFast_dictMatchState` -pub(crate) fn compress_block_double_fast_dict_match_state( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_double_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_doubleFast_extDict_generic` -fn compress_block_double_fast_ext_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); - let dict_start_index = low_limit; - let dict_limit = ms.window.dict_limit; - let prefix_start_index = dict_limit.max(low_limit); - let prefix_start = src.pos_of(prefix_start_index); - let dict_start = src.dict_pos_of(dict_start_index); - let dict_end = src.dict_pos_of(prefix_start_index); - let [mut offset_1, mut offset_2, _] = *rep; - - // Nothing of the older segment is within reach. - if prefix_start_index == dict_start_index { - return compress_block_double_fast(ms, seq_store, rep, src); - } - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: src.dict_buf, - pos: src.dict_pos_of(index), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - - let mut anchor = istart; - let mut ip = istart; - - while ip < ilimit { - let h_small = hash_ptr(buf, ip, h_bits_s, MLS)?; - let match_index = hash_small.at(h_small)?; - let match_s = locate(match_index); - - let h_long = hash_ptr(buf, ip, h_bits_l, 8)?; - let match_long_index = hash_long.at(h_long)?; - let match_long = locate(match_long_index); - - let curr = src.index_of(ip); - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); - hash_long.put(h_long, curr)?; - hash_small.put(h_small, curr)?; - - let m_length; - if (index_overlap_check(prefix_start_index, rep_index) - & (offset_1 <= curr.wrapping_add(1).wrapping_sub(dict_start_index))) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? - { - m_length = count_2segments( - buf, - ip + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - iend, - rep_match.end, - prefix_start, - )? + 4; - ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - } else { - // The match, how much of it is known, and its offset. `ip` is moved to where it is for. - let (found, known, offset) = if match_long_index > dict_start_index - && read_le64(match_long.buf, match_long.pos)? == read_le64(buf, ip)? - { - (match_long, 8, curr.wrapping_sub(match_long_index)) - } else if match_index > dict_start_index - && read_le32(match_s.buf, match_s.pos)? == read_le32(buf, ip)? - { - // A short match: a long one at the next position is preferred. - let h3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; - let match_index3 = hash_long.at(h3)?; - let match3 = locate(match_index3); - hash_long.put(h3, curr + 1)?; - if match_index3 > dict_start_index - && read_le64(match3.buf, match3.pos)? == read_le64(buf, ip + 1)? - { - ip += 1; - (match3, 8, curr.wrapping_add(1).wrapping_sub(match_index3)) - } else { - (match_s, 4, curr.wrapping_sub(match_index)) - } - } else { - ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; - continue; - }; - let forwards = count_2segments( - buf, - ip + known, - found.buf, - found.pos + known, - iend, - found.end, - prefix_start, - )? + known; - let back = count_backwards(buf, ip, anchor, found.buf, found.pos, found.start); - ip -= back; - m_length = forwards + back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - } - - ip += m_length; - anchor = ip; - - if ip <= ilimit { - let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; - - while ip <= ilimit { - let current2 = src.index_of(ip); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !((index_overlap_check(prefix_start_index, rep_index2) - & (offset_2 <= current2.wrapping_sub(dict_start_index))) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_start, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; - ip += rep_length2; - anchor = ip; - } - } - } - - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_doubleFast_extDict` -pub(crate) fn compress_block_double_fast_ext_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_double_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), - } -} diff --git a/src/zstd/wip/work/ldm/snap/fast.rs b/src/zstd/wip/work/ldm/snap/fast.rs deleted file mode 100644 index 4bec128a6786..000000000000 --- a/src/zstd/wip/work/ldm/snap/fast.rs +++ /dev/null @@ -1,829 +0,0 @@ -//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. - -use super::internal::{ - BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, - compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, - write_tagged_index, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, read_le32}; - -pub(super) const FAST_HASH_FILL_STEP: u32 = 3; - -/// Where an index leads, when it can be in one of two segments. -#[derive(Clone, Copy)] -pub(super) struct Located<'a> { - /// The buffer that the segment is in. - pub(super) buf: &'a [u8], - /// The position of the index in `buf`. - pub(super) pos: usize, - /// Where the segment starts in `buf`. - pub(super) start: usize, - /// Where it ends. - pub(super) end: usize, -} - -/// How many bytes before `ip`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. -#[inline(always)] -pub(super) fn count_backwards( - buf: &[u8], - ip: usize, - anchor: usize, - match_buf: &[u8], - match_pos: usize, - match_low: usize, -) -> usize { - let before_ip = buf.get(anchor..ip).unwrap_or_default(); - let before_match = match_buf.get(match_low..match_pos).unwrap_or_default(); - before_ip - .iter() - .rev() - .zip(before_match.iter().rev()) - .take_while(|(a, b)| a == b) - .count() -} - -/// The repeat offsets to leave at the end of a block, at the start of which those that reached too far were saved and -/// set to 0. -#[inline(always)] -pub(super) fn restore_rep( - rep: &mut [u32; ZSTD_REP_NUM], - offset_1: u32, - offset_2: u32, - offset_saved1: u32, - offset_saved2: u32, -) { - // If offset_1 started invalid and became valid, offset_saved1 is what rotates into the second place. - let offset_saved2 = if offset_saved1 != 0 && offset_1 != 0 { - offset_saved1 - } else { - offset_saved2 - }; - let [rep0, rep1, _] = rep; - *rep0 = if offset_1 != 0 { - offset_1 - } else { - offset_saved1 - }; - *rep1 = if offset_2 != 0 { - offset_2 - } else { - offset_saved2 - }; -} - -/// `ZSTD_fillHashTableForCDict` -fn fill_hash_table_for_cdict( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let mls = ms.c_params.min_match; - let hash_table = ms.hash_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { - let curr = prefix.index_of(ip); - write_tagged_index(hash_table, hash_ptr(buf, ip, h_bits, mls)?, curr)?; - if dtlm == DictTableLoadMethod::Full { - // The positions in between only go where there is nothing yet. - for p in 1..FAST_HASH_FILL_STEP { - let hash_and_tag = hash_ptr(buf, ip + p as usize, h_bits, mls)?; - if hash_table.at(hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { - write_tagged_index(hash_table, hash_and_tag, curr + p)?; - } - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillHashTableForCCtx` -fn fill_hash_table_for_cctx( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits = ms.c_params.hash_log; - let mls = ms.c_params.min_match; - let hash_table = ms.hash_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { - let curr = prefix.index_of(ip); - hash_table.put(hash_ptr(buf, ip, h_bits, mls)?, curr)?; - if dtlm == DictTableLoadMethod::Full { - // The positions in between only go where there is nothing yet. - for p in 1..FAST_HASH_FILL_STEP { - let hash = hash_ptr(buf, ip + p as usize, h_bits, mls)?; - if hash_table.at(hash)? == 0 { - hash_table.put(hash, curr + p)?; - } - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_hash_table( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, - tfp: TableFillPurpose, -) -> Result<()> { - match tfp { - TableFillPurpose::ForCDict => fill_hash_table_for_cdict(ms, prefix, end, dtlm), - TableFillPurpose::ForCCtx => fill_hash_table_for_cctx(ms, prefix, end, dtlm), - } -} - -/// `ZSTD_match4Found_branch`, and `ZSTD_match4Found_cmov`, which only differs in how it avoids reading at an index that -/// is not valid. -#[inline(always)] -fn match4_found( - buf: &[u8], - current: usize, - match_pos: usize, - match_idx: u32, - idx_low_limit: u32, -) -> Result { - Ok(match_idx >= idx_low_limit && read_le32(buf, current)? == read_le32(buf, match_pos)?) -} - -/// `ZSTD_compressBlock_fast_noDict_generic` -/// -/// Each turn of the inner loop looks at two positions, `ip0` and `ip1`, for a match, and at `ip2` for a repeat. The -/// hash of a position is computed, and the table is read, one position ahead of the comparison. -fn compress_block_fast_no_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); - - let hlog = ms.c_params.hash_log; - let window_log = ms.c_params.window_log; - let target_length = ms.c_params.target_length; - // The least step is 2, as two positions are looked at per turn. - let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_start_index = ms.get_lowest_prefix_index(end_index, window_log); - let prefix_start = src.pos_of(prefix_start_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip0 = istart + usize::from(istart == prefix_start); - let [mut rep_offset1, mut rep_offset2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let curr = src.index_of(ip0); - let window_low = ms.get_lowest_prefix_index(curr, window_log); - let max_rep = curr.wrapping_sub(window_low); - if rep_offset2 > max_rep { - offset_saved2 = rep_offset2; - rep_offset2 = 0; - } - if rep_offset1 > max_rep { - offset_saved1 = rep_offset1; - rep_offset1 = 0; - } - } - let hash_table = ms.hash_table.as_mut_slice(); - - 'start: loop { - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - - let mut ip1 = ip0 + 1; - let mut ip2 = ip0 + step; - let mut ip3 = ip2 + 1; - if ip3 >= ilimit { - break 'start; - } - - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; - let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; - let mut match_idx = hash_table.at(hash0)?; - let mut current0; - - // Ends with the start and the length so far of a repeat match, or with `None` for a match at `match_idx`. - let rep_match = loop { - let rval = read_le32(buf, ip2.wrapping_sub(rep_offset1 as usize))?; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if (read_le32(buf, ip2)? == rval) & (rep_offset1 > 0) { - ip0 = ip2; - let match0 = ip0.wrapping_sub(rep_offset1 as usize); - let back = usize::from(buf.at(ip0 - 1)? == buf.at(match0.wrapping_sub(1))?); - ip0 -= back; - // The hash of ip1 is there, and ip1 is before the end of the match. - hash_table.put(hash1, src.index_of(ip1))?; - break Some((match0.wrapping_sub(back), 4 + back)); - } - - if match4_found( - buf, - ip0, - src.pos_of(match_idx), - match_idx, - prefix_start_index, - )? { - hash_table.put(hash1, src.index_of(ip1))?; - break None; - } - - match_idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip3; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if match4_found( - buf, - ip0, - src.pos_of(match_idx), - match_idx, - prefix_start_index, - )? { - // ip1 is certain to be before the end of the match only if the step is small. - if step <= 4 { - hash_table.put(hash1, src.index_of(ip1))?; - } - break None; - } - - match_idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip0 + step; - ip3 = ip1 + step; - - if ip2 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - if ip3 >= ilimit { - break 'start; - } - }; - - let (offcode, match0, mut m_length) = match rep_match { - Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), - None => { - let match0 = src.pos_of(match_idx); - rep_offset2 = rep_offset1; - rep_offset1 = ip0.wrapping_sub(match0) as u32; - let back = count_backwards(buf, ip0, anchor, buf, match0, prefix_start); - ip0 -= back; - (offset_to_offbase(rep_offset1), match0 - back, 4 + back) - } - }; - - m_length += count(buf, ip0 + m_length, buf, match0 + m_length, iend)?; - seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; - ip0 += m_length; - anchor = ip0; - - if ip0 <= ilimit { - hash_table.put( - hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, - current0 + 2, - )?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; - - if rep_offset2 > 0 { - while ip0 <= ilimit - && read_le32(buf, ip0)? - == read_le32(buf, ip0.wrapping_sub(rep_offset2 as usize))? - { - let r_length = count( - buf, - ip0 + 4, - buf, - (ip0 + 4).wrapping_sub(rep_offset2 as usize), - iend, - )? + 4; - core::mem::swap(&mut rep_offset1, &mut rep_offset2); - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; - ip0 += r_length; - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; - anchor = ip0; - } - } - } - } - - restore_rep(rep, rep_offset1, rep_offset2, offset_saved1, offset_saved2); - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_fast` -pub(crate) fn compress_block_fast( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_fast_no_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_no_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_no_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_no_dict_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_fast_dictMatchState_generic` -fn compress_block_fast_dict_match_state_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; - - let hlog = ms.c_params.hash_log; - let target_length = ms.c_params.target_length; - let step_size = (target_length + u32::from(target_length == 0)) as usize; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let prefix_start_index = ms.window.dict_limit; - let prefix_start = src.pos_of(prefix_start_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - let [mut offset_1, mut offset_2, _] = *rep; - - let dms = src.dms.as_ref().ok_or(BUG)?; - let dict_hash_table = dms.ms.hash_table.as_slice(); - let dict_start_index = dms.ms.window.dict_limit; - let dict_start = dms.pos_of(dict_start_index); - let dict_end = dms.end; - // What to subtract from an index below prefix_start_index to get the index in the dictionary. - let dict_index_delta = prefix_start_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); - let dict_and_prefix_length = istart - .wrapping_sub(prefix_start) - .wrapping_add(dict_end) - .wrapping_sub(dict_start) as u32; - let dict_h_bits = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: dms.buf, - pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; - let hash_table = ms.hash_table.as_mut_slice(); - - let mut anchor = istart; - let mut ip0 = istart + usize::from(dict_and_prefix_length == 0); - let mut ip1 = istart + step_size; - - 'search: while ip1 <= ilimit { - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; - - let dict_hash_and_tag0 = hash_ptr(buf, ip0, dict_h_bits, MLS)?; - let mut dict_match_index_and_tag = - dict_hash_table.at(dict_hash_and_tag0 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let mut dict_tags_match = - compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag0); - - let mut match_index = hash_table.at(hash0)?; - let mut curr = src.index_of(ip0); - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - - let m_length = loop { - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); - let hash1 = hash_ptr(buf, ip1, hlog, MLS)?; - let dict_hash_and_tag1 = hash_ptr(buf, ip1, dict_h_bits, MLS)?; - hash_table.put(hash0, curr)?; - - if index_overlap_check(prefix_start_index, rep_index) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip0 + 1)? - { - let m_length = count_2segments( - buf, - ip0 + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - iend, - rep_match.end, - prefix_start, - )? + 4; - ip0 += 1; - seq_store.store_seq(ip0 - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - break m_length; - } - - if dict_tags_match { - let dict_match_index = dict_match_index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match = dms.pos_of(dict_match_index); - // A match in the dictionary is only taken if there is no candidate in the prefix. - if dict_match_index > dict_start_index - && read_le32(dms.buf, dict_match)? == read_le32(buf, ip0)? - && match_index <= prefix_start_index - { - let offset = curr - .wrapping_sub(dict_match_index) - .wrapping_sub(dict_index_delta); - let mut m_length = count_2segments( - buf, - ip0 + 4, - dms.buf, - dict_match + 4, - iend, - dict_end, - prefix_start, - )? + 4; - let back = count_backwards(buf, ip0, anchor, dms.buf, dict_match, dict_start); - ip0 -= back; - m_length += back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip0 - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - break m_length; - } - } - - let match_pos = src.pos_of(match_index); - if match4_found(buf, ip0, match_pos, match_index, prefix_start_index)? { - let offset = ip0.wrapping_sub(match_pos) as u32; - let mut m_length = count(buf, ip0 + 4, buf, match_pos + 4, iend)? + 4; - let back = count_backwards(buf, ip0, anchor, buf, match_pos, prefix_start); - ip0 -= back; - m_length += back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip0 - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - break m_length; - } - - dict_match_index_and_tag = - dict_hash_table.at(dict_hash_and_tag1 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - dict_tags_match = - compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag1); - match_index = hash_table.at(hash1)?; - - if ip1 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - ip0 = ip1; - ip1 += step; - if ip1 > ilimit { - break 'search; - } - - curr = src.index_of(ip0); - hash0 = hash1; - }; - - ip0 += m_length; - anchor = ip0; - - if ip0 <= ilimit { - hash_table.put(hash_ptr(buf, src.pos_of(curr + 2), hlog, MLS)?, curr + 2)?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; - - while ip0 <= ilimit { - let current2 = src.index_of(ip0); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !(index_overlap_check(prefix_start_index, rep_index2) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip0 + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_start, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, current2)?; - ip0 += rep_length2; - anchor = ip0; - } - } - - ip1 = ip0 + step_size; - } - - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_fast_dictMatchState` -pub(crate) fn compress_block_fast_dict_match_state( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_fast_extDict_generic` -fn compress_block_fast_ext_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); - - let hlog = ms.c_params.hash_log; - let target_length = ms.c_params.target_length; - let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); - let dict_start_index = low_limit; - let dict_start = src.dict_pos_of(dict_start_index); - let dict_limit = ms.window.dict_limit; - let prefix_start_index = dict_limit.max(low_limit); - let prefix_start = src.pos_of(prefix_start_index); - let dict_end = src.dict_pos_of(prefix_start_index); - - // Nothing of the older segment is within reach. - if prefix_start_index == dict_start_index { - return compress_block_fast(ms, seq_store, rep, src); - } - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip0 = istart; - let [mut offset_1, mut offset_2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let curr = src.index_of(ip0); - let max_rep = curr.wrapping_sub(dict_start_index); - if offset_2 >= max_rep { - offset_saved2 = offset_2; - offset_2 = 0; - } - if offset_1 >= max_rep { - offset_saved1 = offset_1; - offset_1 = 0; - } - } - - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: src.dict_buf, - pos: src.dict_pos_of(index), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; - // Whether there is a match for the position `ip` at `idx`, which is from the table. - let match_found = |ip: usize, idx: u32| -> Result { - let candidate = locate(idx); - Ok(idx >= dict_start_index - && read_le32(candidate.buf, candidate.pos)? == read_le32(buf, ip)?) - }; - let hash_table = ms.hash_table.as_mut_slice(); - - 'start: loop { - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - - let mut ip1 = ip0 + 1; - let mut ip2 = ip0 + step; - let mut ip3 = ip2 + 1; - if ip3 >= ilimit { - break 'start; - } - - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; - let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; - let mut idx = hash_table.at(hash0)?; - let mut current0; - - // Ends with the start and the length so far of a repeat match, or with `None` for a match at `idx`. - let rep_match = loop { - let current2 = src.index_of(ip2); - let rep_index = current2.wrapping_sub(offset_1); - let mut rep_match = locate(rep_index); - // The 4 bytes have to be in one segment, with a byte before them. - let rep_found = ((prefix_start_index.wrapping_sub(rep_index) >= 4) & (offset_1 > 0)) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip2)?; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if rep_found { - ip0 = ip2; - let back = usize::from( - buf.at(ip0 - 1)? == rep_match.buf.at(rep_match.pos.wrapping_sub(1))?, - ); - ip0 -= back; - rep_match.pos = rep_match.pos.wrapping_sub(back); - break Some((rep_match, 4 + back)); - } - - if match_found(ip0, idx)? { - break None; - } - - idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip3; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if match_found(ip0, idx)? { - break None; - } - - idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip0 + step; - ip3 = ip1 + step; - - if ip2 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - if ip3 >= ilimit { - break 'start; - } - }; - - let (offcode, match0, mut m_length) = match rep_match { - Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), - None => { - let offset = current0.wrapping_sub(idx); - let mut match0 = locate(idx); - offset_2 = offset_1; - offset_1 = offset; - let back = count_backwards(buf, ip0, anchor, match0.buf, match0.pos, match0.start); - ip0 -= back; - match0.pos -= back; - (offset_to_offbase(offset), match0, 4 + back) - } - }; - - m_length += count_2segments( - buf, - ip0 + m_length, - match0.buf, - match0.pos + m_length, - iend, - match0.end, - prefix_start, - )?; - seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; - ip0 += m_length; - anchor = ip0; - - if ip1 < ip0 { - hash_table.put(hash1, src.index_of(ip1))?; - } - - if ip0 <= ilimit { - hash_table.put( - hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, - current0 + 2, - )?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; - - while ip0 <= ilimit { - let rep_index2 = src.index_of(ip0).wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !((index_overlap_check(prefix_start_index, rep_index2) & (offset_2 > 0)) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip0 + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_start, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; - ip0 += rep_length2; - anchor = ip0; - } - } - } - - restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_fast_extDict` -pub(crate) fn compress_block_fast_ext_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), - } -} diff --git a/src/zstd/wip/work/ldm/snap/lazy.rs b/src/zstd/wip/work/ldm/snap/lazy.rs deleted file mode 100644 index 94dbe308ce79..000000000000 --- a/src/zstd/wip/work/ldm/snap/lazy.rs +++ /dev/null @@ -1,1353 +0,0 @@ -//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. - -use super::internal::{ - BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, - SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, - hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, - offset_to_offbase, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32}; - -const K_LAZY_SKIPPING_STEP: usize = 8; - -// `ZSTD_dictMode_e`, as an argument of a const generic. -const NO_DICT: u8 = DictMode::NoDict as u8; -const EXT_DICT: u8 = DictMode::ExtDict as u8; -const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; - -/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. -#[inline(always)] -const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { - dms.end.wrapping_sub(dms.base) as u32 -} - -// ── binary tree search ──────────────────────────────────────────────────── - -/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. -#[inline(always)] -fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { - let hash_log = ms.c_params.hash_log; - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let bt = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); - *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; - } - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. -fn insert_dubt1( - ms: &mut MatchState, - src: &BlockSrc<'_>, - curr: u32, - input_end: usize, - mut nb_compares: u32, - bt_low: u32, -) -> Result<()> { - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let (ip_buf, ip, iend) = if curr >= dict_limit { - (src.buf, src.pos_of(curr), input_end) - } else { - (src.dict_buf, src.dict_pos_of(curr), dict_end) - }; - let window_valid = ms.window.low_limit; - let max_distance = 1u32 << ms.c_params.window_log; - let window_low = if curr.wrapping_sub(window_valid) > max_distance { - curr - max_distance - } else { - window_valid - }; - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - // The next sorted candidate. The other slot has the previous unsorted one. - let mut match_index = bt.at(smaller_ptr)?; - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let in_prefix = - DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; - // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = if in_prefix { - (src.buf, src.pos_of(match_index)) - } else { - (src.dict_buf, src.dict_pos_of(match_index)) - }; - - if in_prefix || curr < dict_limit { - // Both are in the same segment. - match_length += count( - ip_buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - match_length += count_2segments( - ip_buf, - ip + match_length, - match_buf, - match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if match_index as usize + match_length >= dict_limit as usize { - (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); - } - } - - if ip + match_length == iend { - // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. - break; - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0) -} - -/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. -fn dubt_find_better_dict_match( - ms: &MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mut best_length: usize, - mut nb_compares: u32, - mls: u32, -) -> Result { - let dms = src.dms.ok_or(BUG)?; - let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(h)?; - - let prefix_start = src.pos_of(ms.window.dict_limit); - let curr = src.index_of(ip); - let dict_high_limit = dms_end_index(&dms); - let dict_low_limit = dms.ms.window.low_limit; - let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); - - let dict_bt = dms.ms.chain_table.as_slice(); - let bt_log = dms.ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { - dict_low_limit - } else { - dict_high_limit - bt_mask - }; - - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - - while nb_compares != 0 && dict_match_index > dict_low_limit { - let next_ptr = 2 * (dict_match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let match_index = dict_match_index.wrapping_add(dict_index_delta); - match_length += count_2segments( - src.buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - iend, - dms.end, - prefix_start, - )?; - - if match_length > best_length { - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(offset_ptr.wrapping_add(1)) as i32 - { - best_length = match_length; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // No way to know which is smaller. - break; - } - } - - if dict_match_index <= bt_low { - // Beyond the tree. - break; - } - let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { - let match_pos = src.pos_of(match_index); - src.buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? - }; - if match_byte < src.buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dict_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dict_bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - Ok(best_length) -} - -/// `ZSTD_DUBT_findBestMatch` -#[inline(always)] -fn dubt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - off_base_ptr: &mut u32, - mls: u32, -) -> Result { - let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; - let mut match_index = ms.hash_table.at(h)?; - - let curr = src.index_of(ip); - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = curr.saturating_sub(bt_mask); - let unsort_limit = bt_low.max(window_low); - let mut nb_compares = 1u32 << ms.c_params.search_log; - let mut nb_candidates = nb_compares; - let mut previous_candidate = 0u32; - - // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. - let bt = ms.chain_table.as_mut_slice(); - let mut next_candidate = 2 * (match_index & bt_mask) as usize; - while match_index > unsort_limit - && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK - && nb_candidates > 1 - { - bt.put(next_candidate + 1, previous_candidate)?; - previous_candidate = match_index; - match_index = bt.at(next_candidate)?; - next_candidate = 2 * (match_index & bt_mask) as usize; - nb_candidates -= 1; - } - - // If the last candidate is still unsorted, cut the list there. - if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { - *bt.array_at_mut(next_candidate)? = [0, 0]; - } - - // Sort the candidates, the oldest first. - match_index = previous_candidate; - while match_index != 0 { - let next_candidate_idx = ms - .chain_table - .at(2 * (match_index & bt_mask) as usize + 1)?; - insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; - match_index = next_candidate_idx; - nb_candidates += 1; - } - - // Find the longest match. - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - let mut match_end_idx = curr + 8 + 1; - let mut best_length = 0usize; - - match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); - - if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - src.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - let dict_match_pos = src.dict_pos_of(match_index); - match_length += count_2segments( - src.buf, - ip + match_length, - src.dict_buf, - dict_match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - (match_buf, match_pos) = (src.dict_buf, dict_match_pos); - } - } - - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index + match_length as u32; - } - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(*off_base_ptr) as i32 - { - best_length = match_length; - *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // Equal: no way to know which is smaller. - if DICT_MODE == DICT_MATCH_STATE { - // Nothing in the dictionary can be longer. - nb_compares = 0; - } - break; - } - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0)?; - - if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { - best_length = dubt_find_better_dict_match( - ms, - src, - ip, - iend, - off_base_ptr, - best_length, - nb_compares, - mls, - )?; - } - - // Skip repetitive patterns. - ms.next_to_update = match_end_idx - 8; - Ok(best_length) -} - -/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. -/// `off_base_ptr` is written to if there is one. -#[inline(never)] -fn bt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - off_base_ptr: &mut u32, -) -> Result { - if ip < src.pos_of(ms.next_to_update) { - // In an area that is skipped. - return Ok(0); - } - update_dubt(ms, src.prefix(), ip, MLS)?; - dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) -} - -// ── hash chains ─────────────────────────────────────────────────────────── - -/// `ZSTD_insertAndFindFirstIndex_internal` -#[inline(always)] -fn insert_and_find_first_index_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - lazy_skipping: bool, -) -> Result { - let hash_log = ms.c_params.hash_log; - let chain_mask = (1u32 << ms.c_params.chain_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let chain_table = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); - chain_table.put((idx & chain_mask) as usize, next_in_chain)?; - // While skipping, not every position is inserted. - if lazy_skipping { - break; - } - } - - ms.next_to_update = target; - hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) -} - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) -} - -/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn hc_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let chain_size = 1u32 << ms.c_params.chain_log; - let chain_mask = chain_size - 1; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let min_chain = curr.saturating_sub(chain_size); - let mut nb_attempts = 1u32 << ms.c_params.search_log; - let mut ml = 4 - 1; - - let mut match_index = - insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; - let chain_table = ms.chain_table.as_slice(); - - while match_index >= low_limit && nb_attempts > 0 { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - - if match_index <= min_chain { - break; - } - match_index = chain_table.at((match_index & chain_mask) as usize)?; - nb_attempts -= 1; - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_chain_table = dms.ms.chain_table.as_slice(); - let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; - let dms_chain_mask = dms_chain_size - 1; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_size = dms_end_index(&dms); - let dms_index_delta = dict_limit.wrapping_sub(dms_size); - let dms_min_chain = dms_size.saturating_sub(dms_chain_size); - - let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; - match_index = dms.ms.hash_table.at(h)?; - - while match_index >= dms_lowest_index && nb_attempts > 0 { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - - if match_index <= dms_min_chain { - break; - } - match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; - nb_attempts -= 1; - } - } - - Ok(ml) -} - -// ── row-based match finder ──────────────────────────────────────────────── -// -// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits -// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag -// table is the "head": which entry was written last. - -pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; -const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; -const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; - -/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. -#[inline(always)] -fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { - let mut next = u32::from(*head).wrapping_sub(1) & row_mask; - if next == 0 { - next = row_mask; - } - *head = next as u8; - next -} - -/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. -#[inline(always)] -fn row_fill_hash_cache( - ms: &mut MatchState, - prefix: Prefix<'_>, - mls: u32, - idx: u32, - i_limit: usize, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); - let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; - - for idx in idx..lim { - let hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - ms.hash_salt, - )? as u32; - let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; - ms.hash_cache.put(cached, hash)?; - } - Ok(()) -} - -/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes -/// further takes its place. -#[inline(always)] -fn row_next_cached_hash( - cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], - prefix: Prefix<'_>, - idx: u32, - hash_log: u32, - mls: u32, - hash_salt: u64, -) -> Result { - let new_hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32; - let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; - Ok(core::mem::replace(cached, new_hash)) -} - -/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. -#[inline(always)] -fn row_update_internal_impl( - ms: &mut MatchState, - prefix: Prefix<'_>, - update_start_idx: u32, - update_end_idx: u32, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let hash_salt = ms.hash_salt; - let hash_table = ms.hash_table.as_mut_slice(); - let tag_table = ms.tag_table.as_mut_slice(); - - for idx in update_start_idx..update_end_idx { - let hash = if use_cache { - row_next_cached_hash(&mut ms.hash_cache, prefix, idx, hash_log, mls, hash_salt)? - } else { - hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32 - }; - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; - tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; - hash_table.put(rel_row + pos, idx)?; - } - Ok(()) -} - -/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is -/// when compressing, not all of them if they are many. -#[inline(always)] -fn row_update_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - const K_SKIP_THRESHOLD: u32 = 384; - const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; - const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; - - let mut idx = ms.next_to_update; - let target = prefix.index_of(ip); - - if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { - let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; - row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; - idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; - row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; - } - row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the -/// row-based match finder. -pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { - let row_log = ms.c_params.search_log.clamp(4, 6); - let row_mask = (1u32 << row_log) - 1; - let mls = ms.c_params.min_match.min(6); - row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) -} - -/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. -#[inline(always)] -fn row_get_match_mask( - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - head: u32, -) -> u64 { - let mut matches = 0u64; - let (chunks, _) = tag_row.as_chunks::<16>(); - for (n, chunk) in chunks.iter().enumerate() { - let mut chunk_matches = 0u16; - for (i, &entry) in chunk.iter().enumerate() { - chunk_matches |= u16::from(entry == tag) << i; - } - matches |= u64::from(chunk_matches) << (16 * n); - } - match ROW_ENTRIES { - 16 => u64::from((matches as u16).rotate_right(head)), - 32 => u64::from((matches as u32).rotate_right(head)), - _ => matches.rotate_right(head), - } -} - -/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, -/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than -/// `low_limit`. Returns how many there are. -#[inline(always)] -fn row_collect_matches( - row: &[u32; ROW_ENTRIES], - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - low_limit: u32, - nb_attempts: &mut u32, - match_buffer: &mut [u32; ROW_ENTRIES], -) -> Result { - let row_mask = ROW_ENTRIES as u32 - 1; - let head = u32::from(tag_row.at(0)?) & row_mask; - let mut num_matches = 0usize; - let mut matches = row_get_match_mask(tag_row, tag, head); - - while matches > 0 && *nb_attempts > 0 { - let match_pos = (head + matches.trailing_zeros()) & row_mask; - matches &= matches - 1; - if match_pos == 0 { - // The head, which happens to be the same as the tag. - continue; - } - let match_index = row.at(match_pos as usize)?; - if match_index < low_limit { - break; - } - match_buffer.put(num_matches, match_index)?; - num_matches += 1; - *nb_attempts -= 1; - } - Ok(num_matches) -} - -/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn row_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let hash_log = ms.row_hash_log; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let row_log = ROW_ENTRIES.trailing_zeros(); - let row_mask = ROW_ENTRIES as u32 - 1; - // There are no more searches than entries in a row. - let capped_search_log = ms.c_params.search_log.min(row_log); - let hash_salt = ms.hash_salt; - let mut nb_attempts = 1u32 << capped_search_log; - let mut ml = 4 - 1; - let mut match_buffer = [0u32; ROW_ENTRIES]; - - // Update the tables up to, and not including, `ip`. - let hash = if ms.lazy_skipping { - // While skipping, not every position is inserted. - ms.next_to_update = curr; - hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 - } else { - let prefix = src.prefix(); - row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; - row_next_cached_hash(&mut ms.hash_cache, prefix, curr, hash_log, MLS, hash_salt)? - }; - ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); - - { - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; - let row = ms.hash_table.array_at_mut::(rel_row)?; - let tag_row = ms.tag_table.array_at_mut::(rel_row)?; - let num_matches = row_collect_matches( - row, - tag_row, - tag, - low_limit, - &mut nb_attempts, - &mut match_buffer, - )?; - - // Insert `ip`. - let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; - tag_row.put(pos, tag)?; - row.put(pos, ms.next_to_update)?; - ms.next_to_update += 1; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - } - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); - - let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; - let dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; - let dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let num_matches = row_collect_matches( - dms.ms.hash_table.array_at::(dms_rel_row)?, - dms.ms.tag_table.array_at::(dms_rel_row)?, - (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, - dms_lowest_index, - &mut nb_attempts, - &mut match_buffer, - )?; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - } - } - - Ok(ml) -} - -// ── the parsers ─────────────────────────────────────────────────────────── - -// `searchMethod_e`, as an argument of a const generic. -const SEARCH_HASH_CHAIN: u8 = 0; -const SEARCH_BINARY_TREE: u8 = 1; -const SEARCH_ROW_HASH: u8 = 2; - -/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. -#[inline(always)] -fn search_max( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mls: u32, - row_log: u32, -) -> Result { - match SEARCH_METHOD { - SEARCH_HASH_CHAIN => match mls { - 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - SEARCH_BINARY_TREE => match mls { - 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - _ => match (mls, row_log) { - (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - } -} - -/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match -/// state is searched. -#[derive(Clone, Copy)] -struct DictSegment<'a> { - buf: &'a [u8], - /// Its sum with an index of the window is a position in `buf`. It wraps around. - base: usize, - /// Where the segment starts in `buf`. - start: usize, - /// Where it ends. - end: usize, -} - -/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. -#[inline(always)] -fn rep_length( - ms: &MatchState, - src: &BlockSrc<'_>, - dict: &DictSegment<'_>, - ip: usize, - offset: u32, -) -> Result { - let buf = src.buf; - let iend = src.end; - if DICT_MODE == NO_DICT { - let rep_match = ip.wrapping_sub(offset as usize); - if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { - return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); - } - return Ok(0); - } - - let dict_limit = ms.window.dict_limit; - let curr = src.index_of(ip); - let rep_index = curr.wrapping_sub(offset); - if !index_overlap_check(dict_limit, rep_index) { - return Ok(0); - } - if DICT_MODE == EXT_DICT { - // The offset may lead to what is no longer in the window. - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - if offset > curr.wrapping_sub(window_low) { - return Ok(0); - } - } - let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { - let rep_match = dict.base.wrapping_add(rep_index as usize); - (dict.buf, rep_match, dict.end) - } else { - (buf, src.pos_of(rep_index), iend) - }; - if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { - return Ok(0); - } - let prefix_start = src.pos_of(dict_limit); - Ok(count_2segments( - buf, - ip + 4, - rep_buf, - rep_match + 4, - iend, - rep_end, - prefix_start, - )? + 4) -} - -/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions -/// further a better match is looked for, once there is one. -#[inline(never)] -fn compress_block_lazy_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let buf = src.buf; - let istart = src.start; - let mut ip = istart; - let mut anchor = istart; - let iend = src.end; - let prefix_lowest_index = ms.window.dict_limit; - let prefix_lowest = src.pos_of(prefix_lowest_index); - let mls = ms.c_params.min_match.clamp(4, 6); - let row_log = ms.c_params.search_log.clamp(4, 6); - - let [rep_0, rep_1, _] = rep; - let mut offset_1 = *rep_0; - let mut offset_2 = *rep_1; - let mut offset_saved_1 = 0u32; - let mut offset_saved_2 = 0u32; - - ms.lazy_skipping = false; - - let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { - 8 + ZSTD_ROW_HASH_CACHE_SIZE - } else { - 8 - }; - let Some(ilimit) = iend.checked_sub(margin) else { - // There is no position to search, nor to hash. - return Ok(iend - istart); - }; - - let dict = match DICT_MODE { - NO_DICT => DictSegment { - buf: &[], - base: 0, - start: 0, - end: 0, - }, - EXT_DICT => DictSegment { - buf: src.dict_buf, - base: src.dict_base, - start: src.dict_pos_of(ms.window.low_limit), - end: src.dict_pos_of(prefix_lowest_index), - }, - _ => { - let dms = src.dms.ok_or(BUG)?; - let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); - DictSegment { - buf: dms.buf, - base: dms.base.wrapping_sub(dict_index_delta as usize), - start: dms.pos_of(dms.ms.window.dict_limit), - end: dms.end, - } - } - }; - - if DICT_MODE == EXT_DICT { - ip += usize::from(ip == prefix_lowest); - } else { - let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); - ip += usize::from(dict_and_prefix_length == 0); - } - if DICT_MODE == NO_DICT { - let curr = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); - let max_rep = curr - window_low; - if offset_2 > max_rep { - offset_saved_2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved_1 = offset_1; - offset_1 = 0; - } - } - - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; - } - - 'parse: while ip < ilimit { - let mut match_length; - let mut off_base = REPCODE1_TO_OFFBASE; - let mut start = ip + 1; - - 'found: { - // Check the repeat offset. - match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; - if DEPTH == 0 && match_length != 0 { - break 'found; - } - - // First search. - { - let mut off_base_found = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut off_base_found, - mls, - row_log, - )?; - if ml2 > match_length { - match_length = ml2; - start = ip; - off_base = off_base_found; - } - } - - if match_length < 4 { - // Jump faster over incompressible sections. - let step = (ip - anchor) >> K_SEARCH_STRENGTH; - ip += step + 1; - // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of - // no use out of the tables. As in the C code, it does not start at the same step in all modes. - ms.lazy_skipping = if DICT_MODE == EXT_DICT { - step > K_LAZY_SKIPPING_STEP - } else { - step + 1 > K_LAZY_SKIPPING_STEP - }; - continue 'parse; - } - - // Look for something better at the next positions. - if DEPTH >= 1 { - while ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 3) as i32; - let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - - // Nothing better: try one position further. - if DEPTH == 2 && ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 4) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - } - break; - } - } - - // Catch up: the match may start earlier. - if offbase_is_offset(off_base) { - let offset = offbase_to_offset(off_base); - let match_index = src.index_of(start).wrapping_sub(offset); - let (match_buf, mut match_pos, m_start) = - if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { - let match_pos = dict.base.wrapping_add(match_index as usize); - (dict.buf, match_pos, dict.start) - } else { - (buf, src.pos_of(match_index), prefix_lowest) - }; - while start > anchor - && match_pos > m_start - && buf.at(start - 1)? == match_buf.at(match_pos - 1)? - { - start -= 1; - match_pos -= 1; - match_length += 1; - } - offset_2 = offset_1; - offset_1 = offset; - } - } - - // Store the sequence. - seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; - ip = start + match_length; - anchor = ip; - - if ms.lazy_skipping { - // There was a match: every position is inserted again. - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; - } - ms.lazy_skipping = false; - } - - // Check the second repeat offset right away. - while ip <= ilimit { - match_length = rep_length::(ms, src, &dict, ip, offset_2)?; - if match_length == 0 { - break; - } - (offset_1, offset_2) = (offset_2, offset_1); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; - ip += match_length; - anchor = ip; - } - } - - // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. - if offset_saved_1 != 0 && offset_1 != 0 { - offset_saved_2 = offset_saved_1; - } - - // Save the repeat offsets for the next block. - *rep_0 = if offset_1 != 0 { - offset_1 - } else { - offset_saved_1 - }; - *rep_1 = if offset_2 != 0 { - offset_2 - } else { - offset_saved_2 - }; - - // The size of the last literals. - Ok(iend - anchor) -} - -/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. -#[inline(always)] -fn compress_block_lazy_any_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - match dict_mode { - DictMode::NoDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::ExtDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - compress_block_lazy_generic::( - ms, seq_store, rep, src, - ) - } - } -} - -/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: -/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. -pub(crate) fn compress_block_lazy( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row: bool, - dict_mode: DictMode, -) -> Result { - match (strategy, use_row) { - (Strategy::Greedy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Greedy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( - ms, seq_store, rep, src, dict_mode, - ), - _ => Err(BUG), - } -} diff --git a/src/zstd/wip/work/ldm/snap/opt.rs b/src/zstd/wip/work/ldm/snap/opt.rs deleted file mode 100644 index a7a6022a3a45..000000000000 --- a/src/zstd/wip/work/ldm/snap/opt.rs +++ /dev/null @@ -1,1721 +0,0 @@ -//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. - -use core::mem; - -use super::hist; -use super::huf::HufRepeat; -use super::internal::{ - BlockSrc, DictMode, EntropyCTables, Match, MatchState, OptPrice, OptState, Optimal, - ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, ZSTD_OPT_NUM, count, - count_2segments, hash_ptr, hash3_ptr, index_overlap_check, ll_code, ml_code, new_rep, - offset_to_offbase, -}; -use crate::common::{ - LL_BITS, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, -}; -use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32}; - -/// How much a literal adds to its frequency, so that frequencies adapt fast to new statistics. -const ZSTD_LITFREQ_ADD: u32 = 2; -const ZSTD_MAX_PRICE: i32 = 1 << 30; -/// A first block of at most this size is priced by fixed rules, not by statistics. -const ZSTD_PREDEF_THRESHOLD: usize = 8; - -const MAX_LIT: u32 = 255; - -// ── prices ──────────────────────────────────────────────────────────────── - -/// Prices are in units of `1 / BITCOST_MULTIPLIER` of a bit. -const BITCOST_ACCURACY: u32 = 8; -const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY; - -/// `WEIGHT`: approximate for `btopt`, accurate for `btultra`. -#[inline(always)] -const fn weight(stat: u32, opt_level: i32) -> u32 { - if opt_level != 0 { - frac_weight(stat) - } else { - bit_weight(stat) - } -} - -/// `ZSTD_bitWeight`: the cost of a statistic, in whole bits. -#[inline(always)] -const fn bit_weight(stat: u32) -> u32 { - highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER -} - -/// `ZSTD_fracWeight`: the cost of a statistic, with a fraction of a bit that is interpolated linearly. -#[inline(always)] -const fn frac_weight(raw_stat: u32) -> u32 { - let stat = raw_stat.wrapping_add(1); - let hb = highbit32(stat); - let b_weight = hb * BITCOST_MULTIPLIER; - // From 1 to 2, in fixed point. - let f_weight = (stat << BITCOST_ACCURACY) >> hb; - b_weight.wrapping_add(f_weight) -} - -/// `ZSTD_compressedLiterals` -#[inline(always)] -fn compressed_literals(opt_ptr: &OptState) -> bool { - opt_ptr.literal_compression_mode != ParamSwitch::Disable -} - -/// `ZSTD_setBasePrices` -#[inline(always)] -fn set_base_prices(opt_ptr: &mut OptState, opt_level: i32) { - if compressed_literals(opt_ptr) { - opt_ptr.lit_sum_base_price = weight(opt_ptr.lit_sum, opt_level); - } - opt_ptr.lit_length_sum_base_price = weight(opt_ptr.lit_length_sum, opt_level); - opt_ptr.match_length_sum_base_price = weight(opt_ptr.match_length_sum, opt_level); - opt_ptr.off_code_sum_base_price = weight(opt_ptr.off_code_sum, opt_level); -} - -/// `sum_u32` -fn sum_u32(table: &[u32]) -> u32 { - table.iter().fold(0, |total, &elt| total.wrapping_add(elt)) -} - -/// `base_directive_e`: the least a scaled down statistic can be. -#[derive(Clone, Copy, PartialEq, Eq)] -enum BaseDirective { - /// 0, if it is 0 already. - Base0Possible, - Base1Guaranteed, -} - -/// `ZSTD_downscaleStats`: returns the new sum of the elements. -fn downscale_stats( - table: &mut [u32], - last_elt_index: u32, - shift: u32, - base1: BaseDirective, -) -> Result { - let mut sum = 0u32; - for elt in table.sub_mut(..=last_elt_index as usize)? { - let base = u32::from(base1 == BaseDirective::Base1Guaranteed || *elt > 0); - let new_stat = base.wrapping_add(*elt >> (shift & 31)); - sum = sum.wrapping_add(new_stat); - *elt = new_stat; - } - Ok(sum) -} - -/// `ZSTD_scaleStats`: scales all the elements down if their sum is too large. Returns the sum that results. -fn scale_stats(table: &mut [u32], last_elt_index: u32, log_target: u32) -> Result { - let prevsum = sum_u32(table.sub(..=last_elt_index as usize)?); - let factor = prevsum >> log_target; - if factor <= 1 { - return Ok(prevsum); - } - downscale_stats( - table, - last_elt_index, - highbit32(factor), - BaseDirective::Base1Guaranteed, - ) -} - -/// Frequencies, out of about `1 << scale_log`, that make each symbol cost the bits `bit_cost` tells. Returns their sum. -fn freqs_from_bit_costs( - freqs: &mut [u32], - last_elt_index: u32, - scale_log: u32, - bit_cost: impl Fn(u32) -> u32, -) -> Result { - let mut sum = 0u32; - for (symbol, freq) in (0u32..).zip(freqs.sub_mut(..=last_elt_index as usize)?) { - let bit_cost = bit_cost(symbol); - // At least 1, to have a cost. - *freq = if bit_cost != 0 { - 1 << (scale_log.wrapping_sub(bit_cost) & 31) - } else { - 1 - }; - sum = sum.wrapping_add(*freq); - } - Ok(sum) -} - -/// `ZSTD_rescaleFreqs`: for the first block, which no statistics tell, starts them from the tables of the dictionary -/// if there is one, else from the literals of `src` and from fixed values for the sequences. For the next ones, scales -/// down what the blocks before have accumulated. -fn rescale_freqs( - opt_ptr: &mut OptState, - symbol_costs: &EntropyCTables, - src: &[u8], - opt_level: i32, -) -> Result<()> { - #[rustfmt::skip] - const BASE_LL_FREQS: [u32; MAX_LL as usize + 1] = [ - 4, 2, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, - ]; - #[rustfmt::skip] - const BASE_OFC_FREQS: [u32; MAX_OFF as usize + 1] = [ - 6, 2, 1, 1, 2, 3, 4, 4, - 4, 3, 2, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - ]; - - let compressed_literals = compressed_literals(opt_ptr); - opt_ptr.price_type = OptPrice::Dynamic; - - if opt_ptr.lit_length_sum == 0 { - if src.len() <= ZSTD_PREDEF_THRESHOLD { - opt_ptr.price_type = OptPrice::Predef; - } - - if symbol_costs.huf.repeat_mode == HufRepeat::Valid { - // A Huffman table that has all the values is presumed to come from a dictionary. - opt_ptr.price_type = OptPrice::Dynamic; - - if compressed_literals { - let huf = &symbol_costs.huf.c_table; - opt_ptr.lit_sum = - freqs_from_bit_costs(&mut opt_ptr.lit_freq, MAX_LIT, 11, |lit| { - huf.get_nb_bits(lit) - })?; - } - let fse = &symbol_costs.fse; - opt_ptr.lit_length_sum = - freqs_from_bit_costs(&mut opt_ptr.lit_length_freq, MAX_LL, 10, |ll| { - fse.litlength_c_table.get_max_nb_bits(ll) - })?; - opt_ptr.match_length_sum = - freqs_from_bit_costs(&mut opt_ptr.match_length_freq, MAX_ML, 10, |ml| { - fse.matchlength_c_table.get_max_nb_bits(ml) - })?; - opt_ptr.off_code_sum = - freqs_from_bit_costs(&mut opt_ptr.off_code_freq, MAX_OFF, 10, |of| { - fse.offcode_c_table.get_max_nb_bits(of) - })?; - } else { - if compressed_literals { - let mut lit = MAX_LIT; - hist::count_simple(&mut opt_ptr.lit_freq, &mut lit, src)?; - opt_ptr.lit_sum = downscale_stats( - &mut opt_ptr.lit_freq, - MAX_LIT, - 8, - BaseDirective::Base0Possible, - )?; - } - - opt_ptr.lit_length_freq.write_at(0, &BASE_LL_FREQS)?; - opt_ptr.lit_length_sum = sum_u32(&BASE_LL_FREQS); - - opt_ptr - .match_length_freq - .sub_mut(..=MAX_ML as usize)? - .fill(1); - opt_ptr.match_length_sum = MAX_ML + 1; - - opt_ptr.off_code_freq.write_at(0, &BASE_OFC_FREQS)?; - opt_ptr.off_code_sum = sum_u32(&BASE_OFC_FREQS); - } - } else { - if compressed_literals { - opt_ptr.lit_sum = scale_stats(&mut opt_ptr.lit_freq, MAX_LIT, 12)?; - } - opt_ptr.lit_length_sum = scale_stats(&mut opt_ptr.lit_length_freq, MAX_LL, 11)?; - opt_ptr.match_length_sum = scale_stats(&mut opt_ptr.match_length_freq, MAX_ML, 11)?; - opt_ptr.off_code_sum = scale_stats(&mut opt_ptr.off_code_freq, MAX_OFF, 11)?; - } - - set_base_prices(opt_ptr, opt_level); - Ok(()) -} - -/// `ZSTD_rawLiteralsCost`: the price of `literals`, without that of their length. -#[inline(always)] -fn raw_literals_cost(literals: &[u8], opt_ptr: &OptState, opt_level: i32) -> Result { - let lit_length = literals.len() as u32; - if lit_length == 0 { - return Ok(0); - } - - if !compressed_literals(opt_ptr) { - // 8 bits per literal. - return Ok((lit_length << 3).wrapping_mul(BITCOST_MULTIPLIER)); - } - - if opt_ptr.price_type == OptPrice::Predef { - // 6 bits per literal. - return Ok(lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER)); - } - - let mut price = opt_ptr.lit_sum_base_price.wrapping_mul(lit_length); - let lit_price_max = opt_ptr.lit_sum_base_price.wrapping_sub(BITCOST_MULTIPLIER); - for &literal in literals { - let lit_price = weight(opt_ptr.lit_freq.at(usize::from(literal))?, opt_level); - price = price.wrapping_sub(lit_price.min(lit_price_max)); - } - Ok(price) -} - -/// `ZSTD_litLengthPrice`: the price of the symbol for a length of literals. -#[inline(always)] -fn lit_length_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - if opt_ptr.price_type == OptPrice::Predef { - return Ok(weight(lit_length, opt_level)); - } - - // The format has no code for the length of a whole block of the largest size, which is then all literals. It is - // made to cost 1 bit more than the length before it. - let (lit_length, extra) = if lit_length == ZSTD_BLOCKSIZE_MAX as u32 { - (ZSTD_BLOCKSIZE_MAX as u32 - 1, BITCOST_MULTIPLIER) - } else { - (lit_length, 0) - }; - - let ll_code = ll_code(lit_length) as usize; - Ok(extra - .wrapping_add(u32::from(LL_BITS.at(ll_code)?) * BITCOST_MULTIPLIER) - .wrapping_add(opt_ptr.lit_length_sum_base_price) - .wrapping_sub(weight(opt_ptr.lit_length_freq.at(ll_code)?, opt_level))) -} - -/// `ZSTD_getMatchPrice`: the price of the offset and of the match length of a sequence. `off_base` is as in -/// [`SeqStore::store_seq`]. Below level 2, it favours small offsets, which are faster to decompress. -#[inline(always)] -fn get_match_price( - off_base: u32, - match_length: u32, - opt_ptr: &OptState, - opt_level: i32, -) -> Result { - let off_code = highbit32(off_base); - let ml_base = match_length.wrapping_sub(MINMATCH); - - if opt_ptr.price_type == OptPrice::Predef { - return Ok(weight(ml_base, opt_level).wrapping_add((16 + off_code) * BITCOST_MULTIPLIER)); - } - - let off_code_weight = weight(opt_ptr.off_code_freq.at(off_code as usize)?, opt_level); - let mut price = (off_code * BITCOST_MULTIPLIER).wrapping_add( - opt_ptr - .off_code_sum_base_price - .wrapping_sub(off_code_weight), - ); - if opt_level < 2 && off_code >= 20 { - price = price.wrapping_add((off_code - 19) * 2 * BITCOST_MULTIPLIER); - } - - let ml_code = ml_code(ml_base) as usize; - let ml_code_weight = weight(opt_ptr.match_length_freq.at(ml_code)?, opt_level); - price = price - .wrapping_add(u32::from(ML_BITS.at(ml_code)?) * BITCOST_MULTIPLIER) - .wrapping_add( - opt_ptr - .match_length_sum_base_price - .wrapping_sub(ml_code_weight), - ); - - // Matches cost a bit more, to have fewer sequences, which is faster to decompress. - Ok(price.wrapping_add(BITCOST_MULTIPLIER / 5)) -} - -/// `ZSTD_updateStats` -#[inline(always)] -fn update_stats( - opt_ptr: &mut OptState, - literals: &[u8], - off_base: u32, - match_length: u32, -) -> Result<()> { - #[inline(always)] - fn increment(freqs: &mut [u32], code: u32, sum: &mut u32) -> Result<()> { - let freq = freqs.at_mut(code as usize)?; - *freq = freq.wrapping_add(1); - *sum = sum.wrapping_add(1); - Ok(()) - } - - let lit_length = literals.len() as u32; - if compressed_literals(opt_ptr) { - for &literal in literals { - let freq = opt_ptr.lit_freq.at_mut(usize::from(literal))?; - *freq = freq.wrapping_add(ZSTD_LITFREQ_ADD); - } - opt_ptr.lit_sum = opt_ptr - .lit_sum - .wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD)); - } - - increment( - &mut opt_ptr.lit_length_freq, - ll_code(lit_length), - &mut opt_ptr.lit_length_sum, - )?; - increment( - &mut opt_ptr.off_code_freq, - highbit32(off_base), - &mut opt_ptr.off_code_sum, - )?; - increment( - &mut opt_ptr.match_length_freq, - ml_code(match_length.wrapping_sub(MINMATCH)), - &mut opt_ptr.match_length_sum, - ) -} - -/// `ZSTD_readMINMATCH`: `length` bytes, 3 or 4, to compare with others. Reads 4. -#[inline(always)] -fn read_minmatch(buf: &[u8], pos: usize, length: u32) -> Result { - let value = read_le32(buf, pos)?; - Ok(if length == 3 { value << 8 } else { value }) -} - -/// `ZSTD_insertAndFindFirstIndexHash3`: indexes the 3-byte hashes up to `ip`, which is not included, and looks it up. -/// All of it is in the current segment. -fn insert_and_find_first_index_hash3( - ms: &mut MatchState, - next_to_update3: &mut u32, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - let hash_table3 = ms.hash_table3.as_mut_slice(); - let hash_log3 = ms.hash_log3; - let target = prefix.index_of(ip); - let hash3 = hash3_ptr(prefix.buf, ip, hash_log3)?; - - for idx in *next_to_update3..target { - hash_table3.put(hash3_ptr(prefix.buf, prefix.pos_of(idx), hash_log3)?, idx)?; - } - - *next_to_update3 = target; - hash_table3.at(hash3) -} - -// ── binary tree search ──────────────────────────────────────────────────── - -/// `*ptr = value`, where `ptr` is an element of `bt`, or `None` for a place that nothing reads. -#[inline(always)] -fn store(bt: &mut [u32], ptr: Option, value: u32) -> Result<()> { - match ptr { - Some(ptr) => bt.put(ptr, value), - None => Ok(()), - } -} - -/// `ZSTD_insertBt1`: adds the position `ip`, which is at most `iend - 8`, to the tree. `target` is up to where the -/// caller is filling it, and `dict` the older segment, which is looked at if `ext_dict`. Returns how many positions -/// to move forward. -#[inline(always)] -fn insert_bt1( - ms: &mut MatchState, - prefix: Prefix<'_>, - dict: Prefix<'_>, - ip: usize, - iend: usize, - target: u32, - mls: u32, - ext_dict: bool, -) -> Result { - let c_params = ms.c_params; - let h = hash_ptr(prefix.buf, ip, c_params.hash_log, mls)?; - let bt_log = c_params.chain_log.wrapping_sub(1); - let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = dict.pos_of(dict_limit); - let prefix_start = prefix.pos_of(dict_limit); - let curr = prefix.index_of(ip); - let bt_low = curr.saturating_sub(bt_mask); - let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); - let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); - // Only the positions that are still in the window once the tree is filled are of use. - let window_low = ms.get_lowest_match_index(target, c_params.window_log); - let mut match_end_idx = curr.wrapping_add(8 + 1); - let mut best_length = 8usize; - let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); - - let bt = ms.chain_table.as_mut_slice(); - let hash_entry = ms.hash_table.at_mut(h)?; - let mut match_index = *hash_entry; - *hash_entry = curr; - - while nb_compares != 0 && match_index >= window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - // What is sure to be common. - let mut match_length = common_length_smaller.min(common_length_larger); - - // A position of the older segment, as one of the current segment, only makes sense with a length that - // brings it there. - let mut match_buf = prefix.buf; - let mut match_pos = prefix.pos_of(match_index); - if !ext_dict || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - prefix.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - match_length += count_2segments( - prefix.buf, - ip + match_length, - dict.buf, - dict.pos_of(match_index) + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - match_buf = dict.buf; - match_pos = dict.pos_of(match_index); - } - } - - if match_length > best_length { - best_length = match_length; - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); - } - } - - if ip + match_length == iend { - // They are equal: there is no telling which is smaller. Dropping it keeps the tree consistent. - break; - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < prefix.buf.at(ip + match_length)? { - // All the smaller ones will have at least this much in common. - store(bt, smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. - smaller_ptr = None; - break; - } - smaller_ptr = Some(next_ptr + 1); - match_index = bt.at(next_ptr + 1)?; - } else { - store(bt, larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = None; - break; - } - larger_ptr = Some(next_ptr); - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - store(bt, smaller_ptr, 0)?; - store(bt, larger_ptr, 0)?; - - let positions = if best_length > 384 { - 192.min((best_length - 384) as u32) - } else { - 0 - }; - Ok(positions.max(match_end_idx.wrapping_sub(curr.wrapping_add(8)))) -} - -/// `ZSTD_updateTree_internal` -#[inline(always)] -fn update_tree_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - dict: Prefix<'_>, - ip: usize, - iend: usize, - mls: u32, - dict_mode: DictMode, -) -> Result<()> { - let target = prefix.index_of(ip); - let mut idx = ms.next_to_update; - - while idx < target { - let forward = insert_bt1( - ms, - prefix, - dict, - prefix.pos_of(idx), - iend, - target, - mls, - dict_mode == DictMode::ExtDict, - )?; - idx = idx.wrapping_add(forward); - } - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. -/// Matches are not compared past the position `iend`. -pub(crate) fn update_tree( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - iend: usize, -) -> Result<()> { - let no_dict = Prefix { buf: &[], base: 0 }; - let mls = ms.c_params.min_match; - update_tree_internal(ms, prefix, no_dict, ip, iend, mls, DictMode::NoDict) -} - -/// `ZSTD_insertBtAndGetAllMatches`: adds the position `ip` to the tree, and stores in `matches` the matches found on -/// the way, each longer than the one before and all at least `length_to_beat` long. `ll0` is whether no literals -/// come before `ip`. Returns how many there are. -#[inline(always)] -fn insert_bt_and_get_all_matches( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - dict_mode: DictMode, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, - mls: u32, -) -> Result { - let buf = src.buf; - let c_params = ms.c_params; - let sufficient_len = c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); - let curr = src.index_of(ip); - let min_match = if mls == 3 { 3 } else { 4 }; - let h = hash_ptr(buf, ip, c_params.hash_log, mls)?; - let bt_log = c_params.chain_log.wrapping_sub(1); - let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt_low = curr.saturating_sub(bt_mask); - let window_low = ms.get_lowest_match_index(curr, c_params.window_log); - let match_low = window_low.max(1); - let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); - let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); - // The farthest position any match refers to, which detects repetitive patterns. - let mut match_end_idx = curr.wrapping_add(8 + 1); - let mut mnum = 0u32; - let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); - - let dms = if dict_mode == DictMode::DictMatchState { - Some(src.dms.ok_or(BUG)?) - } else { - None - }; - let dms_high_limit = dms.map_or(0, |dms| dms.end.wrapping_sub(dms.base) as u32); - let dms_low_limit = dms.map_or(0, |dms| dms.ms.window.low_limit); - let dms_index_delta = dms.map_or(0, |_| window_low.wrapping_sub(dms_high_limit)); - - let mut best_length = length_to_beat.wrapping_sub(1) as usize; - - // The repeat offsets. - let ll0 = u32::from(ll0); - let [rep0, _, _] = *rep; - for rep_code in ll0..ZSTD_REP_NUM as u32 + ll0 { - let rep_offset = match rep.get(rep_code as usize) { - Some(&offset) => offset, - None => rep0.wrapping_sub(1), - }; - let rep_index = curr.wrapping_sub(rep_offset); - let mut rep_len = 0u32; - // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. - if rep_offset.wrapping_sub(1) < curr.wrapping_sub(dict_limit) { - // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. - let rep_match = ip.wrapping_sub(rep_offset as usize); - if rep_index >= window_low - && read_minmatch(buf, ip, min_match)? == read_minmatch(buf, rep_match, min_match)? - { - rep_len = count( - buf, - ip + min_match as usize, - buf, - rep_match + min_match as usize, - i_limit, - )? as u32 - + min_match; - } - } else if dict_mode == DictMode::ExtDict { - // `curr > rep_index >= window_low` - if rep_offset.wrapping_sub(1) < curr.wrapping_sub(window_low) - && index_overlap_check(dict_limit, rep_index) - { - let rep_match = src.dict_pos_of(rep_index); - if read_minmatch(buf, ip, min_match)? - == read_minmatch(src.dict_buf, rep_match, min_match)? - { - rep_len = count_2segments( - buf, - ip + min_match as usize, - src.dict_buf, - rep_match + min_match as usize, - i_limit, - dict_end, - prefix_start, - )? as u32 - + min_match; - } - } - } else if let Some(dms) = dms { - // `curr > rep_index >= dms_low_limit` - if rep_offset.wrapping_sub(1) - < curr.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta)) - && index_overlap_check(dict_limit, rep_index) - { - let rep_match = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); - if read_minmatch(buf, ip, min_match)? - == read_minmatch(dms.buf, rep_match, min_match)? - { - rep_len = count_2segments( - buf, - ip + min_match as usize, - dms.buf, - rep_match + min_match as usize, - i_limit, - dms.end, - prefix_start, - )? as u32 - + min_match; - } - } - } - if rep_len as usize > best_length { - best_length = rep_len as usize; - matches.put( - mnum as usize, - Match { - off: rep_code - ll0 + 1, - len: rep_len, - }, - )?; - mnum += 1; - if rep_len > sufficient_len || ip + rep_len as usize == i_limit { - // The best there can be. - return Ok(mnum); - } - } - } - - // The table of 3-byte hashes. That of a dictionary is not filled. - if mls == 3 && best_length < mls as usize { - let match_index3 = - insert_and_find_first_index_hash3(ms, next_to_update3, src.prefix(), ip)?; - // Farther than that is likely to cost too much. - if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) { - let mlen = if dict_mode != DictMode::ExtDict || match_index3 >= dict_limit { - count(buf, ip, buf, src.pos_of(match_index3), i_limit)? - } else { - count_2segments( - buf, - ip, - src.dict_buf, - src.dict_pos_of(match_index3), - i_limit, - dict_end, - prefix_start, - )? - }; - - if mlen >= mls as usize { - best_length = mlen; - matches.put( - 0, - Match { - off: offset_to_offbase(curr.wrapping_sub(match_index3)), - len: mlen as u32, - }, - )?; - mnum = 1; - if mlen > sufficient_len as usize || ip + mlen == i_limit { - // The best there can be: `ip` is not inserted. - ms.next_to_update = curr.wrapping_add(1); - return Ok(1); - } - } - } - } - - let bt = ms.chain_table.as_mut_slice(); - let hash_entry = ms.hash_table.at_mut(h)?; - let mut match_index = *hash_entry; - *hash_entry = curr; - - while nb_compares != 0 && match_index >= match_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - // What is sure to be common. - let mut match_length = common_length_smaller.min(common_length_larger); - - // A position of the older segment, as one of the current segment, only makes sense with a length that - // brings it there. - let mut match_buf = buf; - let mut match_pos = src.pos_of(match_index); - if dict_mode != DictMode::ExtDict - || match_index as usize + match_length >= dict_limit as usize - { - match_length += count( - buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - i_limit, - )?; - } else { - match_length += count_2segments( - buf, - ip + match_length, - src.dict_buf, - src.dict_pos_of(match_index) + match_length, - i_limit, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - match_buf = src.dict_buf; - match_pos = src.dict_pos_of(match_index); - } - } - - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); - } - best_length = match_length; - matches.put( - mnum as usize, - Match { - off: offset_to_offbase(curr.wrapping_sub(match_index)), - len: match_length as u32, - }, - )?; - mnum += 1; - if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { - // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. - if dict_mode == DictMode::DictMatchState { - // The dictionary is not to be searched either. - nb_compares = 0; - } - break; - } - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < buf.at(ip + match_length)? { - // All the smaller ones will have at least this much in common. - store(bt, smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. - smaller_ptr = None; - break; - } - smaller_ptr = Some(next_ptr + 1); - match_index = bt.at(next_ptr + 1)?; - } else { - store(bt, larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = None; - break; - } - larger_ptr = Some(next_ptr); - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - store(bt, smaller_ptr, 0)?; - store(bt, larger_ptr, 0)?; - - if let Some(dms) = dms - && nb_compares != 0 - { - let dms_c_params = &dms.ms.c_params; - let dms_bt_log = dms_c_params.chain_log.wrapping_sub(1); - let dms_bt_mask = 1u32.wrapping_shl(dms_bt_log).wrapping_sub(1); - let dms_bt_low = if dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit) { - dms_high_limit.wrapping_sub(dms_bt_mask) - } else { - dms_low_limit - }; - let dms_h = hash_ptr(buf, ip, dms_c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(dms_h)?; - let dms_bt = dms.ms.chain_table.as_slice(); - common_length_smaller = 0; - common_length_larger = 0; - while nb_compares != 0 && dict_match_index > dms_low_limit { - let next_ptr = 2 * (dict_match_index & dms_bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - match_length += count_2segments( - buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - i_limit, - dms.end, - prefix_start, - )?; - - if match_length > best_length { - match_index = dict_match_index.wrapping_add(dms_index_delta); - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); - } - best_length = match_length; - matches.put( - mnum as usize, - Match { - off: offset_to_offbase(curr.wrapping_sub(match_index)), - len: match_length as u32, - }, - )?; - mnum += 1; - if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { - break; - } - } - - if dict_match_index <= dms_bt_low { - // Beyond the tree. - break; - } - // A match that reaches the end of the dictionary goes on at the start of the current segment. - let match_byte = if dict_match_index as usize + match_length >= dms_high_limit as usize - { - let match_pos = src.pos_of(dict_match_index.wrapping_add(dms_index_delta)); - buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? - }; - if match_byte < buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dms_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dms_bt.at(next_ptr)?; - } - nb_compares -= 1; - } - } - - // Repetitive patterns are skipped. - ms.next_to_update = match_end_idx.wrapping_sub(8); - Ok(mnum) -} - -/// `ZSTD_getAllMatchesFn`: the arguments are `matches`, `ms`, `next_to_update3`, `src`, `ip`, `i_high_limit`, `rep`, -/// `ll0` and `length_to_beat`. -type GetAllMatchesFn = fn( - &mut [Match], - &mut MatchState, - &mut u32, - &BlockSrc<'_>, - usize, - usize, - &[u32; ZSTD_REP_NUM], - bool, - u32, -) -> Result; - -/// `ZSTD_btGetAllMatches_internal` -#[inline(always)] -fn bt_get_all_matches_internal( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, - dict_mode: DictMode, - mls: u32, -) -> Result { - if src.index_of(ip) < ms.next_to_update { - // An area that is skipped. - return Ok(0); - } - let dict = Prefix { - buf: src.dict_buf, - base: src.dict_base, - }; - update_tree_internal(ms, src.prefix(), dict, ip, i_high_limit, mls, dict_mode)?; - insert_bt_and_get_all_matches( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - dict_mode, - rep, - ll0, - length_to_beat, - mls, - ) -} - -/// `ZSTD_btGetAllMatches_noDict_3` to `ZSTD_btGetAllMatches_noDict_6` -fn bt_get_all_matches_no_dict( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::NoDict, - MLS, - ) -} - -/// `ZSTD_btGetAllMatches_extDict_3` to `ZSTD_btGetAllMatches_extDict_6` -fn bt_get_all_matches_ext_dict( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::ExtDict, - MLS, - ) -} - -/// `ZSTD_btGetAllMatches_dictMatchState_3` to `ZSTD_btGetAllMatches_dictMatchState_6` -fn bt_get_all_matches_dict_match_state( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::DictMatchState, - MLS, - ) -} - -/// `ZSTD_selectBtGetAllMatches` -fn select_bt_get_all_matches(ms: &MatchState, dict_mode: DictMode) -> GetAllMatchesFn { - match (dict_mode, ms.c_params.min_match) { - (DictMode::NoDict, ..=3) => bt_get_all_matches_no_dict::<3>, - (DictMode::NoDict, 4) => bt_get_all_matches_no_dict::<4>, - (DictMode::NoDict, 5) => bt_get_all_matches_no_dict::<5>, - (DictMode::NoDict, 6..) => bt_get_all_matches_no_dict::<6>, - (DictMode::ExtDict, ..=3) => bt_get_all_matches_ext_dict::<3>, - (DictMode::ExtDict, 4) => bt_get_all_matches_ext_dict::<4>, - (DictMode::ExtDict, 5) => bt_get_all_matches_ext_dict::<5>, - (DictMode::ExtDict, 6..) => bt_get_all_matches_ext_dict::<6>, - (DictMode::DictMatchState, ..=3) => bt_get_all_matches_dict_match_state::<3>, - (DictMode::DictMatchState, 4) => bt_get_all_matches_dict_match_state::<4>, - (DictMode::DictMatchState, 5) => bt_get_all_matches_dict_match_state::<5>, - (DictMode::DictMatchState, 6..) => bt_get_all_matches_dict_match_state::<6>, - } -} - -// ── candidates from long distance matching ──────────────────────────────── - -/// `ZSTD_optLdm_t`: what it takes to decide whether to use a long distance match. -struct OptLdm<'a> { - /// `seqStore.seq`: the `seqStore.size` candidates for this block and the next ones. - seq: &'a [RawSeq], - /// `seqStore.pos` - pos: usize, - /// `seqStore.posInSequence` - pos_in_sequence: usize, - /// Where the current candidate starts. - start_pos_in_block: u32, - /// Where it ends. - end_pos_in_block: u32, - offset: u32, -} - -impl<'a> OptLdm<'a> { - fn new(seq_store: Option<&'a RawSeqStore>) -> Self { - Self { - seq: seq_store - .and_then(|store| store.seq.get(..store.size)) - .unwrap_or_default(), - pos: seq_store.map_or(0, |store| store.pos), - pos_in_sequence: seq_store.map_or(0, |store| store.pos_in_sequence), - start_pos_in_block: 0, - end_pos_in_block: 0, - offset: 0, - } - } - - /// `ZSTD_optLdm_skipRawSeqStoreBytes`: moves forward by `nb_bytes`. - fn skip_raw_seq_store_bytes(&mut self, nb_bytes: usize) { - let mut curr_pos = self.pos_in_sequence.wrapping_add(nb_bytes) as u32; - while curr_pos != 0 { - let Some(curr_seq) = self.seq.get(self.pos) else { - break; - }; - let seq_length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); - if curr_pos >= seq_length { - curr_pos -= seq_length; - self.pos += 1; - } else { - self.pos_in_sequence = curr_pos as usize; - break; - } - } - if curr_pos == 0 || self.pos == self.seq.len() { - self.pos_in_sequence = 0; - } - } - - /// `ZSTD_opt_getNextMatchAndUpdateSeqStore`: finds where the next candidate starts and ends in the block, and - /// moves past it. - fn get_next_match_and_update_seq_store( - &mut self, - curr_pos_in_block: u32, - block_bytes_remaining: u32, - ) { - let Some(&curr_seq) = self.seq.get(self.pos) else { - // No position gets there, so that there is no candidate in this block. - self.start_pos_in_block = u32::MAX; - self.end_pos_in_block = u32::MAX; - return; - }; - // What is left of the literals and of the match. - let pos_in_sequence = self.pos_in_sequence as u32; - let curr_block_end_pos = curr_pos_in_block.wrapping_add(block_bytes_remaining); - let literals_bytes_remaining = curr_seq.lit_length.saturating_sub(pos_in_sequence); - let match_bytes_remaining = if literals_bytes_remaining == 0 { - curr_seq - .match_length - .wrapping_sub(pos_in_sequence.wrapping_sub(curr_seq.lit_length)) - } else { - curr_seq.match_length - }; - - if literals_bytes_remaining >= block_bytes_remaining { - // The match starts after the block. - self.start_pos_in_block = u32::MAX; - self.end_pos_in_block = u32::MAX; - self.skip_raw_seq_store_bytes(block_bytes_remaining as usize); - return; - } - - // What is left of a match can be too short to use, which is checked later. - self.start_pos_in_block = curr_pos_in_block.wrapping_add(literals_bytes_remaining); - self.end_pos_in_block = self.start_pos_in_block.wrapping_add(match_bytes_remaining); - self.offset = curr_seq.offset; - - if self.end_pos_in_block > curr_block_end_pos { - // The match ends after the block, which can only use its start. - self.end_pos_in_block = curr_block_end_pos; - self.skip_raw_seq_store_bytes( - curr_block_end_pos.wrapping_sub(curr_pos_in_block) as usize - ); - } else { - self.skip_raw_seq_store_bytes( - literals_bytes_remaining.wrapping_add(match_bytes_remaining) as usize, - ); - } - } - - /// `ZSTD_optLdm_maybeAddMatch`: adds what is left of the candidate at `curr_pos_in_block` to `matches`, if it is - /// long enough and longer than all of them, since they are in order of length. - fn maybe_add_match( - &self, - matches: &mut [Match], - nb_matches: &mut u32, - curr_pos_in_block: u32, - min_match: u32, - ) -> Result<()> { - let pos_diff = curr_pos_in_block.wrapping_sub(self.start_pos_in_block); - let candidate_match_length = self - .end_pos_in_block - .wrapping_sub(self.start_pos_in_block) - .wrapping_sub(pos_diff); - - if curr_pos_in_block < self.start_pos_in_block - || curr_pos_in_block >= self.end_pos_in_block - || candidate_match_length < min_match - { - return Ok(()); - } - - if *nb_matches == 0 - || (candidate_match_length > matches.at(*nb_matches as usize - 1)?.len - && (*nb_matches as usize) < ZSTD_OPT_NUM) - { - matches.put( - *nb_matches as usize, - Match { - off: offset_to_offbase(self.offset), - len: candidate_match_length, - }, - )?; - *nb_matches += 1; - } - Ok(()) - } - - /// `ZSTD_optLdm_processMatchCandidate` - #[inline(always)] - fn process_match_candidate( - &mut self, - matches: &mut [Match], - nb_matches: &mut u32, - curr_pos_in_block: u32, - remaining_bytes: u32, - min_match: u32, - ) -> Result<()> { - if self.pos >= self.seq.len() { - return Ok(()); - } - - if curr_pos_in_block >= self.end_pos_in_block { - if curr_pos_in_block > self.end_pos_in_block { - // The parser does not stop at the end of each candidate: it is often some bytes past it. - let pos_overshoot = curr_pos_in_block - self.end_pos_in_block; - self.skip_raw_seq_store_bytes(pos_overshoot as usize); - } - self.get_next_match_and_update_seq_store(curr_pos_in_block, remaining_bytes); - } - self.maybe_add_match(matches, nb_matches, curr_pos_in_block, min_match) - } -} - -// ── the optimal parser ──────────────────────────────────────────────────── - -/// `LIT_PRICE`: the price of the literal at `pos`. -#[inline(always)] -fn lit_price(buf: &[u8], pos: usize, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok(raw_literals_cost(buf.span(pos, 1)?, opt_ptr, opt_level)? as i32) -} - -/// `LL_PRICE` -#[inline(always)] -fn ll_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok(lit_length_price(lit_length, opt_ptr, opt_level)? as i32) -} - -/// `LL_INCPRICE`: what one more literal, which makes `lit_length` of them, adds to the price of their length. -#[inline(always)] -fn ll_inc_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok( - ll_price(lit_length, opt_ptr, opt_level)?.wrapping_sub(ll_price( - lit_length.wrapping_sub(1), - opt_ptr, - opt_level, - )?), - ) -} - -/// `ZSTD_compressBlock_opt_generic` -#[inline(always)] -fn compress_block_opt_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - opt_level: i32, - dict_mode: DictMode, -) -> Result { - // The match finders borrow the whole of `ms`. - let mut price_table = mem::take(&mut ms.opt.price_table); - let mut match_table = mem::take(&mut ms.opt.match_table); - let result = parse_block( - ms, - seq_store, - rep, - src, - &mut price_table, - &mut match_table, - opt_level, - dict_mode, - ); - ms.opt.price_table = price_table; - ms.opt.match_table = match_table; - result -} - -/// The body of `ZSTD_compressBlock_opt_generic`. `opt` is `ms.opt.price_table` and `matches` is `ms.opt.match_table`. -#[inline(always)] -fn parse_block( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - opt: &mut [Optimal], - matches: &mut [Match], - opt_level: i32, - dict_mode: DictMode, -) -> Result { - let buf = src.buf; - let istart = src.start; - let mut ip = istart; - let mut anchor = istart; - let iend = src.end; - let ilimit = iend.saturating_sub(8); - let prefix_start = src.pos_of(ms.window.dict_limit); - - let get_all_matches = select_bt_get_all_matches(ms, dict_mode); - - let sufficient_len = ms.c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); - let min_match = if ms.c_params.min_match == 3 { 3 } else { 4 }; - let mut next_to_update3 = ms.next_to_update; - - let mut last_stretch = Optimal::default(); - let mut opt_ldm = OptLdm::new(src.ldm_seq_store); - opt_ldm.get_next_match_and_update_seq_store(0, (iend - ip) as u32); - - rescale_freqs( - &mut ms.opt, - src.symbol_costs, - buf.sub(istart..iend)?, - opt_level, - )?; - ip += usize::from(ip == prefix_start); - - 'series: while ip < ilimit { - let mut last_pos: u32; - - // The forward pass. `opt` holds stretches, each a match followed by literals, unlike a sequence, which is - // literals followed by a match: that way each position of a run of literals can have another match before it. - // It ends with the last stretch in `last_stretch`, which starts at `cur` and ends at `last_pos`. - let mut cur = 'forward: { - // Find a first match. - { - let litlen = (ip - anchor) as u32; - let mut nb_matches = get_all_matches( - matches, - ms, - &mut next_to_update3, - src, - ip, - iend, - rep, - litlen == 0, - min_match, - )?; - opt_ldm.process_match_candidate( - matches, - &mut nb_matches, - (ip - istart) as u32, - (iend - ip) as u32, - min_match, - )?; - if nb_matches == 0 { - ip += 1; - continue 'series; - } - let matches = matches.sub(..nb_matches as usize)?; - - // The price of the literals before the first match is the same whatever comes next, so it is left - // out. But how the price of their length changes depends on their length. - let first_price = ll_price(litlen, &ms.opt, opt_level)?; - { - let first = opt.at_mut(0)?; - first.mlen = 0; - first.litlen = litlen; - first.price = first_price; - first.rep = *rep; - } - - // A large match is encoded at once. - let longest = matches.at(matches.len() - 1)?; - if longest.len > sufficient_len { - last_stretch.litlen = 0; - last_stretch.mlen = longest.len; - last_stretch.off = longest.off; - last_pos = longest.len; - break 'forward 0; - } - - // Set the prices of the first matches. - let ll0_price = ll_price(0, &ms.opt, opt_level)?; - let mut pos = 1u32; - while pos < min_match { - let stretch = opt.at_mut(pos as usize)?; - stretch.price = ZSTD_MAX_PRICE; - stretch.mlen = 0; - stretch.litlen = litlen.wrapping_add(pos); - pos += 1; - } - for candidate in matches { - while pos <= candidate.len { - let match_price = - get_match_price(candidate.off, pos, &ms.opt, opt_level)? as i32; - let sequence_price = first_price.wrapping_add(match_price); - let stretch = opt.at_mut(pos as usize)?; - stretch.mlen = pos; - stretch.off = candidate.off; - stretch.litlen = 0; - stretch.price = sequence_price.wrapping_add(ll0_price); - pos += 1; - } - } - last_pos = pos - 1; - opt.at_mut(pos as usize)?.price = ZSTD_MAX_PRICE; - } - - // Check the positions that follow. - let mut cur = 0u32; - while cur < last_pos { - cur += 1; - let inr = ip + cur as usize; - - // Get to this position by a literal, if it is cheaper. - { - let before = opt.at(cur as usize - 1)?; - let litlen = before.litlen.wrapping_add(1); - let price = before - .price - .wrapping_add(lit_price(buf, inr - 1, &ms.opt, opt_level)?) - .wrapping_add(ll_inc_price(litlen, &ms.opt, opt_level)?); - let prev_match = opt.at(cur as usize)?; - if price <= prev_match.price { - opt.put( - cur as usize, - Optimal { - litlen, - price, - ..before - }, - )?; - // If this replaces a match, and 1 literal is cheaper than none, the match followed by a - // literal may be a cheaper way to the next position. - if opt_level >= 1 - && prev_match.litlen == 0 - && ll_inc_price(1, &ms.opt, opt_level)? < 0 - && inr < iend - { - let next_lit_price = lit_price(buf, inr, &ms.opt, opt_level)?; - let with_1_literal = prev_match - .price - .wrapping_add(next_lit_price) - .wrapping_add(ll_inc_price(1, &ms.opt, opt_level)?); - let with_more_literals = price - .wrapping_add(next_lit_price) - .wrapping_add(ll_inc_price( - litlen.wrapping_add(1), - &ms.opt, - opt_level, - )?); - if with_1_literal < with_more_literals - && with_1_literal < opt.at(cur as usize + 1)?.price - { - let prev = opt.at(cur.wrapping_sub(prev_match.mlen) as usize)?; - opt.put( - cur as usize + 1, - Optimal { - rep: new_rep(&prev.rep, prev_match.off, prev.litlen == 0), - litlen: 1, - price: with_1_literal, - ..prev_match - }, - )?; - last_pos = last_pos.max(cur + 1); - } - } - } - } - - // The repeat offsets are not updated while matches are compared, but here, now that one is selected. - let mut here = opt.at(cur as usize)?; - if here.litlen == 0 { - let prev = opt.at(cur.wrapping_sub(here.mlen) as usize)?; - here.rep = new_rep(&prev.rep, here.off, prev.litlen == 0); - opt.at_mut(cur as usize)?.rep = here.rep; - } - - // The last match starts at least 8 bytes before the end. - if inr > ilimit { - continue; - } - - if cur == last_pos { - break; - } - - if opt_level == 0 - && opt.at(cur as usize + 1)?.price - <= here.price.wrapping_add(BITCOST_MULTIPLIER as i32 / 2) - { - // The next position is cheaper to get to: this one is not promising. - continue; - } - - { - let base_price = here.price.wrapping_add(ll_price(0, &ms.opt, opt_level)?); - let mut nb_matches = get_all_matches( - matches, - ms, - &mut next_to_update3, - src, - inr, - iend, - &here.rep, - here.litlen == 0, - min_match, - )?; - opt_ldm.process_match_candidate( - matches, - &mut nb_matches, - (inr - istart) as u32, - (iend - inr) as u32, - min_match, - )?; - if nb_matches == 0 { - continue; - } - let matches = matches.sub(..nb_matches as usize)?; - - let longest = matches.at(matches.len() - 1)?; - if longest.len > sufficient_len - || cur.wrapping_add(longest.len) as usize >= ZSTD_OPT_NUM - || inr + longest.len as usize >= iend - { - last_stretch.mlen = longest.len; - last_stretch.off = longest.off; - last_stretch.litlen = 0; - last_pos = cur.wrapping_add(longest.len); - break 'forward cur; - } - - // Set the prices that the matches found here give. - let mut start_ml = min_match; - for candidate in matches { - let mut mlen = candidate.len; - while mlen >= start_ml { - let pos = cur + mlen; - let price = base_price.wrapping_add(get_match_price( - candidate.off, - mlen, - &ms.opt, - opt_level, - )? - as i32); - - if pos > last_pos || price < opt.at(pos as usize)?.price { - while last_pos < pos { - // Positions that nothing leads to yet. - last_pos += 1; - let stretch = opt.at_mut(last_pos as usize)?; - stretch.price = ZSTD_MAX_PRICE; - // Not 0, which would be the end of a match. - stretch.litlen = 1; - } - let stretch = opt.at_mut(pos as usize)?; - stretch.mlen = mlen; - stretch.off = candidate.off; - stretch.litlen = 0; - stretch.price = price; - } else if opt_level == 0 { - // Shorter lengths are not tried. - break; - } - mlen -= 1; - } - start_ml = candidate.len + 1; - } - } - opt.at_mut(last_pos as usize + 1)?.price = ZSTD_MAX_PRICE; - } - - last_stretch = opt.at(last_pos as usize)?; - last_pos.wrapping_sub(last_stretch.mlen) - }; - - if last_stretch.mlen == 0 { - // All the matches have been turned into literals. - ip += last_pos as usize; - continue; - } - - if last_stretch.litlen == 0 { - // It ends with a match. - let prev = opt.at(cur as usize)?; - *rep = new_rep(&prev.rep, last_stretch.off, prev.litlen == 0); - } else { - *rep = last_stretch.rep; - cur = cur.wrapping_sub(last_stretch.litlen); - } - - // Write the cheapest path in `opt`, backwards from `store_end`, over the stretches that are not on it, and as - // sequences. The literals that end the last stretch are in no sequence: they are parsed again. (The C code - // starts by making them a last sequence without a match, and overwrites it at once.) - let store_end = cur.wrapping_add(2) as usize; - let mut store_start = store_end; - let mut stretch_pos = cur; - opt.put(store_end, last_stretch)?; - loop { - let next_stretch = opt.at(stretch_pos as usize)?; - opt.at_mut(store_start)?.litlen = next_stretch.litlen; - if next_stretch.mlen == 0 { - // The start of the series. - break; - } - store_start = store_start.wrapping_sub(1); - opt.put(store_start, next_stretch)?; - stretch_pos = - stretch_pos.wrapping_sub(next_stretch.litlen.wrapping_add(next_stretch.mlen)); - } - - for sequence in opt.sub(store_start..=store_end)? { - let llen = sequence.litlen as usize; - let mlen = sequence.mlen as usize; - update_stats( - &mut ms.opt, - buf.span(anchor, llen)?, - sequence.off, - sequence.mlen, - )?; - seq_store.store_seq(llen, buf, anchor, sequence.off, mlen)?; - anchor += llen + mlen; - ip = anchor; - } - - set_base_prices(&mut ms.opt, opt_level); - } - - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_opt0` -fn compress_block_opt0( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode) -} - -/// `ZSTD_compressBlock_opt2` -fn compress_block_opt2( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode) -} - -/// `ZSTD_initStats_ultra`: a first pass over the block, only to have better statistics to start from. It has to be the -/// first block, with no dictionary and no long distance matching. Returns the block as the window then sees it, since -/// its indices change. -fn init_stats_ultra<'a>( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &[u32; ZSTD_REP_NUM], - src: &BlockSrc<'a>, -) -> Result> { - // The repeat offsets it leaves are of no use. - let mut tmp_rep = *rep; - compress_block_opt2(ms, seq_store, &mut tmp_rep, src, DictMode::NoDict)?; - - // Of the first pass, only the statistics are kept: what the tables have is made out of reach. - let src_size = src.end - src.start; - seq_store.reset(); - ms.window.base = ms.window.base.wrapping_sub(src_size); - ms.window.dict_limit = ms.window.dict_limit.wrapping_add(src_size as u32); - ms.window.low_limit = ms.window.dict_limit; - ms.next_to_update = ms.window.dict_limit; - Ok(BlockSrc { - base: src.base.wrapping_sub(src_size), - ..*src - }) -} - -/// `ZSTD_compressBlock_btultra2` -fn compress_block_btultra2( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let curr = src.index_of(src.start); - - // The first block is parsed twice, the first time to collect statistics for the second. Since the first pass is - // then forgotten, nothing must be in the tables before it: no dictionary, no prefix, no long distance matching. - if ms.opt.lit_length_sum == 0 - && seq_store.sequences.is_empty() - && ms.window.dict_limit == ms.window.low_limit - && curr == ms.window.dict_limit - && src.end - src.start > ZSTD_PREDEF_THRESHOLD - { - let src = init_stats_ultra(ms, seq_store, rep, src)?; - return compress_block_opt2(ms, seq_store, rep, &src, DictMode::NoDict); - } - - compress_block_opt2(ms, seq_store, rep, src, DictMode::NoDict) -} - -/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` -/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. -/// -/// `btultra2` can move `ms.window.base`, which `src.base` comes from. -pub(crate) fn compress_block_opt( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - dict_mode: DictMode, -) -> Result { - match (strategy, dict_mode) { - (Strategy::BtUltra2, DictMode::NoDict) => compress_block_btultra2(ms, seq_store, rep, src), - (Strategy::BtUltra | Strategy::BtUltra2, _) => { - compress_block_opt2(ms, seq_store, rep, src, dict_mode) - } - _ => compress_block_opt0(ms, seq_store, rep, src, dict_mode), - } -} diff --git a/src/zstd/wip/work/ldm/stubs/double_fast.rs b/src/zstd/wip/work/ldm/stubs/double_fast.rs deleted file mode 100644 index 65d401675d1b..000000000000 --- a/src/zstd/wip/work/ldm/stubs/double_fast.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. - -use super::internal::{ - BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_double_hash_table( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _end: usize, - _dtlm: DictTableLoadMethod, - _tfp: TableFillPurpose, -) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast` -pub(crate) fn compress_block_double_fast( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast_dictMatchState` -pub(crate) fn compress_block_double_fast_dict_match_state( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast_extDict` -pub(crate) fn compress_block_double_fast_ext_dict( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/ldm/stubs/fast.rs b/src/zstd/wip/work/ldm/stubs/fast.rs deleted file mode 100644 index 1f4788af28b4..000000000000 --- a/src/zstd/wip/work/ldm/stubs/fast.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. - -use super::internal::{ - BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_hash_table( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _end: usize, - _dtlm: DictTableLoadMethod, - _tfp: TableFillPurpose, -) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast` -pub(crate) fn compress_block_fast( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast_dictMatchState` -pub(crate) fn compress_block_fast_dict_match_state( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast_extDict` -pub(crate) fn compress_block_fast_ext_dict( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/ldm/stubs/lazy.rs b/src/zstd/wip/work/ldm/stubs/lazy.rs deleted file mode 100644 index 9f653fd684f3..000000000000 --- a/src/zstd/wip/work/ldm/stubs/lazy.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. - -use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _ip: usize, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_row_update`: the same, in the rows of the row-based match finder. -pub(crate) fn row_update(_ms: &mut MatchState, _prefix: Prefix<'_>, _ip: usize) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: -/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. -pub(crate) fn compress_block_lazy( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, - _strategy: Strategy, - _use_row: bool, - _dict_mode: DictMode, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/ldm/stubs/opt.rs b/src/zstd/wip/work/ldm/stubs/opt.rs deleted file mode 100644 index d055a44cf982..000000000000 --- a/src/zstd/wip/work/ldm/stubs/opt.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. - -use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. -/// Matches are not compared past the position `iend`. -pub(crate) fn update_tree( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _ip: usize, - _iend: usize, -) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` -/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. -pub(crate) fn compress_block_opt( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, - _strategy: Strategy, - _dict_mode: DictMode, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/ldm/zstd/compress/double_fast.rs b/src/zstd/wip/work/ldm/zstd/compress/double_fast.rs deleted file mode 100644 index fbf2d4120cad..000000000000 --- a/src/zstd/wip/work/ldm/zstd/compress/double_fast.rs +++ /dev/null @@ -1,785 +0,0 @@ -//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. - -use super::fast::{FAST_HASH_FILL_STEP, Located, count_backwards, restore_rep}; -use super::internal::{ - BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, - compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, - write_tagged_index, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, read_le32, read_le64}; - -/// `ZSTD_fillDoubleHashTableForCDict` -fn fill_double_hash_table_for_cdict( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits_l = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let h_bits_s = ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; - let mls = ms.c_params.min_match; - let hash_large = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - // The positions in between only go in the table of long hashes, where there is nothing yet. - while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { - let curr = prefix.index_of(ip); - for i in 0..FAST_HASH_FILL_STEP { - let sm_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; - let lg_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; - if i == 0 { - write_tagged_index(hash_small, sm_hash_and_tag, curr + i)?; - } - if i == 0 || hash_large.at(lg_hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { - write_tagged_index(hash_large, lg_hash_and_tag, curr + i)?; - } - if dtlm == DictTableLoadMethod::Fast { - break; - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillDoubleHashTableForCCtx` -fn fill_double_hash_table_for_cctx( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let mls = ms.c_params.min_match; - let hash_large = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - // The positions in between only go in the table of long hashes, where there is nothing yet. - while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { - let curr = prefix.index_of(ip); - for i in 0..FAST_HASH_FILL_STEP { - let sm_hash = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; - let lg_hash = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; - if i == 0 { - hash_small.put(sm_hash, curr + i)?; - } - if i == 0 || hash_large.at(lg_hash)? == 0 { - hash_large.put(lg_hash, curr + i)?; - } - if dtlm == DictTableLoadMethod::Fast { - break; - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_double_hash_table( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, - tfp: TableFillPurpose, -) -> Result<()> { - match tfp { - TableFillPurpose::ForCDict => fill_double_hash_table_for_cdict(ms, prefix, end, dtlm), - TableFillPurpose::ForCCtx => fill_double_hash_table_for_cctx(ms, prefix, end, dtlm), - } -} - -/// `ZSTD_compressBlock_doubleFast_noDict_generic` -fn compress_block_double_fast_no_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; - - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let window_log = ms.c_params.window_log; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, window_log); - let prefix_lowest = src.pos_of(prefix_lowest_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip = istart + usize::from(istart == prefix_lowest); - let [mut offset_1, mut offset_2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let current = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(current, window_log); - let max_rep = current.wrapping_sub(window_low); - if offset_2 > max_rep { - offset_saved2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved1 = offset_1; - offset_1 = 0; - } - } - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - - // One turn per match. - 'start: loop { - let mut step = 1usize; - let mut next_step = ip + K_STEP_INCR; - let mut ip1 = ip + step; - if ip1 > ilimit { - break 'start; - } - - let mut hl0 = hash_ptr(buf, ip, h_bits_l, 8)?; - let mut idxl0 = hash_long.at(hl0)?; - let mut curr; - - let m_length = 'match_stored: { - // One turn per position. Ends with the offset and the length of a match, which `ip` is moved to the start - // of. - let (offset, m_length, hl1) = loop { - let hs0 = hash_ptr(buf, ip, h_bits_s, MLS)?; - let idxs0 = hash_small.at(hs0)?; - curr = src.index_of(ip); - hash_long.put(hl0, curr)?; - hash_small.put(hs0, curr)?; - - if (offset_1 > 0) - & (read_le32(buf, (ip + 1).wrapping_sub(offset_1 as usize))? - == read_le32(buf, ip + 1)?) - { - let m_length = count( - buf, - ip + 1 + 4, - buf, - (ip + 1 + 4).wrapping_sub(offset_1 as usize), - iend, - )? + 4; - ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - break 'match_stored m_length; - } - - let hl1 = hash_ptr(buf, ip1, h_bits_l, 8)?; - - let matchl0 = src.pos_of(idxl0); - if idxl0 >= prefix_lowest_index && read_le64(buf, matchl0)? == read_le64(buf, ip)? { - let m_length = count(buf, ip + 8, buf, matchl0 + 8, iend)? + 8; - let offset = ip.wrapping_sub(matchl0) as u32; - let back = count_backwards(buf, ip, anchor, buf, matchl0, prefix_lowest); - ip -= back; - break (offset, m_length + back, hl1); - } - - let idxl1 = hash_long.at(hl1)?; - - let mut matchs0 = src.pos_of(idxs0); - if idxs0 >= prefix_lowest_index && read_le32(buf, matchs0)? == read_le32(buf, ip)? { - // A short match: a longer one at the next position is preferred. - let mut m_length = count(buf, ip + 4, buf, matchs0 + 4, iend)? + 4; - let mut offset = ip.wrapping_sub(matchs0) as u32; - let matchl1 = src.pos_of(idxl1); - if idxl1 > prefix_lowest_index - && read_le64(buf, matchl1)? == read_le64(buf, ip1)? - { - let l1len = count(buf, ip1 + 8, buf, matchl1 + 8, iend)? + 8; - if l1len > m_length { - ip = ip1; - m_length = l1len; - offset = ip.wrapping_sub(matchl1) as u32; - matchs0 = matchl1; - } - } - let back = count_backwards(buf, ip, anchor, buf, matchs0, prefix_lowest); - ip -= back; - break (offset, m_length + back, hl1); - } - - if ip1 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - ip = ip1; - ip1 += step; - - hl0 = hl1; - idxl0 = idxl1; - if ip1 > ilimit { - break 'start; - } - }; - - offset_2 = offset_1; - offset_1 = offset; - // With a larger step, ip1 may be past the end of the match. - if step < 4 { - hash_long.put(hl1, src.index_of(ip1))?; - } - seq_store.store_seq( - ip - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - m_length - }; - - ip += m_length; - anchor = ip; - - if ip <= ilimit { - let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; - - while ip <= ilimit - && (offset_2 > 0) - & (read_le32(buf, ip)? == read_le32(buf, ip.wrapping_sub(offset_2 as usize))?) - { - let r_length = count( - buf, - ip + 4, - buf, - (ip + 4).wrapping_sub(offset_2 as usize), - iend, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, src.index_of(ip))?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, src.index_of(ip))?; - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; - ip += r_length; - anchor = ip; - } - } - } - - restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` -fn compress_block_double_fast_dict_match_state_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, ms.c_params.window_log); - let prefix_lowest = src.pos_of(prefix_lowest_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - let [mut offset_1, mut offset_2, _] = *rep; - - let dms = src.dms.as_ref().ok_or(BUG)?; - let dict_hash_long = dms.ms.hash_table.as_slice(); - let dict_hash_small = dms.ms.chain_table.as_slice(); - let dict_start_index = dms.ms.window.dict_limit; - let dict_start = dms.pos_of(dict_start_index); - let dict_end = dms.end; - // What to subtract from an index below prefix_lowest_index to get the index in the dictionary. - let dict_index_delta = prefix_lowest_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); - let dict_h_bits_l = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let dict_h_bits_s = dms.ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; - let dict_and_prefix_length = istart - .wrapping_sub(prefix_lowest) - .wrapping_add(dict_end.wrapping_sub(dict_start)) as u32; - - let locate = |index: u32| { - if index < prefix_lowest_index { - Located { - buf: dms.buf, - pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_lowest, - end: iend, - } - } - }; - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - - let mut anchor = istart; - let mut ip = istart + usize::from(dict_and_prefix_length == 0); - - 'search: while ip < ilimit { - let h2 = hash_ptr(buf, ip, h_bits_l, 8)?; - let h = hash_ptr(buf, ip, h_bits_s, MLS)?; - let dict_hash_and_tag_l = hash_ptr(buf, ip, dict_h_bits_l, 8)?; - let dict_hash_and_tag_s = hash_ptr(buf, ip, dict_h_bits_s, MLS)?; - let dict_match_index_and_tag_l = - dict_hash_long.at(dict_hash_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_match_index_and_tag_s = - dict_hash_small.at(dict_hash_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_tags_match_l = - compare_packed_tags(dict_match_index_and_tag_l as usize, dict_hash_and_tag_l); - let dict_tags_match_s = - compare_packed_tags(dict_match_index_and_tag_s as usize, dict_hash_and_tag_s); - let curr = src.index_of(ip); - let match_index_l = hash_long.at(h2)?; - let mut match_index_s = hash_small.at(h)?; - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); - hash_long.put(h2, curr)?; - hash_small.put(h, curr)?; - - let m_length = 'match_stored: { - if index_overlap_check(prefix_lowest_index, rep_index) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? - { - let m_length = count_2segments( - buf, - ip + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - iend, - rep_match.end, - prefix_lowest, - )? + 4; - ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - break 'match_stored m_length; - } - - // The offset and the length of a match, which `ip` is moved to the start of. - let (offset, m_length) = 'match_found: { - let match_long = src.pos_of(match_index_l); - if match_index_l >= prefix_lowest_index - && read_le64(buf, match_long)? == read_le64(buf, ip)? - { - let m_length = count(buf, ip + 8, buf, match_long + 8, iend)? + 8; - let offset = ip.wrapping_sub(match_long) as u32; - let back = count_backwards(buf, ip, anchor, buf, match_long, prefix_lowest); - ip -= back; - break 'match_found (offset, m_length + back); - } else if dict_tags_match_l { - let dict_match_index_l = - dict_match_index_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match_l = dms.pos_of(dict_match_index_l); - if dict_match_index_l > dict_start_index - && read_le64(dms.buf, dict_match_l)? == read_le64(buf, ip)? - { - let m_length = count_2segments( - buf, - ip + 8, - dms.buf, - dict_match_l + 8, - iend, - dict_end, - prefix_lowest, - )? + 8; - let offset = curr - .wrapping_sub(dict_match_index_l) - .wrapping_sub(dict_index_delta); - let back = - count_backwards(buf, ip, anchor, dms.buf, dict_match_l, dict_start); - ip -= back; - break 'match_found (offset, m_length + back); - } - } - - let short_found = if match_index_s > prefix_lowest_index { - read_le32(buf, src.pos_of(match_index_s))? == read_le32(buf, ip)? - } else if dict_tags_match_s { - let dict_match_index_s = - dict_match_index_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS; - match_index_s = dict_match_index_s.wrapping_add(dict_index_delta); - dict_match_index_s > dict_start_index - && read_le32(dms.buf, dms.pos_of(dict_match_index_s))? - == read_le32(buf, ip)? - } else { - false - }; - if !short_found { - ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; - continue 'search; - } - - // A short match: a long one at the next position is preferred. - let hl3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; - let dict_hash_and_tag_l3 = hash_ptr(buf, ip + 1, dict_h_bits_l, 8)?; - let match_index_l3 = hash_long.at(hl3)?; - let dict_match_index_and_tag_l3 = - dict_hash_long.at(dict_hash_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_tags_match_l3 = - compare_packed_tags(dict_match_index_and_tag_l3 as usize, dict_hash_and_tag_l3); - let match_l3 = src.pos_of(match_index_l3); - hash_long.put(hl3, curr + 1)?; - - if match_index_l3 >= prefix_lowest_index - && read_le64(buf, match_l3)? == read_le64(buf, ip + 1)? - { - let m_length = count(buf, ip + 9, buf, match_l3 + 8, iend)? + 8; - ip += 1; - let offset = ip.wrapping_sub(match_l3) as u32; - let back = count_backwards(buf, ip, anchor, buf, match_l3, prefix_lowest); - ip -= back; - break 'match_found (offset, m_length + back); - } else if dict_tags_match_l3 { - let dict_match_index_l3 = - dict_match_index_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match_l3 = dms.pos_of(dict_match_index_l3); - if dict_match_index_l3 > dict_start_index - && read_le64(dms.buf, dict_match_l3)? == read_le64(buf, ip + 1)? - { - let m_length = count_2segments( - buf, - ip + 1 + 8, - dms.buf, - dict_match_l3 + 8, - iend, - dict_end, - prefix_lowest, - )? + 8; - ip += 1; - let offset = curr - .wrapping_add(1) - .wrapping_sub(dict_match_index_l3) - .wrapping_sub(dict_index_delta); - let back = - count_backwards(buf, ip, anchor, dms.buf, dict_match_l3, dict_start); - ip -= back; - break 'match_found (offset, m_length + back); - } - } - - let match_s = locate(match_index_s); - let m_length = count_2segments( - buf, - ip + 4, - match_s.buf, - match_s.pos + 4, - iend, - match_s.end, - prefix_lowest, - )? + 4; - let offset = curr.wrapping_sub(match_index_s); - let back = - count_backwards(buf, ip, anchor, match_s.buf, match_s.pos, match_s.start); - ip -= back; - (offset, m_length + back) - }; - - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - m_length - }; - - ip += m_length; - anchor = ip; - - if ip <= ilimit { - let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; - - while ip <= ilimit { - let current2 = src.index_of(ip); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !(index_overlap_check(prefix_lowest_index, rep_index2) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_lowest, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; - ip += rep_length2; - anchor = ip; - } - } - } - - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_doubleFast` -pub(crate) fn compress_block_double_fast( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_double_fast_no_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_no_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_no_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_no_dict_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_doubleFast_dictMatchState` -pub(crate) fn compress_block_double_fast_dict_match_state( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_double_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_doubleFast_extDict_generic` -fn compress_block_double_fast_ext_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); - let dict_start_index = low_limit; - let dict_limit = ms.window.dict_limit; - let prefix_start_index = dict_limit.max(low_limit); - let prefix_start = src.pos_of(prefix_start_index); - let dict_start = src.dict_pos_of(dict_start_index); - let dict_end = src.dict_pos_of(prefix_start_index); - let [mut offset_1, mut offset_2, _] = *rep; - - // Nothing of the older segment is within reach. - if prefix_start_index == dict_start_index { - return compress_block_double_fast(ms, seq_store, rep, src); - } - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: src.dict_buf, - pos: src.dict_pos_of(index), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - - let mut anchor = istart; - let mut ip = istart; - - while ip < ilimit { - let h_small = hash_ptr(buf, ip, h_bits_s, MLS)?; - let match_index = hash_small.at(h_small)?; - let match_s = locate(match_index); - - let h_long = hash_ptr(buf, ip, h_bits_l, 8)?; - let match_long_index = hash_long.at(h_long)?; - let match_long = locate(match_long_index); - - let curr = src.index_of(ip); - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); - hash_long.put(h_long, curr)?; - hash_small.put(h_small, curr)?; - - let m_length; - if (index_overlap_check(prefix_start_index, rep_index) - & (offset_1 <= curr.wrapping_add(1).wrapping_sub(dict_start_index))) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? - { - m_length = count_2segments( - buf, - ip + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - iend, - rep_match.end, - prefix_start, - )? + 4; - ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - } else { - // The match, how much of it is known, and its offset. `ip` is moved to where it is for. - let (found, known, offset) = if match_long_index > dict_start_index - && read_le64(match_long.buf, match_long.pos)? == read_le64(buf, ip)? - { - (match_long, 8, curr.wrapping_sub(match_long_index)) - } else if match_index > dict_start_index - && read_le32(match_s.buf, match_s.pos)? == read_le32(buf, ip)? - { - // A short match: a long one at the next position is preferred. - let h3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; - let match_index3 = hash_long.at(h3)?; - let match3 = locate(match_index3); - hash_long.put(h3, curr + 1)?; - if match_index3 > dict_start_index - && read_le64(match3.buf, match3.pos)? == read_le64(buf, ip + 1)? - { - ip += 1; - (match3, 8, curr.wrapping_add(1).wrapping_sub(match_index3)) - } else { - (match_s, 4, curr.wrapping_sub(match_index)) - } - } else { - ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; - continue; - }; - let forwards = count_2segments( - buf, - ip + known, - found.buf, - found.pos + known, - iend, - found.end, - prefix_start, - )? + known; - let back = count_backwards(buf, ip, anchor, found.buf, found.pos, found.start); - ip -= back; - m_length = forwards + back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - } - - ip += m_length; - anchor = ip; - - if ip <= ilimit { - let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; - - while ip <= ilimit { - let current2 = src.index_of(ip); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !((index_overlap_check(prefix_start_index, rep_index2) - & (offset_2 <= current2.wrapping_sub(dict_start_index))) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_start, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; - ip += rep_length2; - anchor = ip; - } - } - } - - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_doubleFast_extDict` -pub(crate) fn compress_block_double_fast_ext_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_double_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), - } -} diff --git a/src/zstd/wip/work/ldm/zstd/compress/fast.rs b/src/zstd/wip/work/ldm/zstd/compress/fast.rs deleted file mode 100644 index 4bec128a6786..000000000000 --- a/src/zstd/wip/work/ldm/zstd/compress/fast.rs +++ /dev/null @@ -1,829 +0,0 @@ -//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. - -use super::internal::{ - BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, - compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, - write_tagged_index, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, read_le32}; - -pub(super) const FAST_HASH_FILL_STEP: u32 = 3; - -/// Where an index leads, when it can be in one of two segments. -#[derive(Clone, Copy)] -pub(super) struct Located<'a> { - /// The buffer that the segment is in. - pub(super) buf: &'a [u8], - /// The position of the index in `buf`. - pub(super) pos: usize, - /// Where the segment starts in `buf`. - pub(super) start: usize, - /// Where it ends. - pub(super) end: usize, -} - -/// How many bytes before `ip`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. -#[inline(always)] -pub(super) fn count_backwards( - buf: &[u8], - ip: usize, - anchor: usize, - match_buf: &[u8], - match_pos: usize, - match_low: usize, -) -> usize { - let before_ip = buf.get(anchor..ip).unwrap_or_default(); - let before_match = match_buf.get(match_low..match_pos).unwrap_or_default(); - before_ip - .iter() - .rev() - .zip(before_match.iter().rev()) - .take_while(|(a, b)| a == b) - .count() -} - -/// The repeat offsets to leave at the end of a block, at the start of which those that reached too far were saved and -/// set to 0. -#[inline(always)] -pub(super) fn restore_rep( - rep: &mut [u32; ZSTD_REP_NUM], - offset_1: u32, - offset_2: u32, - offset_saved1: u32, - offset_saved2: u32, -) { - // If offset_1 started invalid and became valid, offset_saved1 is what rotates into the second place. - let offset_saved2 = if offset_saved1 != 0 && offset_1 != 0 { - offset_saved1 - } else { - offset_saved2 - }; - let [rep0, rep1, _] = rep; - *rep0 = if offset_1 != 0 { - offset_1 - } else { - offset_saved1 - }; - *rep1 = if offset_2 != 0 { - offset_2 - } else { - offset_saved2 - }; -} - -/// `ZSTD_fillHashTableForCDict` -fn fill_hash_table_for_cdict( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let mls = ms.c_params.min_match; - let hash_table = ms.hash_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { - let curr = prefix.index_of(ip); - write_tagged_index(hash_table, hash_ptr(buf, ip, h_bits, mls)?, curr)?; - if dtlm == DictTableLoadMethod::Full { - // The positions in between only go where there is nothing yet. - for p in 1..FAST_HASH_FILL_STEP { - let hash_and_tag = hash_ptr(buf, ip + p as usize, h_bits, mls)?; - if hash_table.at(hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { - write_tagged_index(hash_table, hash_and_tag, curr + p)?; - } - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillHashTableForCCtx` -fn fill_hash_table_for_cctx( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { - let h_bits = ms.c_params.hash_log; - let mls = ms.c_params.min_match; - let hash_table = ms.hash_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { - let curr = prefix.index_of(ip); - hash_table.put(hash_ptr(buf, ip, h_bits, mls)?, curr)?; - if dtlm == DictTableLoadMethod::Full { - // The positions in between only go where there is nothing yet. - for p in 1..FAST_HASH_FILL_STEP { - let hash = hash_ptr(buf, ip + p as usize, h_bits, mls)?; - if hash_table.at(hash)? == 0 { - hash_table.put(hash, curr + p)?; - } - } - } - ip += FAST_HASH_FILL_STEP as usize; - } - Ok(()) -} - -/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_hash_table( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, - tfp: TableFillPurpose, -) -> Result<()> { - match tfp { - TableFillPurpose::ForCDict => fill_hash_table_for_cdict(ms, prefix, end, dtlm), - TableFillPurpose::ForCCtx => fill_hash_table_for_cctx(ms, prefix, end, dtlm), - } -} - -/// `ZSTD_match4Found_branch`, and `ZSTD_match4Found_cmov`, which only differs in how it avoids reading at an index that -/// is not valid. -#[inline(always)] -fn match4_found( - buf: &[u8], - current: usize, - match_pos: usize, - match_idx: u32, - idx_low_limit: u32, -) -> Result { - Ok(match_idx >= idx_low_limit && read_le32(buf, current)? == read_le32(buf, match_pos)?) -} - -/// `ZSTD_compressBlock_fast_noDict_generic` -/// -/// Each turn of the inner loop looks at two positions, `ip0` and `ip1`, for a match, and at `ip2` for a repeat. The -/// hash of a position is computed, and the table is read, one position ahead of the comparison. -fn compress_block_fast_no_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); - - let hlog = ms.c_params.hash_log; - let window_log = ms.c_params.window_log; - let target_length = ms.c_params.target_length; - // The least step is 2, as two positions are looked at per turn. - let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_start_index = ms.get_lowest_prefix_index(end_index, window_log); - let prefix_start = src.pos_of(prefix_start_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip0 = istart + usize::from(istart == prefix_start); - let [mut rep_offset1, mut rep_offset2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let curr = src.index_of(ip0); - let window_low = ms.get_lowest_prefix_index(curr, window_log); - let max_rep = curr.wrapping_sub(window_low); - if rep_offset2 > max_rep { - offset_saved2 = rep_offset2; - rep_offset2 = 0; - } - if rep_offset1 > max_rep { - offset_saved1 = rep_offset1; - rep_offset1 = 0; - } - } - let hash_table = ms.hash_table.as_mut_slice(); - - 'start: loop { - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - - let mut ip1 = ip0 + 1; - let mut ip2 = ip0 + step; - let mut ip3 = ip2 + 1; - if ip3 >= ilimit { - break 'start; - } - - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; - let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; - let mut match_idx = hash_table.at(hash0)?; - let mut current0; - - // Ends with the start and the length so far of a repeat match, or with `None` for a match at `match_idx`. - let rep_match = loop { - let rval = read_le32(buf, ip2.wrapping_sub(rep_offset1 as usize))?; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if (read_le32(buf, ip2)? == rval) & (rep_offset1 > 0) { - ip0 = ip2; - let match0 = ip0.wrapping_sub(rep_offset1 as usize); - let back = usize::from(buf.at(ip0 - 1)? == buf.at(match0.wrapping_sub(1))?); - ip0 -= back; - // The hash of ip1 is there, and ip1 is before the end of the match. - hash_table.put(hash1, src.index_of(ip1))?; - break Some((match0.wrapping_sub(back), 4 + back)); - } - - if match4_found( - buf, - ip0, - src.pos_of(match_idx), - match_idx, - prefix_start_index, - )? { - hash_table.put(hash1, src.index_of(ip1))?; - break None; - } - - match_idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip3; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if match4_found( - buf, - ip0, - src.pos_of(match_idx), - match_idx, - prefix_start_index, - )? { - // ip1 is certain to be before the end of the match only if the step is small. - if step <= 4 { - hash_table.put(hash1, src.index_of(ip1))?; - } - break None; - } - - match_idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip0 + step; - ip3 = ip1 + step; - - if ip2 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - if ip3 >= ilimit { - break 'start; - } - }; - - let (offcode, match0, mut m_length) = match rep_match { - Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), - None => { - let match0 = src.pos_of(match_idx); - rep_offset2 = rep_offset1; - rep_offset1 = ip0.wrapping_sub(match0) as u32; - let back = count_backwards(buf, ip0, anchor, buf, match0, prefix_start); - ip0 -= back; - (offset_to_offbase(rep_offset1), match0 - back, 4 + back) - } - }; - - m_length += count(buf, ip0 + m_length, buf, match0 + m_length, iend)?; - seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; - ip0 += m_length; - anchor = ip0; - - if ip0 <= ilimit { - hash_table.put( - hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, - current0 + 2, - )?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; - - if rep_offset2 > 0 { - while ip0 <= ilimit - && read_le32(buf, ip0)? - == read_le32(buf, ip0.wrapping_sub(rep_offset2 as usize))? - { - let r_length = count( - buf, - ip0 + 4, - buf, - (ip0 + 4).wrapping_sub(rep_offset2 as usize), - iend, - )? + 4; - core::mem::swap(&mut rep_offset1, &mut rep_offset2); - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; - ip0 += r_length; - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; - anchor = ip0; - } - } - } - } - - restore_rep(rep, rep_offset1, rep_offset2, offset_saved1, offset_saved2); - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_fast` -pub(crate) fn compress_block_fast( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_fast_no_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_no_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_no_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_no_dict_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_fast_dictMatchState_generic` -fn compress_block_fast_dict_match_state_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; - - let hlog = ms.c_params.hash_log; - let target_length = ms.c_params.target_length; - let step_size = (target_length + u32::from(target_length == 0)) as usize; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let prefix_start_index = ms.window.dict_limit; - let prefix_start = src.pos_of(prefix_start_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - let [mut offset_1, mut offset_2, _] = *rep; - - let dms = src.dms.as_ref().ok_or(BUG)?; - let dict_hash_table = dms.ms.hash_table.as_slice(); - let dict_start_index = dms.ms.window.dict_limit; - let dict_start = dms.pos_of(dict_start_index); - let dict_end = dms.end; - // What to subtract from an index below prefix_start_index to get the index in the dictionary. - let dict_index_delta = prefix_start_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); - let dict_and_prefix_length = istart - .wrapping_sub(prefix_start) - .wrapping_add(dict_end) - .wrapping_sub(dict_start) as u32; - let dict_h_bits = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: dms.buf, - pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; - let hash_table = ms.hash_table.as_mut_slice(); - - let mut anchor = istart; - let mut ip0 = istart + usize::from(dict_and_prefix_length == 0); - let mut ip1 = istart + step_size; - - 'search: while ip1 <= ilimit { - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; - - let dict_hash_and_tag0 = hash_ptr(buf, ip0, dict_h_bits, MLS)?; - let mut dict_match_index_and_tag = - dict_hash_table.at(dict_hash_and_tag0 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let mut dict_tags_match = - compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag0); - - let mut match_index = hash_table.at(hash0)?; - let mut curr = src.index_of(ip0); - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - - let m_length = loop { - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); - let hash1 = hash_ptr(buf, ip1, hlog, MLS)?; - let dict_hash_and_tag1 = hash_ptr(buf, ip1, dict_h_bits, MLS)?; - hash_table.put(hash0, curr)?; - - if index_overlap_check(prefix_start_index, rep_index) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip0 + 1)? - { - let m_length = count_2segments( - buf, - ip0 + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - iend, - rep_match.end, - prefix_start, - )? + 4; - ip0 += 1; - seq_store.store_seq(ip0 - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - break m_length; - } - - if dict_tags_match { - let dict_match_index = dict_match_index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match = dms.pos_of(dict_match_index); - // A match in the dictionary is only taken if there is no candidate in the prefix. - if dict_match_index > dict_start_index - && read_le32(dms.buf, dict_match)? == read_le32(buf, ip0)? - && match_index <= prefix_start_index - { - let offset = curr - .wrapping_sub(dict_match_index) - .wrapping_sub(dict_index_delta); - let mut m_length = count_2segments( - buf, - ip0 + 4, - dms.buf, - dict_match + 4, - iend, - dict_end, - prefix_start, - )? + 4; - let back = count_backwards(buf, ip0, anchor, dms.buf, dict_match, dict_start); - ip0 -= back; - m_length += back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip0 - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - break m_length; - } - } - - let match_pos = src.pos_of(match_index); - if match4_found(buf, ip0, match_pos, match_index, prefix_start_index)? { - let offset = ip0.wrapping_sub(match_pos) as u32; - let mut m_length = count(buf, ip0 + 4, buf, match_pos + 4, iend)? + 4; - let back = count_backwards(buf, ip0, anchor, buf, match_pos, prefix_start); - ip0 -= back; - m_length += back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip0 - anchor, - buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - break m_length; - } - - dict_match_index_and_tag = - dict_hash_table.at(dict_hash_and_tag1 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - dict_tags_match = - compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag1); - match_index = hash_table.at(hash1)?; - - if ip1 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - ip0 = ip1; - ip1 += step; - if ip1 > ilimit { - break 'search; - } - - curr = src.index_of(ip0); - hash0 = hash1; - }; - - ip0 += m_length; - anchor = ip0; - - if ip0 <= ilimit { - hash_table.put(hash_ptr(buf, src.pos_of(curr + 2), hlog, MLS)?, curr + 2)?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; - - while ip0 <= ilimit { - let current2 = src.index_of(ip0); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !(index_overlap_check(prefix_start_index, rep_index2) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip0 + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_start, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, current2)?; - ip0 += rep_length2; - anchor = ip0; - } - } - - ip1 = ip0 + step_size; - } - - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_fast_dictMatchState` -pub(crate) fn compress_block_fast_dict_match_state( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), - } -} - -/// `ZSTD_compressBlock_fast_extDict_generic` -fn compress_block_fast_ext_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); - - let hlog = ms.c_params.hash_log; - let target_length = ms.c_params.target_length; - let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; - let buf = src.buf; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); - let dict_start_index = low_limit; - let dict_start = src.dict_pos_of(dict_start_index); - let dict_limit = ms.window.dict_limit; - let prefix_start_index = dict_limit.max(low_limit); - let prefix_start = src.pos_of(prefix_start_index); - let dict_end = src.dict_pos_of(prefix_start_index); - - // Nothing of the older segment is within reach. - if prefix_start_index == dict_start_index { - return compress_block_fast(ms, seq_store, rep, src); - } - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip0 = istart; - let [mut offset_1, mut offset_2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let curr = src.index_of(ip0); - let max_rep = curr.wrapping_sub(dict_start_index); - if offset_2 >= max_rep { - offset_saved2 = offset_2; - offset_2 = 0; - } - if offset_1 >= max_rep { - offset_saved1 = offset_1; - offset_1 = 0; - } - } - - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: src.dict_buf, - pos: src.dict_pos_of(index), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; - // Whether there is a match for the position `ip` at `idx`, which is from the table. - let match_found = |ip: usize, idx: u32| -> Result { - let candidate = locate(idx); - Ok(idx >= dict_start_index - && read_le32(candidate.buf, candidate.pos)? == read_le32(buf, ip)?) - }; - let hash_table = ms.hash_table.as_mut_slice(); - - 'start: loop { - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - - let mut ip1 = ip0 + 1; - let mut ip2 = ip0 + step; - let mut ip3 = ip2 + 1; - if ip3 >= ilimit { - break 'start; - } - - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; - let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; - let mut idx = hash_table.at(hash0)?; - let mut current0; - - // Ends with the start and the length so far of a repeat match, or with `None` for a match at `idx`. - let rep_match = loop { - let current2 = src.index_of(ip2); - let rep_index = current2.wrapping_sub(offset_1); - let mut rep_match = locate(rep_index); - // The 4 bytes have to be in one segment, with a byte before them. - let rep_found = ((prefix_start_index.wrapping_sub(rep_index) >= 4) & (offset_1 > 0)) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip2)?; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if rep_found { - ip0 = ip2; - let back = usize::from( - buf.at(ip0 - 1)? == rep_match.buf.at(rep_match.pos.wrapping_sub(1))?, - ); - ip0 -= back; - rep_match.pos = rep_match.pos.wrapping_sub(back); - break Some((rep_match, 4 + back)); - } - - if match_found(ip0, idx)? { - break None; - } - - idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip3; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if match_found(ip0, idx)? { - break None; - } - - idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip0 + step; - ip3 = ip1 + step; - - if ip2 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - if ip3 >= ilimit { - break 'start; - } - }; - - let (offcode, match0, mut m_length) = match rep_match { - Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), - None => { - let offset = current0.wrapping_sub(idx); - let mut match0 = locate(idx); - offset_2 = offset_1; - offset_1 = offset; - let back = count_backwards(buf, ip0, anchor, match0.buf, match0.pos, match0.start); - ip0 -= back; - match0.pos -= back; - (offset_to_offbase(offset), match0, 4 + back) - } - }; - - m_length += count_2segments( - buf, - ip0 + m_length, - match0.buf, - match0.pos + m_length, - iend, - match0.end, - prefix_start, - )?; - seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; - ip0 += m_length; - anchor = ip0; - - if ip1 < ip0 { - hash_table.put(hash1, src.index_of(ip1))?; - } - - if ip0 <= ilimit { - hash_table.put( - hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, - current0 + 2, - )?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; - - while ip0 <= ilimit { - let rep_index2 = src.index_of(ip0).wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !((index_overlap_check(prefix_start_index, rep_index2) & (offset_2 > 0)) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) - { - break; - } - let rep_length2 = count_2segments( - buf, - ip0 + 4, - rep_match2.buf, - rep_match2.pos + 4, - iend, - rep_match2.end, - prefix_start, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; - ip0 += rep_length2; - anchor = ip0; - } - } - } - - restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_fast_extDict` -pub(crate) fn compress_block_fast_ext_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), - } -} diff --git a/src/zstd/wip/work/ldm/zstd/compress/fse.rs b/src/zstd/wip/work/ldm/zstd/compress/fse.rs deleted file mode 100644 index 1b8f7cbf3173..000000000000 --- a/src/zstd/wip/work/ldm/zstd/compress/fse.rs +++ /dev/null @@ -1,70 +0,0 @@ -//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. - -use crate::common::{MAX_SEQ, ML_FSE_LOG}; -use crate::util::SliceExt; - -/// `FSE_repeat` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum FseRepeat { - /// The table cannot be used again. - None, - /// It can, if it has a code for every symbol to encode. - Check, - /// It can. - Valid, -} - -/// `FSE_symbolCompressionTransform` -#[derive(Clone, Copy, Default)] -pub(crate) struct FseSymbolTransform { - pub(crate) delta_find_state: i32, - pub(crate) delta_nb_bits: u32, -} - -/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. -#[derive(Clone)] -pub(crate) struct FseCTable { - pub(crate) table_log: u32, - pub(crate) max_symbol_value: u32, - pub(crate) state_table: [u16; 1 << ML_FSE_LOG], - pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], -} - -impl FseCTable { - pub(crate) const fn new() -> Self { - Self { - table_log: 0, - max_symbol_value: 0, - state_table: [0; 1 << ML_FSE_LOG], - symbol_tt: [FseSymbolTransform { - delta_find_state: 0, - delta_nb_bits: 0, - }; MAX_SEQ as usize + 1], - } - } - - /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log - /// if the table has no code for it. - #[inline(always)] - pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { - let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); - transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 - } - - /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. - /// The table has a code for it. - #[inline(always)] - pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { - let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); - let min_nb_bits = transform.delta_nb_bits >> 16; - let threshold = (min_nb_bits + 1) << 16; - let table_size = 1u32 << self.table_log; - let delta_from_threshold = - threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); - // A linear interpolation, which is very approximate. - let normalized_delta_from_threshold = - (delta_from_threshold << accuracy_log) >> self.table_log; - let bit_multiplier = 1u32 << accuracy_log; - ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) - } -} diff --git a/src/zstd/wip/work/ldm/zstd/compress/hist.rs b/src/zstd/wip/work/ldm/zstd/compress/hist.rs deleted file mode 100644 index becf42516867..000000000000 --- a/src/zstd/wip/work/ldm/zstd/compress/hist.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! `hist.c`: how many times each byte value occurs. - -use crate::error::Result; -use crate::util::SliceExt; - -/// `HIST_add`: `count` has room for every value in `src`. -pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { - for &byte in src { - *count.at_mut(usize::from(byte))? += 1; - } - Ok(()) -} - -/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. -/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. -pub(crate) fn count_simple( - count: &mut [u32], - max_symbol_value: &mut u32, - src: &[u8], -) -> Result { - let mut max_symbol = *max_symbol_value as usize; - count.sub_mut(..=max_symbol)?.fill(0); - if src.is_empty() { - *max_symbol_value = 0; - return Ok(0); - } - add(count, src)?; - while count.at(max_symbol)? == 0 { - max_symbol -= 1; - } - *max_symbol_value = max_symbol as u32; - Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) -} diff --git a/src/zstd/wip/work/ldm/zstd/compress/huf.rs b/src/zstd/wip/work/ldm/zstd/compress/huf.rs deleted file mode 100644 index 78373a89ebf0..000000000000 --- a/src/zstd/wip/work/ldm/zstd/compress/huf.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! `huf_compress.c`: Huffman coding of literals. - -use crate::common::HUF_SYMBOLVALUE_MAX; -use crate::util::SliceExt; - -/// `HUF_repeat` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum HufRepeat { - /// The table cannot be used again. - None, - /// It can, if it has a code for every symbol to encode. - Check, - /// It can. - Valid, -} - -/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the -/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in -/// its highest bits. -#[derive(Clone)] -pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); - -impl HufCTable { - pub(crate) const fn new() -> Self { - Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) - } - - /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. - #[inline] - pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { - let [header, elements @ ..] = &self.0; - let max_symbol_value = (header >> 8) as u8; - if symbol_value > u32::from(max_symbol_value) { - return 0; - } - (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 - } -} diff --git a/src/zstd/wip/work/ldm/zstd/compress/internal.rs b/src/zstd/wip/work/ldm/zstd/compress/internal.rs deleted file mode 100644 index 51157ea6c1ce..000000000000 --- a/src/zstd/wip/work/ldm/zstd/compress/internal.rs +++ /dev/null @@ -1,1212 +0,0 @@ -//! `zstd_compress_internal.h`: what the parts of the compressor share. - -use super::fse::{FseCTable, FseRepeat}; -use super::huf::{HufCTable, HufRepeat}; -use crate::common::{ - BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, - ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, -}; -use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; - -pub(crate) const K_SEARCH_STRENGTH: u32 = 8; -pub(crate) const HASH_READ_SIZE: usize = 8; -/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. -pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; - -pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; -pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; - -pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; -pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; - -/// `ZSTD_strategy` -#[repr(u8)] -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] -pub(crate) enum Strategy { - Fast = 1, - DFast = 2, - Greedy = 3, - Lazy = 4, - Lazy2 = 5, - BtLazy2 = 6, - BtOpt = 7, - BtUltra = 8, - BtUltra2 = 9, -} - -impl Strategy { - pub(crate) const fn from_int(value: i32) -> Option { - Some(match value { - 1 => Self::Fast, - 2 => Self::DFast, - 3 => Self::Greedy, - 4 => Self::Lazy, - 5 => Self::Lazy2, - 6 => Self::BtLazy2, - 7 => Self::BtOpt, - 8 => Self::BtUltra, - 9 => Self::BtUltra2, - _ => return None, - }) - } -} - -/// `ZSTD_ParamSwitch_e` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum ParamSwitch { - Auto = 0, - Enable = 1, - Disable = 2, -} - -/// `ZSTD_compressionParameters` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) struct CompressionParameters { - pub(crate) window_log: u32, - pub(crate) chain_log: u32, - pub(crate) hash_log: u32, - pub(crate) search_log: u32, - pub(crate) min_match: u32, - pub(crate) target_length: u32, - /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. - pub(crate) strategy: Option, -} - -impl CompressionParameters { - /// The strategy of parameters that are ready to be applied. - #[inline(always)] - pub(crate) fn strat(&self) -> Strategy { - self.strategy.unwrap_or(Strategy::Fast) - } -} - -/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum DictMode { - NoDict, - ExtDict, - DictMatchState, -} - -/// `ZSTD_dictTableLoadMethod_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum DictTableLoadMethod { - Fast, - Full, -} - -/// `ZSTD_tableFillPurpose_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum TableFillPurpose { - ForCCtx, - ForCDict, -} - -// ── entropy tables ──────────────────────────────────────────────────────── - -/// `ZSTD_hufCTables_t` -#[derive(Clone)] -pub(crate) struct HufCTables { - pub(crate) c_table: HufCTable, - pub(crate) repeat_mode: HufRepeat, -} - -/// `ZSTD_fseCTables_t` -#[derive(Clone)] -pub(crate) struct FseCTables { - pub(crate) offcode_c_table: FseCTable, - pub(crate) matchlength_c_table: FseCTable, - pub(crate) litlength_c_table: FseCTable, - pub(crate) offcode_repeat_mode: FseRepeat, - pub(crate) matchlength_repeat_mode: FseRepeat, - pub(crate) litlength_repeat_mode: FseRepeat, -} - -/// `ZSTD_entropyCTables_t` -#[derive(Clone)] -pub(crate) struct EntropyCTables { - pub(crate) huf: HufCTables, - pub(crate) fse: FseCTables, -} - -/// `ZSTD_compressedBlockState_t` -#[derive(Clone)] -pub(crate) struct CompressedBlockState { - pub(crate) entropy: EntropyCTables, - pub(crate) rep: [u32; ZSTD_REP_NUM], -} - -impl CompressedBlockState { - /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. - pub(crate) fn new() -> Self { - Self { - entropy: EntropyCTables { - huf: HufCTables { - c_table: HufCTable::new(), - repeat_mode: HufRepeat::None, - }, - fse: FseCTables { - offcode_c_table: FseCTable::new(), - matchlength_c_table: FseCTable::new(), - litlength_c_table: FseCTable::new(), - offcode_repeat_mode: FseRepeat::None, - matchlength_repeat_mode: FseRepeat::None, - litlength_repeat_mode: FseRepeat::None, - }, - }, - rep: REP_START_VALUE, - } - } - - /// `ZSTD_reset_compressedBlockState` - pub(crate) fn reset(&mut self) { - self.rep = REP_START_VALUE; - self.entropy.huf.repeat_mode = HufRepeat::None; - self.entropy.fse.offcode_repeat_mode = FseRepeat::None; - self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; - self.entropy.fse.litlength_repeat_mode = FseRepeat::None; - } -} - -/// `ZSTD_hufCTablesMetadata_t` -pub(crate) struct HufCTablesMetadata { - pub(crate) h_type: SymbolEncodingType, - pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], - pub(crate) huf_des_size: usize, -} - -/// `ZSTD_fseCTablesMetadata_t` -pub(crate) struct FseCTablesMetadata { - pub(crate) ll_type: SymbolEncodingType, - pub(crate) of_type: SymbolEncodingType, - pub(crate) ml_type: SymbolEncodingType, - pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], - pub(crate) fse_tables_size: usize, - /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. - pub(crate) last_count_size: usize, -} - -/// `ZSTD_entropyCTablesMetadata_t` -pub(crate) struct EntropyCTablesMetadata { - pub(crate) huf_metadata: HufCTablesMetadata, - pub(crate) fse_metadata: FseCTablesMetadata, -} - -impl EntropyCTablesMetadata { - pub(crate) const fn new() -> Self { - Self { - huf_metadata: HufCTablesMetadata { - h_type: SymbolEncodingType::Basic, - huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], - huf_des_size: 0, - }, - fse_metadata: FseCTablesMetadata { - ll_type: SymbolEncodingType::Basic, - of_type: SymbolEncodingType::Basic, - ml_type: SymbolEncodingType::Basic, - fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], - fse_tables_size: 0, - last_count_size: 0, - }, - } - } -} - -// ── sequences ───────────────────────────────────────────────────────────── - -/// `SeqDef` -#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] -pub(crate) struct SeqDef { - /// A repeat code (1 to 3), or an offset plus 3. - pub(crate) off_base: u32, - pub(crate) lit_length: u16, - /// The match length minus [`MINMATCH`]. - pub(crate) ml_base: u16, -} - -/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum LongLengthType { - None, - LiteralLength, - MatchLength, -} - -/// `SeqStore_t`: the sequences and the literals of a block. -pub(crate) struct SeqStore { - /// Never grows beyond the capacity it is given by [`Self::reserve`]. - pub(crate) sequences: Vec, - /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. - pub(crate) lit_buffer: Vec, - /// How much of `lit_buffer` is literals. - pub(crate) lit_len: usize, - pub(crate) ll_code: Vec, - pub(crate) ml_code: Vec, - pub(crate) of_code: Vec, - pub(crate) max_nb_seq: usize, - pub(crate) max_nb_lit: usize, - pub(crate) long_length_type: LongLengthType, - /// Which sequence has the long length. - pub(crate) long_length_pos: u32, -} - -/// `ZSTD_SequenceLength` -#[derive(Clone, Copy)] -pub(crate) struct SequenceLength { - pub(crate) lit_length: u32, - pub(crate) match_length: u32, -} - -impl SeqStore { - pub(crate) const fn new() -> Self { - Self { - sequences: Vec::new(), - lit_buffer: Vec::new(), - lit_len: 0, - ll_code: Vec::new(), - ml_code: Vec::new(), - of_code: Vec::new(), - max_nb_seq: 0, - max_nb_lit: 0, - long_length_type: LongLengthType::None, - long_length_pos: 0, - } - } - - /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. - pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { - fn grow(vec: &mut Vec, len: usize) -> Result<()> { - vec.clear(); - vec.try_reserve_exact(len) - .map_err(|_| ErrorCode::MemoryAllocation)?; - vec.resize(len, T::default()); - Ok(()) - } - grow(&mut self.sequences, max_nb_seq)?; - grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; - grow(&mut self.ll_code, max_nb_seq)?; - grow(&mut self.ml_code, max_nb_seq)?; - grow(&mut self.of_code, max_nb_seq)?; - self.max_nb_seq = max_nb_seq; - self.max_nb_lit = max_nb_lit; - self.reset(); - Ok(()) - } - - /// `ZSTD_resetSeqStore` - #[inline] - pub(crate) fn reset(&mut self) { - self.lit_len = 0; - self.sequences.clear(); - self.long_length_type = LongLengthType::None; - } - - #[inline] - pub(crate) fn literals(&self) -> &[u8] { - self.lit_buffer.get(..self.lit_len).unwrap_or_default() - } - - /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. - #[inline(always)] - pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { - debug_assert!(self.sequences.len() < self.max_nb_seq); - if lit_length > 0xFFFF { - self.long_length_type = LongLengthType::LiteralLength; - self.long_length_pos = self.sequences.len() as u32; - } - let ml_base = match_length.wrapping_sub(MINMATCH as usize); - if ml_base > 0xFFFF { - self.long_length_type = LongLengthType::MatchLength; - self.long_length_pos = self.sequences.len() as u32; - } - self.sequences.push(SeqDef { - off_base, - lit_length: lit_length as u16, - ml_base: ml_base as u16, - }); - } - - /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. - #[inline(always)] - pub(crate) fn store_seq( - &mut self, - lit_length: usize, - buf: &[u8], - literals: usize, - off_base: u32, - match_length: usize, - ) -> Result<()> { - match ( - buf.array_at::<16>(literals), - self.lit_buffer.array_at_mut::<16>(self.lit_len), - ) { - (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, - _ => self - .lit_buffer - .write_at(self.lit_len, buf.span(literals, lit_length)?)?, - } - self.lit_len += lit_length; - self.store_seq_only(lit_length, off_base, match_length); - Ok(()) - } - - /// Appends the literals that follow the last sequence of a block. - #[inline] - pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { - self.lit_buffer.write_at(self.lit_len, literals)?; - self.lit_len += literals.len(); - Ok(()) - } -} - -pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; -pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; -pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; - -/// `OFFSET_TO_OFFBASE`: `offset` is not 0. -#[inline(always)] -pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { - offset.wrapping_add(ZSTD_REP_NUM as u32) -} - -/// `OFFBASE_IS_OFFSET` -#[inline(always)] -pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { - off_base > ZSTD_REP_NUM as u32 -} - -/// `OFFBASE_IS_REPCODE` -#[inline(always)] -pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { - 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 -} - -/// `OFFBASE_TO_OFFSET` -#[inline(always)] -pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { - off_base.wrapping_sub(ZSTD_REP_NUM as u32) -} - -/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. -#[inline(always)] -pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { - let [rep0, rep1, rep2] = *rep; - if offbase_is_offset(off_base) { - *rep = [offbase_to_offset(off_base), rep0, rep1]; - } else { - match off_base - 1 + u32::from(ll0) { - 0 => {} - 1 => *rep = [rep1, rep0, rep2], - 2 => *rep = [rep2, rep0, rep1], - _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], - } - } -} - -/// `ZSTD_newRep` -#[inline(always)] -pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { - let mut new_reps = *rep; - update_rep(&mut new_reps, off_base, ll0); - new_reps -} - -/// `ZSTD_LLcode` -#[inline(always)] -pub(crate) fn ll_code(lit_length: u32) -> u32 { - #[rustfmt::skip] - const LL_CODE: [u8; 64] = [ - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, - ]; - const LL_DELTA_CODE: u32 = 19; - match LL_CODE.get(lit_length as usize) { - Some(&code) => u32::from(code), - None => highbit32(lit_length) + LL_DELTA_CODE, - } -} - -/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. -#[inline(always)] -pub(crate) fn ml_code(ml_base: u32) -> u32 { - #[rustfmt::skip] - const ML_CODE: [u8; 128] = [ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - ]; - const ML_DELTA_CODE: u32 = 36; - match ML_CODE.get(ml_base as usize) { - Some(&code) => u32::from(code), - None => highbit32(ml_base) + ML_DELTA_CODE, - } -} - -/// `ZSTD_noCompressBlock` -pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { - let c_block_header24 = - u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); - if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { - return Err(ErrorCode::DstSizeTooSmall); - } - write_le24(dst, 0, c_block_header24)?; - dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; - Ok(ZSTD_BLOCKHEADERSIZE + src.len()) -} - -/// `ZSTD_rleCompressBlock` -pub(crate) fn rle_compress_block( - dst: &mut [u8], - byte: u8, - src_size: usize, - last_block: bool, -) -> Result { - let c_block_header = - u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); - if dst.len() < 4 { - return Err(ErrorCode::DstSizeTooSmall); - } - write_le24(dst, 0, c_block_header)?; - dst.put(3, byte)?; - Ok(4) -} - -/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. -#[inline] -pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { - let minlog = if strat >= Strategy::BtUltra { - strat as u32 - 1 - } else { - 6 - }; - (src_size >> minlog) + 2 -} - -// ── matching ────────────────────────────────────────────────────────────── - -/// How many bytes `a` and `b` have in common at their start. -#[inline(always)] -pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { - let (a_words, _) = a.as_chunks::<8>(); - let (b_words, _) = b.as_chunks::<8>(); - let mut len = 0usize; - for (a_word, b_word) in a_words.iter().zip(b_words) { - let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); - if diff != 0 { - return len + (diff.trailing_zeros() >> 3) as usize; - } - len += 8; - } - let a_tail = a.get(len..).unwrap_or_default(); - let b_tail = b.get(len..).unwrap_or_default(); - len + a_tail - .iter() - .zip(b_tail) - .take_while(|(a_byte, b_byte)| a_byte == b_byte) - .count() -} - -/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. -#[inline(always)] -pub(crate) fn count( - buf: &[u8], - p_in: usize, - match_buf: &[u8], - p_match: usize, - p_in_limit: usize, -) -> Result { - Ok(common_prefix( - buf.sub(p_in..p_in_limit)?, - match_buf.sub(p_match..)?, - )) -} - -/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at -/// `i_start` in `buf`. -#[inline(always)] -pub(crate) fn count_2segments( - buf: &[u8], - ip: usize, - match_buf: &[u8], - match_pos: usize, - i_end: usize, - m_end: usize, - i_start: usize, -) -> Result { - let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); - if match_pos + match_length != m_end { - return Ok(match_length); - } - Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) -} - -const PRIME_3_BYTES: u32 = 506_832_829; -const PRIME_4_BYTES: u32 = 2_654_435_761; -const PRIME_5_BYTES: u64 = 889_523_592_379; -const PRIME_6_BYTES: u64 = 227_718_039_650_203; -const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; -pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; - -/// `ZSTD_hash3` -#[inline(always)] -const fn hash3(u: u32, h: u32, s: u32) -> u32 { - (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash4(u: u32, h: u32, s: u32) -> u32 { - ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash5(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash6(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash7(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -/// `ZSTD_hash8` -#[inline(always)] -pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { - (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} - -/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. -#[inline(always)] -pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { - Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) -} - -/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, -/// else 8. -#[inline(always)] -pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { - hash_ptr_salted(buf, pos, h_bits, mls, 0) -} - -/// `ZSTD_hashPtrSalted` -#[inline(always)] -pub(crate) fn hash_ptr_salted( - buf: &[u8], - pos: usize, - h_bits: u32, - mls: u32, - hash_salt: u64, -) -> Result { - Ok(match mls { - 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), - 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), - 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), - 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), - _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, - }) -} - -// ── the window ──────────────────────────────────────────────────────────── - -/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are -/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the -/// others, and an address is turned into a position in the buffer only to read. -pub(crate) type Addr = usize; - -/// The buffers that a window can have a segment in. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum BufId { - /// Where an empty window points. - None = 0, - /// The context's own buffer of input. - InBuff = 1, - /// What the caller gave to the call in progress. - User = 2, - /// The content of the dictionary, or the prefix. - Dict = 3, -} - -impl BufId { - /// The address of the first byte of the buffer. - #[inline(always)] - pub(crate) const fn start(self) -> Addr { - (self as usize + 1) << 44 - } -} - -/// The buffers themselves, for as long as a call lasts. -#[derive(Clone, Copy)] -pub(crate) struct Buffers<'a> { - pub(crate) in_buff: &'a [u8], - pub(crate) user: &'a [u8], - pub(crate) dict: &'a [u8], -} - -impl<'a> Buffers<'a> { - #[inline] - pub(crate) fn get(&self, id: BufId) -> &'a [u8] { - match id { - BufId::None => &[], - BufId::InBuff => self.in_buff, - BufId::User => self.user, - BufId::Dict => self.dict, - } - } -} - -/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. -pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; - -/// The index past which indices are brought down. -pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; -/// The most input that can be indexed at once. -pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; - -/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the -/// index is less than `dict_limit`. -#[derive(Clone, Copy, Debug)] -pub(crate) struct Window { - /// Where input has to continue to be contiguous. - pub(crate) next_src: Addr, - pub(crate) base: Addr, - pub(crate) dict_base: Addr, - /// The buffer of the segment that starts at `dict_limit`. - pub(crate) base_buf: BufId, - /// The buffer of the segment from `low_limit` to `dict_limit`. - pub(crate) dict_buf: BufId, - pub(crate) dict_limit: u32, - /// Below this, there is nothing valid. - pub(crate) low_limit: u32, - pub(crate) nb_overflow_corrections: u32, -} - -impl Window { - /// `ZSTD_window_init` - pub(crate) const fn new() -> Self { - let base = BufId::None.start(); - Self { - next_src: base + ZSTD_WINDOW_START_INDEX as usize, - base, - dict_base: base, - base_buf: BufId::None, - dict_buf: BufId::None, - dict_limit: ZSTD_WINDOW_START_INDEX, - low_limit: ZSTD_WINDOW_START_INDEX, - nb_overflow_corrections: 0, - } - } - - /// The index of `addr`, which is in the current segment or at its end. - #[inline(always)] - pub(crate) const fn index_of(&self, addr: Addr) -> u32 { - addr.wrapping_sub(self.base) as u32 - } - - /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. - pub(crate) fn clear(&mut self) { - let end = self.index_of(self.next_src); - self.low_limit = end; - self.dict_limit = end; - } - - /// `ZSTD_window_isEmpty` - pub(crate) fn is_empty(&self) -> bool { - self.dict_limit == ZSTD_WINDOW_START_INDEX - && self.low_limit == ZSTD_WINDOW_START_INDEX - && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize - } - - /// `ZSTD_window_hasExtDict` - #[inline] - pub(crate) fn has_ext_dict(&self) -> bool { - self.low_limit < self.dict_limit - } - - /// `ZSTD_window_needOverflowCorrection` - #[inline] - pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { - self.index_of(src_end) > ZSTD_CURRENT_MAX - } - - /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns - /// what to subtract from every index that is stored. - pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { - let cycle_size = 1u32 << cycle_log; - let cycle_mask = cycle_size - 1; - let curr = self.index_of(src); - let current_cycle = curr & cycle_mask; - // Indices must not fall below the start index. - let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { - cycle_size.max(ZSTD_WINDOW_START_INDEX) - } else { - 0 - }; - let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); - let correction = curr.wrapping_sub(new_current); - - self.base = self.base.wrapping_add(correction as usize); - self.dict_base = self.dict_base.wrapping_add(correction as usize); - self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { - ZSTD_WINDOW_START_INDEX - } else { - self.low_limit - correction - }; - self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { - ZSTD_WINDOW_START_INDEX - } else { - self.dict_limit - correction - }; - self.nb_overflow_corrections += 1; - correction - } - - /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the - /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. - pub(crate) fn enforce_max_dist( - &mut self, - block_end: Addr, - max_dist: u32, - loaded_dict_end: u32, - ) -> bool { - let block_end_idx = self.index_of(block_end); - if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { - let new_low_limit = block_end_idx - max_dist; - if self.low_limit < new_low_limit { - self.low_limit = new_low_limit; - } - if self.dict_limit < self.low_limit { - self.dict_limit = self.low_limit; - } - return true; - } - false - } - - /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not - /// right before the current segment, and is to be forgotten. - pub(crate) fn dict_is_invalid( - &self, - block_end: Addr, - max_dist: u32, - loaded_dict_end: u32, - ) -> bool { - let block_end_idx = self.index_of(block_end); - block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit - } - - /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not - /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. - pub(crate) fn update( - &mut self, - src: Addr, - src_buf: BufId, - src_size: usize, - force_non_contiguous: bool, - ) -> bool { - let mut contiguous = true; - if src_size == 0 { - return contiguous; - } - if src != self.next_src || force_non_contiguous { - let distance_from_base = self.next_src.wrapping_sub(self.base); - self.low_limit = self.dict_limit; - self.dict_limit = distance_from_base as u32; - self.dict_base = self.base; - self.dict_buf = self.base_buf; - self.base = src.wrapping_sub(distance_from_base); - self.base_buf = src_buf; - if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { - // Too small to be of use. - self.low_limit = self.dict_limit; - } - contiguous = false; - } - self.next_src = src + src_size; - // Where the input overlaps the older segment, it has overwritten it. - if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) - & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) - { - let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); - self.low_limit = if high_input_idx > self.dict_limit as usize { - self.dict_limit - } else { - high_input_idx as u32 - }; - } - contiguous - } -} - -/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes -/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. -#[inline(always)] -pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { - prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 -} - -pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; -pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; - -/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. -#[inline(always)] -pub(crate) fn write_tagged_index( - hash_table: &mut [u32], - hash_and_tag: usize, - index: u32, -) -> Result<()> { - let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; - hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) -} - -/// `ZSTD_comparePackedTags` -#[inline(always)] -pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { - (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) - == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) -} - -// ── the state of the match finders ──────────────────────────────────────── - -/// `ZSTD_match_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct Match { - /// As [`SeqDef::off_base`]. - pub(crate) off: u32, - pub(crate) len: u32, -} - -/// `rawSeq`: a sequence found by long distance matching. -#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] -pub(crate) struct RawSeq { - pub(crate) offset: u32, - pub(crate) lit_length: u32, - pub(crate) match_length: u32, -} - -/// `RawSeqStore_t` -#[derive(Clone, Default)] -pub(crate) struct RawSeqStore { - /// `capacity` of them, of which `size` are in use. - pub(crate) seq: Vec, - /// The sequence that is being read. - pub(crate) pos: usize, - /// How many of its bytes, literals first, are read. - pub(crate) pos_in_sequence: usize, - pub(crate) size: usize, - pub(crate) capacity: usize, -} - -/// `ZSTD_optimal_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct Optimal { - /// The cost of getting to this position. - pub(crate) price: i32, - pub(crate) off: u32, - pub(crate) mlen: u32, - pub(crate) litlen: u32, - pub(crate) rep: [u32; ZSTD_REP_NUM], -} - -/// `ZSTD_OptPrice_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum OptPrice { - Dynamic, - Predef, -} - -/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. -pub(crate) struct OptState { - pub(crate) lit_freq: Vec, - pub(crate) lit_length_freq: Vec, - pub(crate) match_length_freq: Vec, - pub(crate) off_code_freq: Vec, - pub(crate) match_table: Vec, - pub(crate) price_table: Vec, - pub(crate) lit_sum: u32, - pub(crate) lit_length_sum: u32, - pub(crate) match_length_sum: u32, - pub(crate) off_code_sum: u32, - pub(crate) lit_sum_base_price: u32, - pub(crate) lit_length_sum_base_price: u32, - pub(crate) match_length_sum_base_price: u32, - pub(crate) off_code_sum_base_price: u32, - pub(crate) price_type: OptPrice, - pub(crate) literal_compression_mode: ParamSwitch, -} - -impl OptState { - pub(crate) const fn new() -> Self { - Self { - lit_freq: Vec::new(), - lit_length_freq: Vec::new(), - match_length_freq: Vec::new(), - off_code_freq: Vec::new(), - match_table: Vec::new(), - price_table: Vec::new(), - lit_sum: 0, - lit_length_sum: 0, - match_length_sum: 0, - off_code_sum: 0, - lit_sum_base_price: 0, - lit_length_sum_base_price: 0, - match_length_sum_base_price: 0, - off_code_sum_base_price: 0, - price_type: OptPrice::Dynamic, - literal_compression_mode: ParamSwitch::Auto, - } - } -} - -pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; - -/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures -/// own, are in [`BlockSrc`]. -pub(crate) struct MatchState { - pub(crate) window: Window, - /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. - pub(crate) loaded_dict_end: u32, - /// The index from which the tables are still to be filled. - pub(crate) next_to_update: u32, - /// 0 if there is no table of 3-byte hashes. - pub(crate) hash_log3: u32, - pub(crate) row_hash_log: u32, - /// For the row-based match finder: a byte of hash for each entry of `hash_table`. - pub(crate) tag_table: Vec, - pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], - /// Makes what a previous frame left in `tag_table` meaningless. - pub(crate) hash_salt: u64, - pub(crate) hash_salt_entropy: u32, - pub(crate) hash_table: Vec, - pub(crate) hash_table3: Vec, - pub(crate) chain_table: Vec, - pub(crate) force_non_contiguous: bool, - pub(crate) opt: OptState, - /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. - pub(crate) has_dict_match_state: bool, - pub(crate) c_params: CompressionParameters, - pub(crate) lazy_skipping: bool, -} - -impl MatchState { - pub(crate) const fn new() -> Self { - Self { - window: Window::new(), - loaded_dict_end: 0, - next_to_update: 0, - hash_log3: 0, - row_hash_log: 0, - tag_table: Vec::new(), - hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], - hash_salt: 0, - hash_salt_entropy: 0, - hash_table: Vec::new(), - hash_table3: Vec::new(), - chain_table: Vec::new(), - force_non_contiguous: false, - opt: OptState::new(), - has_dict_match_state: false, - c_params: CompressionParameters { - window_log: 0, - chain_log: 0, - hash_log: 0, - search_log: 0, - min_match: 0, - target_length: 0, - strategy: None, - }, - lazy_skipping: false, - } - } - - /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. - pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { - if self.window.has_ext_dict() { - return DictMode::ExtDict; - } - if dms.is_some() { - DictMode::DictMatchState - } else { - DictMode::NoDict - } - } - - /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. - #[inline(always)] - pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { - let max_distance = 1u32 << window_log; - let lowest_valid = self.window.low_limit; - let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { - curr - max_distance - } else { - lowest_valid - }; - // A dictionary is valid as a whole until it is invalidated as a whole. - if self.loaded_dict_end != 0 { - lowest_valid - } else { - within_window - } - } - - /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. - #[inline(always)] - pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { - let max_distance = 1u32 << window_log; - let lowest_valid = self.window.dict_limit; - let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { - curr - max_distance - } else { - lowest_valid - }; - if self.loaded_dict_end != 0 { - lowest_valid - } else { - within_window - } - } -} - -/// The match state of a dictionary, and where its indices lead. -#[derive(Clone, Copy)] -pub(crate) struct DictMatchState<'a> { - pub(crate) ms: &'a MatchState, - /// The buffer its window is in, which is one segment. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, - /// `window.nextSrc`, as a position in `buf`. - pub(crate) end: usize, -} - -/// The current segment of a window, for the functions that index a dictionary. -#[derive(Clone, Copy)] -pub(crate) struct Prefix<'a> { - /// The buffer that the segment is in. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, -} - -impl Prefix<'_> { - /// The index of the position `pos` of `buf`. - #[inline(always)] - pub(crate) const fn index_of(&self, pos: usize) -> u32 { - pos.wrapping_sub(self.base) as u32 - } - - /// The position in `buf` of `index`. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } -} - -/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. -#[derive(Clone, Copy)] -pub(crate) struct BlockSrc<'a> { - /// The buffer that the current segment of the window is in, and so the block. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, - /// The buffer that the older segment of the window is in. - pub(crate) dict_buf: &'a [u8], - /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. - pub(crate) dict_base: usize, - /// Where the block starts in `buf`. - pub(crate) start: usize, - /// Where it ends. - pub(crate) end: usize, - pub(crate) dms: Option>, - /// `opt.symbolCosts`: the tables of the previous block. - pub(crate) symbol_costs: &'a EntropyCTables, - /// `ldmSeqStore`: matches that the optimal parser is to consider. - pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, -} - -impl<'a> BlockSrc<'a> { - #[inline(always)] - pub(crate) const fn prefix(&self) -> Prefix<'a> { - Prefix { - buf: self.buf, - base: self.base, - } - } - - /// The index of the position `pos` of `buf`. - #[inline(always)] - pub(crate) const fn index_of(&self, pos: usize) -> u32 { - pos.wrapping_sub(self.base) as u32 - } - - /// The position in `buf` of `index`, which is in the current segment. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } - - /// The position in `dict_buf` of `index`, which is in the older segment. - #[inline(always)] - pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { - self.dict_base.wrapping_add(index as usize) - } -} - -impl DictMatchState<'_> { - /// The position in `buf` of `index`. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } -} - -// ── long distance matching ──────────────────────────────────────────────── - -/// `ldmEntry_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct LdmEntry { - pub(crate) offset: u32, - pub(crate) checksum: u32, -} - -/// `ldmParams_t` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) struct LdmParams { - pub(crate) enable_ldm: ParamSwitch, - pub(crate) hash_log: u32, - pub(crate) bucket_size_log: u32, - pub(crate) min_match_length: u32, - pub(crate) hash_rate_log: u32, - pub(crate) window_log: u32, -} - -/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. -pub(crate) struct LdmState { - pub(crate) window: Window, - pub(crate) hash_table: Vec, - pub(crate) loaded_dict_end: u32, - /// The next entry to replace in each bucket. - pub(crate) bucket_offsets: Vec, -} - -impl LdmState { - pub(crate) const fn new() -> Self { - Self { - window: Window::new(), - hash_table: Vec::new(), - loaded_dict_end: 0, - bucket_offsets: Vec::new(), - } - } -} diff --git a/src/zstd/wip/work/ldm/zstd/compress/lazy.rs b/src/zstd/wip/work/ldm/zstd/compress/lazy.rs deleted file mode 100644 index 94dbe308ce79..000000000000 --- a/src/zstd/wip/work/ldm/zstd/compress/lazy.rs +++ /dev/null @@ -1,1353 +0,0 @@ -//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. - -use super::internal::{ - BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, - SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, - hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, - offset_to_offbase, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32}; - -const K_LAZY_SKIPPING_STEP: usize = 8; - -// `ZSTD_dictMode_e`, as an argument of a const generic. -const NO_DICT: u8 = DictMode::NoDict as u8; -const EXT_DICT: u8 = DictMode::ExtDict as u8; -const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; - -/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. -#[inline(always)] -const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { - dms.end.wrapping_sub(dms.base) as u32 -} - -// ── binary tree search ──────────────────────────────────────────────────── - -/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. -#[inline(always)] -fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { - let hash_log = ms.c_params.hash_log; - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let bt = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); - *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; - } - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. -fn insert_dubt1( - ms: &mut MatchState, - src: &BlockSrc<'_>, - curr: u32, - input_end: usize, - mut nb_compares: u32, - bt_low: u32, -) -> Result<()> { - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let (ip_buf, ip, iend) = if curr >= dict_limit { - (src.buf, src.pos_of(curr), input_end) - } else { - (src.dict_buf, src.dict_pos_of(curr), dict_end) - }; - let window_valid = ms.window.low_limit; - let max_distance = 1u32 << ms.c_params.window_log; - let window_low = if curr.wrapping_sub(window_valid) > max_distance { - curr - max_distance - } else { - window_valid - }; - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - // The next sorted candidate. The other slot has the previous unsorted one. - let mut match_index = bt.at(smaller_ptr)?; - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let in_prefix = - DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; - // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = if in_prefix { - (src.buf, src.pos_of(match_index)) - } else { - (src.dict_buf, src.dict_pos_of(match_index)) - }; - - if in_prefix || curr < dict_limit { - // Both are in the same segment. - match_length += count( - ip_buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - match_length += count_2segments( - ip_buf, - ip + match_length, - match_buf, - match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if match_index as usize + match_length >= dict_limit as usize { - (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); - } - } - - if ip + match_length == iend { - // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. - break; - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0) -} - -/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. -fn dubt_find_better_dict_match( - ms: &MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mut best_length: usize, - mut nb_compares: u32, - mls: u32, -) -> Result { - let dms = src.dms.ok_or(BUG)?; - let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(h)?; - - let prefix_start = src.pos_of(ms.window.dict_limit); - let curr = src.index_of(ip); - let dict_high_limit = dms_end_index(&dms); - let dict_low_limit = dms.ms.window.low_limit; - let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); - - let dict_bt = dms.ms.chain_table.as_slice(); - let bt_log = dms.ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { - dict_low_limit - } else { - dict_high_limit - bt_mask - }; - - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - - while nb_compares != 0 && dict_match_index > dict_low_limit { - let next_ptr = 2 * (dict_match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let match_index = dict_match_index.wrapping_add(dict_index_delta); - match_length += count_2segments( - src.buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - iend, - dms.end, - prefix_start, - )?; - - if match_length > best_length { - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(offset_ptr.wrapping_add(1)) as i32 - { - best_length = match_length; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // No way to know which is smaller. - break; - } - } - - if dict_match_index <= bt_low { - // Beyond the tree. - break; - } - let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { - let match_pos = src.pos_of(match_index); - src.buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? - }; - if match_byte < src.buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dict_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dict_bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - Ok(best_length) -} - -/// `ZSTD_DUBT_findBestMatch` -#[inline(always)] -fn dubt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - off_base_ptr: &mut u32, - mls: u32, -) -> Result { - let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; - let mut match_index = ms.hash_table.at(h)?; - - let curr = src.index_of(ip); - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = curr.saturating_sub(bt_mask); - let unsort_limit = bt_low.max(window_low); - let mut nb_compares = 1u32 << ms.c_params.search_log; - let mut nb_candidates = nb_compares; - let mut previous_candidate = 0u32; - - // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. - let bt = ms.chain_table.as_mut_slice(); - let mut next_candidate = 2 * (match_index & bt_mask) as usize; - while match_index > unsort_limit - && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK - && nb_candidates > 1 - { - bt.put(next_candidate + 1, previous_candidate)?; - previous_candidate = match_index; - match_index = bt.at(next_candidate)?; - next_candidate = 2 * (match_index & bt_mask) as usize; - nb_candidates -= 1; - } - - // If the last candidate is still unsorted, cut the list there. - if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { - *bt.array_at_mut(next_candidate)? = [0, 0]; - } - - // Sort the candidates, the oldest first. - match_index = previous_candidate; - while match_index != 0 { - let next_candidate_idx = ms - .chain_table - .at(2 * (match_index & bt_mask) as usize + 1)?; - insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; - match_index = next_candidate_idx; - nb_candidates += 1; - } - - // Find the longest match. - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - let mut match_end_idx = curr + 8 + 1; - let mut best_length = 0usize; - - match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); - - if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - src.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - let dict_match_pos = src.dict_pos_of(match_index); - match_length += count_2segments( - src.buf, - ip + match_length, - src.dict_buf, - dict_match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - (match_buf, match_pos) = (src.dict_buf, dict_match_pos); - } - } - - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index + match_length as u32; - } - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(*off_base_ptr) as i32 - { - best_length = match_length; - *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // Equal: no way to know which is smaller. - if DICT_MODE == DICT_MATCH_STATE { - // Nothing in the dictionary can be longer. - nb_compares = 0; - } - break; - } - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0)?; - - if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { - best_length = dubt_find_better_dict_match( - ms, - src, - ip, - iend, - off_base_ptr, - best_length, - nb_compares, - mls, - )?; - } - - // Skip repetitive patterns. - ms.next_to_update = match_end_idx - 8; - Ok(best_length) -} - -/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. -/// `off_base_ptr` is written to if there is one. -#[inline(never)] -fn bt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - off_base_ptr: &mut u32, -) -> Result { - if ip < src.pos_of(ms.next_to_update) { - // In an area that is skipped. - return Ok(0); - } - update_dubt(ms, src.prefix(), ip, MLS)?; - dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) -} - -// ── hash chains ─────────────────────────────────────────────────────────── - -/// `ZSTD_insertAndFindFirstIndex_internal` -#[inline(always)] -fn insert_and_find_first_index_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - lazy_skipping: bool, -) -> Result { - let hash_log = ms.c_params.hash_log; - let chain_mask = (1u32 << ms.c_params.chain_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let chain_table = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); - chain_table.put((idx & chain_mask) as usize, next_in_chain)?; - // While skipping, not every position is inserted. - if lazy_skipping { - break; - } - } - - ms.next_to_update = target; - hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) -} - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) -} - -/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn hc_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let chain_size = 1u32 << ms.c_params.chain_log; - let chain_mask = chain_size - 1; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let min_chain = curr.saturating_sub(chain_size); - let mut nb_attempts = 1u32 << ms.c_params.search_log; - let mut ml = 4 - 1; - - let mut match_index = - insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; - let chain_table = ms.chain_table.as_slice(); - - while match_index >= low_limit && nb_attempts > 0 { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - - if match_index <= min_chain { - break; - } - match_index = chain_table.at((match_index & chain_mask) as usize)?; - nb_attempts -= 1; - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_chain_table = dms.ms.chain_table.as_slice(); - let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; - let dms_chain_mask = dms_chain_size - 1; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_size = dms_end_index(&dms); - let dms_index_delta = dict_limit.wrapping_sub(dms_size); - let dms_min_chain = dms_size.saturating_sub(dms_chain_size); - - let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; - match_index = dms.ms.hash_table.at(h)?; - - while match_index >= dms_lowest_index && nb_attempts > 0 { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - - if match_index <= dms_min_chain { - break; - } - match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; - nb_attempts -= 1; - } - } - - Ok(ml) -} - -// ── row-based match finder ──────────────────────────────────────────────── -// -// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits -// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag -// table is the "head": which entry was written last. - -pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; -const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; -const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; - -/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. -#[inline(always)] -fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { - let mut next = u32::from(*head).wrapping_sub(1) & row_mask; - if next == 0 { - next = row_mask; - } - *head = next as u8; - next -} - -/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. -#[inline(always)] -fn row_fill_hash_cache( - ms: &mut MatchState, - prefix: Prefix<'_>, - mls: u32, - idx: u32, - i_limit: usize, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); - let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; - - for idx in idx..lim { - let hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - ms.hash_salt, - )? as u32; - let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; - ms.hash_cache.put(cached, hash)?; - } - Ok(()) -} - -/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes -/// further takes its place. -#[inline(always)] -fn row_next_cached_hash( - cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], - prefix: Prefix<'_>, - idx: u32, - hash_log: u32, - mls: u32, - hash_salt: u64, -) -> Result { - let new_hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32; - let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; - Ok(core::mem::replace(cached, new_hash)) -} - -/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. -#[inline(always)] -fn row_update_internal_impl( - ms: &mut MatchState, - prefix: Prefix<'_>, - update_start_idx: u32, - update_end_idx: u32, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let hash_salt = ms.hash_salt; - let hash_table = ms.hash_table.as_mut_slice(); - let tag_table = ms.tag_table.as_mut_slice(); - - for idx in update_start_idx..update_end_idx { - let hash = if use_cache { - row_next_cached_hash(&mut ms.hash_cache, prefix, idx, hash_log, mls, hash_salt)? - } else { - hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32 - }; - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; - tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; - hash_table.put(rel_row + pos, idx)?; - } - Ok(()) -} - -/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is -/// when compressing, not all of them if they are many. -#[inline(always)] -fn row_update_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - const K_SKIP_THRESHOLD: u32 = 384; - const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; - const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; - - let mut idx = ms.next_to_update; - let target = prefix.index_of(ip); - - if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { - let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; - row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; - idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; - row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; - } - row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the -/// row-based match finder. -pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { - let row_log = ms.c_params.search_log.clamp(4, 6); - let row_mask = (1u32 << row_log) - 1; - let mls = ms.c_params.min_match.min(6); - row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) -} - -/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. -#[inline(always)] -fn row_get_match_mask( - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - head: u32, -) -> u64 { - let mut matches = 0u64; - let (chunks, _) = tag_row.as_chunks::<16>(); - for (n, chunk) in chunks.iter().enumerate() { - let mut chunk_matches = 0u16; - for (i, &entry) in chunk.iter().enumerate() { - chunk_matches |= u16::from(entry == tag) << i; - } - matches |= u64::from(chunk_matches) << (16 * n); - } - match ROW_ENTRIES { - 16 => u64::from((matches as u16).rotate_right(head)), - 32 => u64::from((matches as u32).rotate_right(head)), - _ => matches.rotate_right(head), - } -} - -/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, -/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than -/// `low_limit`. Returns how many there are. -#[inline(always)] -fn row_collect_matches( - row: &[u32; ROW_ENTRIES], - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - low_limit: u32, - nb_attempts: &mut u32, - match_buffer: &mut [u32; ROW_ENTRIES], -) -> Result { - let row_mask = ROW_ENTRIES as u32 - 1; - let head = u32::from(tag_row.at(0)?) & row_mask; - let mut num_matches = 0usize; - let mut matches = row_get_match_mask(tag_row, tag, head); - - while matches > 0 && *nb_attempts > 0 { - let match_pos = (head + matches.trailing_zeros()) & row_mask; - matches &= matches - 1; - if match_pos == 0 { - // The head, which happens to be the same as the tag. - continue; - } - let match_index = row.at(match_pos as usize)?; - if match_index < low_limit { - break; - } - match_buffer.put(num_matches, match_index)?; - num_matches += 1; - *nb_attempts -= 1; - } - Ok(num_matches) -} - -/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn row_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let hash_log = ms.row_hash_log; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let row_log = ROW_ENTRIES.trailing_zeros(); - let row_mask = ROW_ENTRIES as u32 - 1; - // There are no more searches than entries in a row. - let capped_search_log = ms.c_params.search_log.min(row_log); - let hash_salt = ms.hash_salt; - let mut nb_attempts = 1u32 << capped_search_log; - let mut ml = 4 - 1; - let mut match_buffer = [0u32; ROW_ENTRIES]; - - // Update the tables up to, and not including, `ip`. - let hash = if ms.lazy_skipping { - // While skipping, not every position is inserted. - ms.next_to_update = curr; - hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 - } else { - let prefix = src.prefix(); - row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; - row_next_cached_hash(&mut ms.hash_cache, prefix, curr, hash_log, MLS, hash_salt)? - }; - ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); - - { - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; - let row = ms.hash_table.array_at_mut::(rel_row)?; - let tag_row = ms.tag_table.array_at_mut::(rel_row)?; - let num_matches = row_collect_matches( - row, - tag_row, - tag, - low_limit, - &mut nb_attempts, - &mut match_buffer, - )?; - - // Insert `ip`. - let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; - tag_row.put(pos, tag)?; - row.put(pos, ms.next_to_update)?; - ms.next_to_update += 1; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - } - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); - - let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; - let dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; - let dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let num_matches = row_collect_matches( - dms.ms.hash_table.array_at::(dms_rel_row)?, - dms.ms.tag_table.array_at::(dms_rel_row)?, - (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, - dms_lowest_index, - &mut nb_attempts, - &mut match_buffer, - )?; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - } - } - - Ok(ml) -} - -// ── the parsers ─────────────────────────────────────────────────────────── - -// `searchMethod_e`, as an argument of a const generic. -const SEARCH_HASH_CHAIN: u8 = 0; -const SEARCH_BINARY_TREE: u8 = 1; -const SEARCH_ROW_HASH: u8 = 2; - -/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. -#[inline(always)] -fn search_max( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mls: u32, - row_log: u32, -) -> Result { - match SEARCH_METHOD { - SEARCH_HASH_CHAIN => match mls { - 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - SEARCH_BINARY_TREE => match mls { - 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - _ => match (mls, row_log) { - (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - } -} - -/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match -/// state is searched. -#[derive(Clone, Copy)] -struct DictSegment<'a> { - buf: &'a [u8], - /// Its sum with an index of the window is a position in `buf`. It wraps around. - base: usize, - /// Where the segment starts in `buf`. - start: usize, - /// Where it ends. - end: usize, -} - -/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. -#[inline(always)] -fn rep_length( - ms: &MatchState, - src: &BlockSrc<'_>, - dict: &DictSegment<'_>, - ip: usize, - offset: u32, -) -> Result { - let buf = src.buf; - let iend = src.end; - if DICT_MODE == NO_DICT { - let rep_match = ip.wrapping_sub(offset as usize); - if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { - return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); - } - return Ok(0); - } - - let dict_limit = ms.window.dict_limit; - let curr = src.index_of(ip); - let rep_index = curr.wrapping_sub(offset); - if !index_overlap_check(dict_limit, rep_index) { - return Ok(0); - } - if DICT_MODE == EXT_DICT { - // The offset may lead to what is no longer in the window. - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - if offset > curr.wrapping_sub(window_low) { - return Ok(0); - } - } - let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { - let rep_match = dict.base.wrapping_add(rep_index as usize); - (dict.buf, rep_match, dict.end) - } else { - (buf, src.pos_of(rep_index), iend) - }; - if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { - return Ok(0); - } - let prefix_start = src.pos_of(dict_limit); - Ok(count_2segments( - buf, - ip + 4, - rep_buf, - rep_match + 4, - iend, - rep_end, - prefix_start, - )? + 4) -} - -/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions -/// further a better match is looked for, once there is one. -#[inline(never)] -fn compress_block_lazy_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let buf = src.buf; - let istart = src.start; - let mut ip = istart; - let mut anchor = istart; - let iend = src.end; - let prefix_lowest_index = ms.window.dict_limit; - let prefix_lowest = src.pos_of(prefix_lowest_index); - let mls = ms.c_params.min_match.clamp(4, 6); - let row_log = ms.c_params.search_log.clamp(4, 6); - - let [rep_0, rep_1, _] = rep; - let mut offset_1 = *rep_0; - let mut offset_2 = *rep_1; - let mut offset_saved_1 = 0u32; - let mut offset_saved_2 = 0u32; - - ms.lazy_skipping = false; - - let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { - 8 + ZSTD_ROW_HASH_CACHE_SIZE - } else { - 8 - }; - let Some(ilimit) = iend.checked_sub(margin) else { - // There is no position to search, nor to hash. - return Ok(iend - istart); - }; - - let dict = match DICT_MODE { - NO_DICT => DictSegment { - buf: &[], - base: 0, - start: 0, - end: 0, - }, - EXT_DICT => DictSegment { - buf: src.dict_buf, - base: src.dict_base, - start: src.dict_pos_of(ms.window.low_limit), - end: src.dict_pos_of(prefix_lowest_index), - }, - _ => { - let dms = src.dms.ok_or(BUG)?; - let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); - DictSegment { - buf: dms.buf, - base: dms.base.wrapping_sub(dict_index_delta as usize), - start: dms.pos_of(dms.ms.window.dict_limit), - end: dms.end, - } - } - }; - - if DICT_MODE == EXT_DICT { - ip += usize::from(ip == prefix_lowest); - } else { - let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); - ip += usize::from(dict_and_prefix_length == 0); - } - if DICT_MODE == NO_DICT { - let curr = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); - let max_rep = curr - window_low; - if offset_2 > max_rep { - offset_saved_2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved_1 = offset_1; - offset_1 = 0; - } - } - - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; - } - - 'parse: while ip < ilimit { - let mut match_length; - let mut off_base = REPCODE1_TO_OFFBASE; - let mut start = ip + 1; - - 'found: { - // Check the repeat offset. - match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; - if DEPTH == 0 && match_length != 0 { - break 'found; - } - - // First search. - { - let mut off_base_found = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut off_base_found, - mls, - row_log, - )?; - if ml2 > match_length { - match_length = ml2; - start = ip; - off_base = off_base_found; - } - } - - if match_length < 4 { - // Jump faster over incompressible sections. - let step = (ip - anchor) >> K_SEARCH_STRENGTH; - ip += step + 1; - // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of - // no use out of the tables. As in the C code, it does not start at the same step in all modes. - ms.lazy_skipping = if DICT_MODE == EXT_DICT { - step > K_LAZY_SKIPPING_STEP - } else { - step + 1 > K_LAZY_SKIPPING_STEP - }; - continue 'parse; - } - - // Look for something better at the next positions. - if DEPTH >= 1 { - while ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 3) as i32; - let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - - // Nothing better: try one position further. - if DEPTH == 2 && ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 4) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - } - break; - } - } - - // Catch up: the match may start earlier. - if offbase_is_offset(off_base) { - let offset = offbase_to_offset(off_base); - let match_index = src.index_of(start).wrapping_sub(offset); - let (match_buf, mut match_pos, m_start) = - if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { - let match_pos = dict.base.wrapping_add(match_index as usize); - (dict.buf, match_pos, dict.start) - } else { - (buf, src.pos_of(match_index), prefix_lowest) - }; - while start > anchor - && match_pos > m_start - && buf.at(start - 1)? == match_buf.at(match_pos - 1)? - { - start -= 1; - match_pos -= 1; - match_length += 1; - } - offset_2 = offset_1; - offset_1 = offset; - } - } - - // Store the sequence. - seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; - ip = start + match_length; - anchor = ip; - - if ms.lazy_skipping { - // There was a match: every position is inserted again. - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; - } - ms.lazy_skipping = false; - } - - // Check the second repeat offset right away. - while ip <= ilimit { - match_length = rep_length::(ms, src, &dict, ip, offset_2)?; - if match_length == 0 { - break; - } - (offset_1, offset_2) = (offset_2, offset_1); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; - ip += match_length; - anchor = ip; - } - } - - // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. - if offset_saved_1 != 0 && offset_1 != 0 { - offset_saved_2 = offset_saved_1; - } - - // Save the repeat offsets for the next block. - *rep_0 = if offset_1 != 0 { - offset_1 - } else { - offset_saved_1 - }; - *rep_1 = if offset_2 != 0 { - offset_2 - } else { - offset_saved_2 - }; - - // The size of the last literals. - Ok(iend - anchor) -} - -/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. -#[inline(always)] -fn compress_block_lazy_any_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - match dict_mode { - DictMode::NoDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::ExtDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - compress_block_lazy_generic::( - ms, seq_store, rep, src, - ) - } - } -} - -/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: -/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. -pub(crate) fn compress_block_lazy( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row: bool, - dict_mode: DictMode, -) -> Result { - match (strategy, use_row) { - (Strategy::Greedy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Greedy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( - ms, seq_store, rep, src, dict_mode, - ), - _ => Err(BUG), - } -} diff --git a/src/zstd/wip/work/ldm/zstd/compress/ldm.rs b/src/zstd/wip/work/ldm/zstd/compress/ldm.rs deleted file mode 100644 index 7f060fcda748..000000000000 --- a/src/zstd/wip/work/ldm/zstd/compress/ldm.rs +++ /dev/null @@ -1,779 +0,0 @@ -//! `zstd_ldm.c`: long distance matching. - -use super::internal::{ - BlockSrc, Buffers, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, LdmEntry, - LdmParams, LdmState, MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, - TableFillPurpose, count, count_2segments, offset_to_offbase, -}; -use super::{compress_block, double_fast, fast}; -use crate::common::ZSTD_REP_NUM; -use crate::common::xxhash::Xxh64; -use crate::error::{ErrorCode, Result}; -use crate::util::{BUG, SliceExt}; - -const ZSTD_HASHLOG_MIN: u32 = 6; -const ZSTD_HASHLOG_MAX: u32 = 30; -const ZSTD_LDM_BUCKETSIZELOG_MAX: u32 = 8; - -const LDM_BUCKET_SIZE_LOG: u32 = 4; -const LDM_MIN_MATCH_LENGTH: u32 = 64; -const LDM_BATCH_SIZE: usize = 64; - -/// `ZSTD_ldm_gearTab` -#[rustfmt::skip] -static GEAR_TAB: [u64; 256] = [ - 0xf5b8_f72c_5f77_775c, 0x8493_5f26_6b7a_c412, 0xb647_ada9_ca73_0ccc, 0xb065_bb4b_114f_b1de, - 0x3458_4e7e_8c3a_9fd0, 0x4e97_e17c_6ae2_6b05, 0x3a03_d743_bc99_a604, 0xcecd_0424_22c4_044f, - 0x76de_76c5_8524_259e, 0x9c85_28f6_5bad_eaca, 0x8656_3706_e209_7529, 0x2902_475f_a375_d889, - 0xafb3_2a97_39a5_ebe6, 0xce27_14da_3883_e639, 0x021e_af82_1722_e69e, 0x0003_7b62_8620_b628, - 0x049a_8d45_5d88_caf5, 0x8556_d711_e695_8140, 0x04f7_ae74_fc60_5c1f, 0x829f_0c34_68bd_3a20, - 0x4ffd_c885_c625_179e, 0x8473_de04_8a3d_af1b, 0x5100_8822_b056_46b2, 0x69d7_5d12_b2d1_cc5f, - 0x8c9d_4a19_1591_54bc, 0xc3cc_10f4_abbd_4003, 0xd06d_dc1c_ecb9_7391, 0xbe48_e6e7_ed80_302e, - 0x3481_db31_cee0_3547, 0xacc3_f67c_daa1_d210, 0x65cb_771d_8c7f_96cc, 0x8eb2_7177_0557_23dd, - 0xc789_950d_44cd_94be, 0x934f_eadc_3700_b12b, 0x5e48_5f11_edbd_f182, 0x1e2e_2a46_fd64_767a, - 0x2969_ca71_d82e_fa7c, 0x9d46_e993_5ebb_ba2e, 0xe056_b67e_05e6_822b, 0x94d7_3f55_739d_03a0, - 0xcd70_10bd_b69b_5a03, 0x455e_f9fc_d79b_82f4, 0x869c_b54a_8749_c161, 0x38d1_a4fa_6185_d225, - 0xb475_166f_94bb_e9bb, 0xa414_3548_7209_59f1, 0x7aed_4780_ba6b_26ba, 0xd0ce_2644_39e0_2312, - 0x8436_6d74_6078_d508, 0xa8ce_973c_72ed_17be, 0x21c3_23a2_9a43_0b01, 0x9962_d617_e3af_80ee, - 0xab0c_e91d_9c8c_f75b, 0x530e_8ee6_d19a_4dbc, 0x2ef6_8c0c_f53f_5d72, 0xc03a_6816_40a8_5506, - 0x496e_4e9f_9c31_0967, 0x7858_0472_b59b_14a0, 0x2738_24c2_3b38_8577, 0x66bf_923a_d45c_b553, - 0x47ae_1a5a_2492_ba86, 0x35e3_0456_9e22_9659, 0x4765_182a_4687_0b6f, 0x6cba_b625_e909_9412, - 0xddac_9a2e_5985_22c1, 0x7172_086e_6666_24f2, 0xdf50_03ca_503b_7837, 0x88c0_c1db_7856_3d09, - 0x58d5_1865_acfc_289d, 0x1776_71ae_c652_24f1, 0xfb79_d8a2_41e9_67d7, 0x2be1_e101_cad9_a49a, - 0x6625_682f_6e29_186b, 0x3995_5345_7ac0_6e50, 0x035d_ffb4_c23a_bb74, 0x429d_b259_1f54_aade, - 0xc528_02a8_037d_1009, 0x6acb_2738_1f0b_25f3, 0xf45e_2551_ee4f_823b, 0x8b0e_a2d9_9580_c2f7, - 0x3bed_519c_bcb4_e1e1, 0x0ff4_5282_3dbb_010a, 0x9d42_ed61_4f3d_d267, 0x5b93_13c0_6257_c57b, - 0xa114_b800_8b5e_1442, 0xc1fe_311c_11c1_3d4b, 0x66e8_763e_a34c_5568, 0x8b98_2af1_c262_f05d, - 0xee88_76fa_aa75_fbb7, 0x8a62_a4d0_d172_bb2a, 0xc13d_94a3_b744_9a97, 0x6dbb_ba9d_c15d_037c, - 0xc786_101f_1d92_e0f1, 0xd786_81a9_07a0_b79b, 0xf61a_af29_62c9_abb9, 0x2cfd_16fc_d3cb_7ad9, - 0x868c_5b67_4462_4d21, 0x25e6_5089_9c74_ddd7, 0xba04_2af4_a7c3_7463, 0x4eb1_a539_465a_3eca, - 0xbe09_dbf0_3b05_d5ca, 0x774e_5a36_2b54_72ba, 0x47a1_2212_29d1_83cd, 0x504b_0ca1_8ef5_a2df, - 0xdffb_dfbd_e245_6eb9, 0x46cd_2b2f_bee3_4634, 0xf2ae_f8fe_819d_98c3, 0x357f_5276_d459_9d61, - 0x24a5_4838_79c4_53e3, 0x0880_2688_9192_b4b9, 0x28da_9667_1782_dbec, 0x4ef3_7c40_588e_9aaa, - 0x8837_b906_51bc_9fb3, 0xc164_f741_d3f0_e5d6, 0xbc13_5a0a_704b_70ba, 0x069c_d868_f762_2ada, - 0xbc37_ba89_e0b9_c0ab, 0x47c1_4a01_3235_52f6, 0x4f00_794b_acee_98bb, 0x7107_de7d_637a_69d5, - 0x88af_793b_b6f2_255e, 0xf3c6_466b_8799_b598, 0xc288_c616_aa7f_3b59, 0x81ca_63cf_42fc_a3fd, - 0x88d8_5ace_36a2_674b, 0x0d05_6bd3_7923_89e7, 0xe55c_396c_4e9d_d32d, 0xbefb_5045_71e6_c0a6, - 0x96ab_3211_5e91_e8cc, 0xbf8a_cb18_de8f_38d1, 0x66da_e588_0167_2606, 0x833b_6017_8723_17fb, - 0xb87c_16f2_d1c9_2864, 0xdb76_6a74_e58b_669c, 0x8965_9f85_c614_17be, 0xc8da_ad85_6011_ea0c, - 0x76a4_b565_b6fe_7eae, 0xa469_d085_f623_7312, 0xaaf0_3656_83a3_e96c, 0x4dbb_746f_8424_f7b8, - 0x0638_755a_f4e4_acc1, 0x3d78_07f5_bde6_4486, 0x17be_6d8f_5bbb_7639, 0x0903_f0cd_44dc_35dc, - 0x67b6_72ea_fdf1_196c, 0xa676_ff93_ed4c_82f1, 0x521d_1004_c505_3d9d, 0x37ba_9ad0_9ccc_9202, - 0x84e5_4d29_7aac_fb51, 0x0a0b_4b77_6a14_3445, 0x0820_d471_e20b_348e, 0x1874_383c_b83d_46dc, - 0x97ed_eec7_a1ef_e11c, 0xb330_e50b_1bdc_42aa, 0x1dd9_1955_ce70_e032, 0xa514_cdb8_8f29_39d5, - 0x2791_233f_d90d_b9d3, 0x7b67_0a4c_c50f_7a9b, 0x77c0_7d2a_05c6_dfa5, 0xe377_8b66_46d0_a6fa, - 0xb39c_8eda_47b5_6749, 0x933e_d448_addb_ef28, 0xaf84_6af6_ab7d_0bf4, 0x0e5a_f208_eb66_6e49, - 0x5e66_22f7_3534_cd6a, 0x297d_aeca_42ef_5b6e, 0x862d_aef3_d355_39a6, 0xe687_2249_8f8e_1ea9, - 0x981c_5309_3dc0_d572, 0xfa09_b0bf_bf86_fbf5, 0x30b1_e961_6621_9f15, 0x70e7_d466_bdc4_fb83, - 0x5a66_736e_35f2_a8e9, 0xcddb_59d2_b7c1_baef, 0xd6c7_d247_d26d_8996, 0xea4e_39ea_c8de_1ba3, - 0x539c_8bb1_9fa3_aff2, 0x09f9_0e4c_5fd5_08d8, 0xa34e_5956_fbaf_3385, 0x2e2f_8e15_1d3e_f375, - 0x1736_91e9_b83f_aec1, 0xb85a_8d56_bf01_6379, 0x8382_3812_6740_8ae3, 0xb90f_901b_bdc0_096d, - 0x7c6a_d329_33bc_ec65, 0x76bb_5e2f_2c8a_d595, 0x390f_851a_6cf4_6d28, 0xc3e6_064d_a1c2_da72, - 0xc52a_0c10_1cfa_5389, 0xd78e_af84_a3fb_c530, 0x3781_b9e2_288b_997e, 0x73c2_f6de_a83d_05c4, - 0x0422_8e36_4c5b_5ed7, 0x9d7a_3edf_0da4_3911, 0x8edc_feda_2468_6756, 0x5e76_67a7_b7a9_b3a1, - 0x4c4f_389f_a143_791d, 0xb08b_c102_3da7_cddc, 0x7ab4_be3a_e529_b1cc, 0x754e_6132_dbe7_4ff9, - 0x7163_5442_a839_df45, 0x2f6f_b164_3fbe_52de, 0x961e_0a42_cf7a_8177, 0xf3b4_5d83_d89e_f2ea, - 0xee3d_e4cf_4a6e_3e9b, 0xcd68_4854_2c32_95e7, 0xe4ce_e166_4c78_662f, 0x9947_548b_474c_68c4, - 0x25d7_3777_a5ed_8b0b, 0x00c9_15b1_d636_b7fc, 0x21c2_ba75_d9b0_d2da, 0x5f6b_5dcf_608a_64a1, - 0xdcf3_3325_5ff9_570c, 0x633b_9224_18ce_d4ee, 0xc136_dde0_b004_b34a, 0x58cc_83b0_5d4b_2f5a, - 0x5eb4_24dd_a28e_42d2, 0x62df_4736_9739_cd98, 0xb4e0_b424_85e4_ce17, 0x16e1_f0c1_f9a8_d1e7, - 0x8ec3_9167_0756_0ebf, 0x62ba_6e2d_f2cc_9db3, 0xcbf9_f4ff_77d8_3a16, 0x78d9_d7d0_7d2b_bcc4, - 0xef55_4ce1_e02c_41f4, 0x8d75_8112_7ecc_f94d, 0xa9b5_3336_cb3c_8a05, 0x38c4_2c0b_f45c_4f91, - 0x6408_93cd_f448_8863, 0x80ec_34bc_575e_a568, 0x39f3_24f5_b48e_aa40, 0xe9d9_ed1f_8eff_527f, - 0x9224_fc05_8cc5_a214, 0xbaba_00b0_4cfe_7741, 0x309a_9f12_0fcf_52af, 0xa558_f3ec_6562_6212, - 0x424b_ec8b_7ada_be2f, 0x4162_2513_a6ae_a433, 0xb88d_a2d5_324c_a798, 0xd287_733b_2455_28a4, - 0x9a44_697e_6d68_aec3, 0x7b10_93be_2f49_bb28, 0x50bb_ec63_2e3d_8aad, 0x6cd9_0723_e1ea_8283, - 0x897b_9e74_31b0_2bf3, 0x219e_fdcb_338a_7047, 0x3b03_11f0_a27c_0656, 0xdb17_bf91_c0db_96e7, - 0x8cd4_fd6b_4e85_a5b2, 0xfab0_7105_4ba6_409d, 0x40d6_fe83_1fa9_dfd9, 0xaf35_8deb_ad7d_791e, - 0xeb8d_0e25_a65e_3e58, 0xbbcb_d3df_14e0_8580, 0x0cf7_51f2_7ecd_ab2b, 0x2b4d_a14f_2613_d8f4, -]; - -/// `ldmRollingHashState_t` -struct RollingHashState { - rolling: u64, - stop_mask: u64, -} - -impl RollingHashState { - /// `ZSTD_ldm_gear_init` - fn new(params: &LdmParams) -> Self { - let max_bits_in_mask = params.min_match_length.min(64); - let hash_rate_log = params.hash_rate_log; - let rate_mask = (1u64 << (hash_rate_log & 63)) - 1; - // A split is to happen every `1 << hash_rate_log` bytes on average, and bit n of the hash depends on the last - // n bytes: the mask has the highest bits that depend on no more than `min_match_length` bytes. - let stop_mask = if hash_rate_log > 0 && hash_rate_log <= max_bits_in_mask { - rate_mask << ((max_bits_in_mask - hash_rate_log) & 63) - } else { - rate_mask - }; - Self { - rolling: u64::from(u32::MAX), - stop_mask, - } - } - - /// `ZSTD_ldm_gear_feed`: writes to `splits` the split points of `data`, as positions in it. Stops at the end of - /// `data` or once `splits` is full. Returns how many bytes it went through, and how many splits it found. - #[inline] - fn feed(&mut self, data: &[u8], splits: &mut [usize; LDM_BATCH_SIZE]) -> (usize, usize) { - let mut hash = self.rolling; - let mask = self.stop_mask; - let mut n = 0usize; - let mut num_splits = 0usize; - // Returns whether `splits` is full. - let mut iter_once = |byte: u8| { - let gear = GEAR_TAB.get(usize::from(byte)).copied().unwrap_or(0); - hash = (hash << 1).wrapping_add(gear); - n += 1; - if hash & mask == 0 { - if let Some(split) = splits.get_mut(num_splits) { - *split = n; - } - num_splits += 1; - return num_splits == LDM_BATCH_SIZE; - } - false - }; - let (quads, rest) = data.as_chunks::<4>(); - let full = quads - .iter() - .any(|&[a, b, c, d]| iter_once(a) || iter_once(b) || iter_once(c) || iter_once(d)); - if !full { - for &byte in rest { - if iter_once(byte) { - break; - } - } - } - self.rolling = hash; - (n, num_splits) - } -} - -/// `XXH64`, with a seed of 0. -#[inline] -fn xxh64(data: &[u8]) -> u64 { - let mut state = Xxh64::new(); - state.update(data); - state.digest() -} - -/// `ZSTD_ldm_adjustParameters`: gives a value to the parameters that are 0. `c_params` are those that are applied. -pub(crate) fn adjust_parameters(params: &mut LdmParams, c_params: &CompressionParameters) { - let strategy = c_params.strat(); - params.window_log = c_params.window_log; - if params.hash_rate_log == 0 { - if params.hash_log > 0 { - if params.window_log > params.hash_log { - params.hash_rate_log = params.window_log - params.hash_log; - } - } else { - // From 7 for `fast` to 4 for `btultra2`. - params.hash_rate_log = 7 - (strategy as u32 / 3); - } - } - if params.hash_log == 0 { - params.hash_log = params - .window_log - .wrapping_sub(params.hash_rate_log) - .clamp(ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - } - if params.min_match_length == 0 { - params.min_match_length = LDM_MIN_MATCH_LENGTH; - if strategy >= Strategy::BtUltra { - params.min_match_length /= 2; - } - } - if params.bucket_size_log == 0 { - params.bucket_size_log = - (strategy as u32).clamp(LDM_BUCKET_SIZE_LOG, ZSTD_LDM_BUCKETSIZELOG_MAX); - } - params.bucket_size_log = params.bucket_size_log.min(params.hash_log); -} - -/// `ZSTD_cwksp_alloc_size`, which adds something only under ASAN. -const fn cwksp_alloc_size(size: usize) -> usize { - size -} - -/// `ZSTD_ldm_getTableSize`: how many bytes of workspace the C code takes for the tables. -pub(crate) fn get_table_size(params: LdmParams) -> usize { - let ldm_h_size = 1usize << (params.hash_log & 63); - let ldm_bucket_size_log = params.bucket_size_log.min(params.hash_log); - let ldm_bucket_size = 1usize << ((params.hash_log - ldm_bucket_size_log) & 63); - let total_size = cwksp_alloc_size(ldm_bucket_size) - + cwksp_alloc_size(ldm_h_size.wrapping_mul(size_of::())); - if params.enable_ldm == ParamSwitch::Enable { - total_size - } else { - 0 - } -} - -/// `ZSTD_ldm_getMaxNbSeq`: the most sequences that `max_chunk_size` bytes can give. -pub(crate) fn get_max_nb_seq(params: LdmParams, max_chunk_size: usize) -> usize { - if params.enable_ldm == ParamSwitch::Enable { - max_chunk_size - .checked_div(params.min_match_length as usize) - .unwrap_or(0) - } else { - 0 - } -} - -/// The mask that keeps, of a hash, the number of a bucket. -#[inline] -const fn bucket_mask(params: &LdmParams) -> u32 { - (1u32 << (params.hash_log.wrapping_sub(params.bucket_size_log) & 31)) - 1 -} - -/// `ZSTD_ldm_getBucket`: where the bucket `hash` starts in the hash table. -#[inline(always)] -const fn get_bucket(hash: usize, bucket_size_log: u32) -> usize { - hash << (bucket_size_log & 31) -} - -/// `ZSTD_ldm_insertEntry`: in place of the oldest entry of the bucket `hash`. -#[inline] -fn insert_entry( - ldm_state: &mut LdmState, - hash: usize, - entry: LdmEntry, - bucket_size_log: u32, -) -> Result<()> { - let p_offset = ldm_state.bucket_offsets.at_mut(hash)?; - let offset = u32::from(*p_offset); - *p_offset = ((offset + 1) & ((1u32 << (bucket_size_log & 31)) - 1)) as u8; - ldm_state - .hash_table - .put(get_bucket(hash, bucket_size_log) + offset as usize, entry) -} - -/// `ZSTD_ldm_countBackwardsMatch`: how many bytes before `p_in` in `buf`, down to `p_anchor`, are the same as those -/// before `p_match` in `match_buf`, down to `p_match_base`. -#[inline] -fn count_backwards_match( - buf: &[u8], - p_in: usize, - p_anchor: usize, - match_buf: &[u8], - p_match: usize, - p_match_base: usize, -) -> Result { - Ok(buf - .sub(p_anchor..p_in)? - .iter() - .rev() - .zip(match_buf.sub(p_match_base..p_match)?.iter().rev()) - .take_while(|(in_byte, match_byte)| in_byte == match_byte) - .count()) -} - -/// `ZSTD_ldm_countBackwardsMatch_2segments`, for a match in the current segment, which starts at `p_match_base`: from -/// there, the match goes on at the end of the older segment. A match in the older segment has nowhere to go on. -#[inline] -fn count_backwards_match_2segments( - buf: &[u8], - p_in: usize, - p_anchor: usize, - p_match: usize, - p_match_base: usize, - dict_buf: &[u8], - p_ext_dict_start: usize, - p_ext_dict_end: usize, -) -> Result { - let match_length = count_backwards_match(buf, p_in, p_anchor, buf, p_match, p_match_base)?; - if p_match - match_length != p_match_base { - return Ok(match_length); - } - Ok(match_length - + count_backwards_match( - buf, - p_in - match_length, - p_anchor, - dict_buf, - p_ext_dict_end, - p_ext_dict_start, - )?) -} - -/// `ZSTD_ldm_fillFastTables`: the strategies that index only what they search have to be told about what they skip. -fn fill_fast_tables(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) -> Result<()> { - match ms.c_params.strat() { - Strategy::Fast => fast::fill_hash_table( - ms, - prefix, - end, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - ), - Strategy::DFast => double_fast::fill_double_hash_table( - ms, - prefix, - end, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - ), - Strategy::Greedy - | Strategy::Lazy - | Strategy::Lazy2 - | Strategy::BtLazy2 - | Strategy::BtOpt - | Strategy::BtUltra - | Strategy::BtUltra2 => Ok(()), - } -} - -/// `ZSTD_ldm_fillHashTable`: indexes the content of a dictionary, which is `prefix.buf` from `ip` to `iend`. -pub(crate) fn fill_hash_table( - ldm_state: &mut LdmState, - prefix: Prefix<'_>, - mut ip: usize, - iend: usize, - params: &LdmParams, -) -> Result<()> { - let min_match_length = params.min_match_length as usize; - let bucket_mask = bucket_mask(params); - let istart = ip; - let mut hash_state = RollingHashState::new(params); - let mut splits = [0usize; LDM_BATCH_SIZE]; - - while ip < iend { - let (hashed, num_splits) = hash_state.feed(prefix.buf.sub(ip..iend)?, &mut splits); - for &split in splits.iter().take(num_splits) { - if ip + split >= istart + min_match_length { - let split = ip + split - min_match_length; - let xxhash = xxh64(prefix.buf.span(split, min_match_length)?); - let hash = xxhash as u32 & bucket_mask; - let entry = LdmEntry { - offset: prefix.index_of(split), - checksum: (xxhash >> 32) as u32, - }; - insert_entry(ldm_state, hash as usize, entry, params.bucket_size_log)?; - } - } - ip += hashed; - } - Ok(()) -} - -/// `ZSTD_ldm_limitTableUpdate`: if the tables of `ms` are far behind `anchor`, skips most of the way. -fn limit_table_update(ms: &mut MatchState, src: &BlockSrc<'_>, anchor: usize) { - let curr = src.index_of(anchor); - if curr > ms.next_to_update.wrapping_add(1024) { - let behind = curr.wrapping_sub(ms.next_to_update).wrapping_sub(1024); - ms.next_to_update = curr.wrapping_sub(behind.min(512)); - } -} - -/// `ldmMatchCandidate_t` -#[derive(Clone, Copy, Default)] -struct MatchCandidate { - split: usize, - hash: u32, - checksum: u32, - /// Where the bucket starts in the hash table. - bucket: usize, -} - -/// `ZSTD_ldm_generateSequences_internal`: the input is from `istart` to `iend` in the buffer of the current segment. -/// Returns how many bytes at its end are in no sequence. -fn generate_sequences_internal( - ldm_state: &mut LdmState, - raw_seq_store: &mut RawSeqStore, - params: &LdmParams, - bufs: &Buffers<'_>, - istart: usize, - iend: usize, -) -> Result { - let window = ldm_state.window; - let ext_dict = window.has_ext_dict(); - let min_match_length = params.min_match_length as usize; - let ents_per_bucket = 1usize << (params.bucket_size_log & 31); - let bucket_mask = bucket_mask(params); - let dict_limit = window.dict_limit; - let lowest_index = if ext_dict { - window.low_limit - } else { - dict_limit - }; - let prefix = Prefix { - buf: bufs.get(window.base_buf), - base: window.base.wrapping_sub(window.base_buf.start()), - }; - let buf = prefix.buf; - let dict_buf = bufs.get(window.dict_buf); - let dict_base = window.dict_base.wrapping_sub(window.dict_buf.start()); - let dict_start = dict_base.wrapping_add(lowest_index as usize); - let dict_end = dict_base.wrapping_add(dict_limit as usize); - let low_prefix = prefix.pos_of(dict_limit); - let ilimit = iend.saturating_sub(HASH_READ_SIZE); - let mut anchor = istart; - let mut ip = istart; - let mut splits = [0usize; LDM_BATCH_SIZE]; - let mut candidates = [MatchCandidate::default(); LDM_BATCH_SIZE]; - - if iend.checked_sub(istart).ok_or(BUG)? < min_match_length { - return Ok(iend - anchor); - } - - // Here and where it skips over a match, the C code calls `ZSTD_ldm_gear_reset`, which computes the hash of the - // `min_match_length` bytes before `ip` and drops it: the rolling hash goes on from what it was. - let mut hash_state = RollingHashState::new(params); - ip += min_match_length; - - while ip < ilimit { - let (hashed, num_splits) = hash_state.feed(buf.sub(ip..ilimit)?, &mut splits); - - for (candidate, &split) in candidates.iter_mut().zip(&splits).take(num_splits) { - let split = ip + split - min_match_length; - let xxhash = xxh64(buf.span(split, min_match_length)?); - let hash = xxhash as u32 & bucket_mask; - *candidate = MatchCandidate { - split, - hash, - checksum: (xxhash >> 32) as u32, - bucket: get_bucket(hash as usize, params.bucket_size_log), - }; - } - - for candidate in candidates.iter().take(num_splits) { - let MatchCandidate { - split, - hash, - checksum, - bucket, - } = *candidate; - let hash = hash as usize; - let mut forward_match_length = 0usize; - let mut backward_match_length = 0usize; - let mut best_match_length = 0usize; - let mut best_entry = None; - let new_entry = LdmEntry { - offset: prefix.index_of(split), - checksum, - }; - - // A split inside the previous match is indexed, but not searched. - if split < anchor { - insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; - continue; - } - - for cur in ldm_state.hash_table.span(bucket, ents_per_bucket)? { - if cur.checksum != checksum || cur.offset <= lowest_index { - continue; - } - let (cur_forward_match_length, cur_backward_match_length) = if ext_dict - && cur.offset < dict_limit - { - let p_match = dict_base.wrapping_add(cur.offset as usize); - let forward = - count_2segments(buf, split, dict_buf, p_match, iend, dict_end, low_prefix)?; - if forward < min_match_length { - continue; - } - let backward = - count_backwards_match(buf, split, anchor, dict_buf, p_match, dict_start)?; - (forward, backward) - } else { - let p_match = prefix.pos_of(cur.offset); - let forward = count(buf, split, buf, p_match, iend)?; - if forward < min_match_length { - continue; - } - let backward = if ext_dict { - count_backwards_match_2segments( - buf, split, anchor, p_match, low_prefix, dict_buf, dict_start, dict_end, - )? - } else { - count_backwards_match(buf, split, anchor, buf, p_match, low_prefix)? - }; - (forward, backward) - }; - let cur_total_match_length = cur_forward_match_length + cur_backward_match_length; - - if cur_total_match_length > best_match_length { - best_match_length = cur_total_match_length; - forward_match_length = cur_forward_match_length; - backward_match_length = cur_backward_match_length; - best_entry = Some(*cur); - } - } - - let Some(best_entry) = best_entry else { - insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; - continue; - }; - - let offset = new_entry.offset.wrapping_sub(best_entry.offset); - let m_length = forward_match_length + backward_match_length; - if raw_seq_store.size == raw_seq_store.capacity { - return Err(ErrorCode::DstSizeTooSmall); - } - raw_seq_store.seq.put( - raw_seq_store.size, - RawSeq { - offset, - lit_length: (split - backward_match_length - anchor) as u32, - match_length: m_length as u32, - }, - )?; - raw_seq_store.size += 1; - - // Only now, so that the position does not match itself. - insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; - - anchor = split + forward_match_length; - - // The match goes beyond what was hashed: what is left of it is skipped, and the other candidates dropped. - if anchor > ip + hashed { - ip = anchor - hashed; - break; - } - } - - ip += hashed; - } - - Ok(iend - anchor) -} - -/// `ZSTD_ldm_reduceTable` -fn reduce_table(table: &mut [LdmEntry], reducer_value: u32) { - for entry in table { - entry.offset = entry.offset.saturating_sub(reducer_value); - } -} - -/// `ZSTD_ldm_generateSequences`: adds to `sequences` the long distance matches of the input, which is the bytes from -/// `start` to `end` of the buffer of the current segment of the window, and which the window was updated with. -/// Stops early, without an error, at a chunk boundary where `sequences` is full. -pub(crate) fn generate_sequences( - ldm_state: &mut LdmState, - sequences: &mut RawSeqStore, - params: &LdmParams, - bufs: &Buffers<'_>, - start: usize, - end: usize, -) -> Result<()> { - const K_MAX_CHUNK_SIZE: usize = 1 << 20; - let max_dist = 1u32 << (params.window_log & 31); - let buf_start = ldm_state.window.base_buf.start(); - let mut leftover_size = 0usize; - let mut chunk_start = start; - - while chunk_start < end && sequences.size < sequences.capacity { - let chunk_end = end.min(chunk_start + K_MAX_CHUNK_SIZE); - let chunk_size = chunk_end - chunk_start; - let prev_size = sequences.size; - - if ldm_state - .window - .need_overflow_correction(buf_start + chunk_end) - { - let ldm_h_size = 1usize << (params.hash_log & 31); - let correction = - ldm_state - .window - .correct_overflow(0, max_dist, buf_start + chunk_start); - reduce_table(ldm_state.hash_table.sub_mut(..ldm_h_size)?, correction); - // An overflow correction invalidates the dictionary. - ldm_state.loaded_dict_end = 0; - } - // The whole chunk is searched with the window that its end has, so that no match is too far for any position - // of it, at the cost of some matches at its start. - if ldm_state.window.enforce_max_dist( - buf_start + chunk_end, - max_dist, - ldm_state.loaded_dict_end, - ) { - ldm_state.loaded_dict_end = 0; - } - let new_leftover_size = generate_sequences_internal( - ldm_state, - sequences, - params, - bufs, - chunk_start, - chunk_end, - )?; - // What the previous chunks left goes to the literals of the first sequence of this one. - if prev_size < sequences.size { - let first = sequences.seq.at_mut(prev_size)?; - first.lit_length = first.lit_length.wrapping_add(leftover_size as u32); - leftover_size = new_leftover_size; - } else { - leftover_size += chunk_size; - } - chunk_start = chunk_end; - } - Ok(()) -} - -/// `ZSTD_ldm_skipSequences`: moves `src_size` bytes forward in the sequences. A match of which less than `min_match` -/// is left becomes literals of the next sequence. -pub(crate) fn skip_sequences( - raw_seq_store: &mut RawSeqStore, - mut src_size: usize, - min_match: u32, -) -> Result<()> { - while src_size > 0 && raw_seq_store.pos < raw_seq_store.size { - let seq = raw_seq_store.seq.at_mut(raw_seq_store.pos)?; - if src_size <= seq.lit_length as usize { - seq.lit_length -= src_size as u32; - return Ok(()); - } - src_size -= seq.lit_length as usize; - seq.lit_length = 0; - if src_size < seq.match_length as usize { - seq.match_length -= src_size as u32; - if seq.match_length < min_match { - let match_length = seq.match_length; - if raw_seq_store.pos + 1 < raw_seq_store.size { - let next = raw_seq_store.seq.at_mut(raw_seq_store.pos + 1)?; - next.lit_length = next.lit_length.wrapping_add(match_length); - } - raw_seq_store.pos += 1; - } - return Ok(()); - } - src_size -= seq.match_length as usize; - seq.match_length = 0; - raw_seq_store.pos += 1; - } - Ok(()) -} - -/// `maybeSplitSequence`: the next sequence, cut to the `remaining` bytes of the block. What is cut off stays in the -/// store. If there is no match of `min_match` left in the block, the offset is 0. -fn maybe_split_sequence( - raw_seq_store: &mut RawSeqStore, - remaining: u32, - min_match: u32, -) -> Result { - let mut sequence = raw_seq_store.seq.at(raw_seq_store.pos)?; - let length = sequence.lit_length.wrapping_add(sequence.match_length); - if remaining >= length { - raw_seq_store.pos += 1; - return Ok(sequence); - } - if remaining <= sequence.lit_length { - sequence.offset = 0; - } else { - sequence.match_length = remaining - sequence.lit_length; - if sequence.match_length < min_match { - sequence.offset = 0; - } - } - skip_sequences(raw_seq_store, remaining as usize, min_match)?; - Ok(sequence) -} - -/// `ZSTD_ldm_skipRawSeqStoreBytes`: moves `nb_bytes` forward in the sequences, leaving them as they are. -pub(crate) fn skip_raw_seq_store_bytes( - raw_seq_store: &mut RawSeqStore, - nb_bytes: usize, -) -> Result<()> { - let mut curr_pos = raw_seq_store.pos_in_sequence.wrapping_add(nb_bytes) as u32; - while curr_pos != 0 && raw_seq_store.pos < raw_seq_store.size { - let curr_seq = raw_seq_store.seq.at(raw_seq_store.pos)?; - let length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); - if curr_pos >= length { - curr_pos -= length; - raw_seq_store.pos += 1; - } else { - raw_seq_store.pos_in_sequence = curr_pos as usize; - break; - } - } - if curr_pos == 0 || raw_seq_store.pos == raw_seq_store.size { - raw_seq_store.pos_in_sequence = 0; - } - Ok(()) -} - -/// `ZSTD_ldm_blockCompress`: finds the sequences of the block `src`, given the long distance matches, which it moves -/// forward in by the size of the block. Returns how many bytes at the end of the block are in no sequence. -pub(crate) fn block_compress( - raw_seq_store: &mut RawSeqStore, - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - use_row_match_finder: bool, - src: &BlockSrc<'_>, -) -> Result { - let strategy = ms.c_params.strat(); - let min_match = ms.c_params.min_match; - let dict_mode = ms.dict_mode(src.dms.as_ref()); - let iend = src.end; - let mut ip = src.start; - - // The optimal parser weighs the long distance matches against its own. - if strategy >= Strategy::BtOpt { - let block = BlockSrc { - ldm_seq_store: Some(&*raw_seq_store), - ..*src - }; - let last_ll_size = compress_block( - ms, - seq_store, - rep, - &block, - strategy, - use_row_match_finder, - dict_mode, - )?; - skip_raw_seq_store_bytes(raw_seq_store, iend.wrapping_sub(ip))?; - return Ok(last_ll_size); - } - - while raw_seq_store.pos < raw_seq_store.size && ip < iend { - let sequence = maybe_split_sequence(raw_seq_store, (iend - ip) as u32, min_match)?; - if sequence.offset == 0 { - break; - } - - limit_table_update(ms, src, ip); - fill_fast_tables(ms, src.prefix(), ip)?; - // The block compressor takes care of the literals, and what it leaves goes with the long distance match. - let literals = BlockSrc { - start: ip, - end: ip + sequence.lit_length as usize, - ..*src - }; - let new_lit_length = compress_block( - ms, - seq_store, - rep, - &literals, - strategy, - use_row_match_finder, - dict_mode, - )?; - ip += sequence.lit_length as usize; - let [rep0, rep1, _] = *rep; - *rep = [sequence.offset, rep0, rep1]; - seq_store.store_seq( - new_lit_length, - src.buf, - ip.checked_sub(new_lit_length).ok_or(BUG)?, - offset_to_offbase(sequence.offset), - sequence.match_length as usize, - )?; - ip += sequence.match_length as usize; - } - limit_table_update(ms, src, ip); - fill_fast_tables(ms, src.prefix(), ip)?; - let last = BlockSrc { start: ip, ..*src }; - compress_block( - ms, - seq_store, - rep, - &last, - strategy, - use_row_match_finder, - dict_mode, - ) -} diff --git a/src/zstd/wip/work/ldm/zstd/compress/mod.rs b/src/zstd/wip/work/ldm/zstd/compress/mod.rs deleted file mode 100644 index 13ddecdf82e4..000000000000 --- a/src/zstd/wip/work/ldm/zstd/compress/mod.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! `lib/compress` - -pub(crate) mod double_fast; -pub(crate) mod fast; -pub(crate) mod fse; -pub(crate) mod hist; -pub(crate) mod huf; -pub(crate) mod internal; -pub(crate) mod lazy; -pub(crate) mod ldm; -pub(crate) mod opt; -pub(crate) mod presplit; - -use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; - -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; - -/// `ZSTD_rowMatchFinderSupported` -pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { - strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 -} - -/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to -/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies -/// that support it look at. Returns how many bytes at the end of the block are in no sequence. -pub(crate) fn compress_block( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row_match_finder: bool, - dict_mode: DictMode, -) -> Result { - match strategy { - Strategy::Fast => match dict_mode { - DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), - DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), - DictMode::DictMatchState => { - fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) - } - }, - Strategy::DFast => match dict_mode { - DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), - DictMode::ExtDict => { - double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) - } - }, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { - lazy::compress_block_lazy( - ms, - seq_store, - rep, - src, - strategy, - use_row_match_finder && row_match_finder_supported(strategy), - dict_mode, - ) - } - Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { - opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) - } - } -} diff --git a/src/zstd/wip/work/ldm/zstd/compress/opt.rs b/src/zstd/wip/work/ldm/zstd/compress/opt.rs deleted file mode 100644 index a7a6022a3a45..000000000000 --- a/src/zstd/wip/work/ldm/zstd/compress/opt.rs +++ /dev/null @@ -1,1721 +0,0 @@ -//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. - -use core::mem; - -use super::hist; -use super::huf::HufRepeat; -use super::internal::{ - BlockSrc, DictMode, EntropyCTables, Match, MatchState, OptPrice, OptState, Optimal, - ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, ZSTD_OPT_NUM, count, - count_2segments, hash_ptr, hash3_ptr, index_overlap_check, ll_code, ml_code, new_rep, - offset_to_offbase, -}; -use crate::common::{ - LL_BITS, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, -}; -use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32}; - -/// How much a literal adds to its frequency, so that frequencies adapt fast to new statistics. -const ZSTD_LITFREQ_ADD: u32 = 2; -const ZSTD_MAX_PRICE: i32 = 1 << 30; -/// A first block of at most this size is priced by fixed rules, not by statistics. -const ZSTD_PREDEF_THRESHOLD: usize = 8; - -const MAX_LIT: u32 = 255; - -// ── prices ──────────────────────────────────────────────────────────────── - -/// Prices are in units of `1 / BITCOST_MULTIPLIER` of a bit. -const BITCOST_ACCURACY: u32 = 8; -const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY; - -/// `WEIGHT`: approximate for `btopt`, accurate for `btultra`. -#[inline(always)] -const fn weight(stat: u32, opt_level: i32) -> u32 { - if opt_level != 0 { - frac_weight(stat) - } else { - bit_weight(stat) - } -} - -/// `ZSTD_bitWeight`: the cost of a statistic, in whole bits. -#[inline(always)] -const fn bit_weight(stat: u32) -> u32 { - highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER -} - -/// `ZSTD_fracWeight`: the cost of a statistic, with a fraction of a bit that is interpolated linearly. -#[inline(always)] -const fn frac_weight(raw_stat: u32) -> u32 { - let stat = raw_stat.wrapping_add(1); - let hb = highbit32(stat); - let b_weight = hb * BITCOST_MULTIPLIER; - // From 1 to 2, in fixed point. - let f_weight = (stat << BITCOST_ACCURACY) >> hb; - b_weight.wrapping_add(f_weight) -} - -/// `ZSTD_compressedLiterals` -#[inline(always)] -fn compressed_literals(opt_ptr: &OptState) -> bool { - opt_ptr.literal_compression_mode != ParamSwitch::Disable -} - -/// `ZSTD_setBasePrices` -#[inline(always)] -fn set_base_prices(opt_ptr: &mut OptState, opt_level: i32) { - if compressed_literals(opt_ptr) { - opt_ptr.lit_sum_base_price = weight(opt_ptr.lit_sum, opt_level); - } - opt_ptr.lit_length_sum_base_price = weight(opt_ptr.lit_length_sum, opt_level); - opt_ptr.match_length_sum_base_price = weight(opt_ptr.match_length_sum, opt_level); - opt_ptr.off_code_sum_base_price = weight(opt_ptr.off_code_sum, opt_level); -} - -/// `sum_u32` -fn sum_u32(table: &[u32]) -> u32 { - table.iter().fold(0, |total, &elt| total.wrapping_add(elt)) -} - -/// `base_directive_e`: the least a scaled down statistic can be. -#[derive(Clone, Copy, PartialEq, Eq)] -enum BaseDirective { - /// 0, if it is 0 already. - Base0Possible, - Base1Guaranteed, -} - -/// `ZSTD_downscaleStats`: returns the new sum of the elements. -fn downscale_stats( - table: &mut [u32], - last_elt_index: u32, - shift: u32, - base1: BaseDirective, -) -> Result { - let mut sum = 0u32; - for elt in table.sub_mut(..=last_elt_index as usize)? { - let base = u32::from(base1 == BaseDirective::Base1Guaranteed || *elt > 0); - let new_stat = base.wrapping_add(*elt >> (shift & 31)); - sum = sum.wrapping_add(new_stat); - *elt = new_stat; - } - Ok(sum) -} - -/// `ZSTD_scaleStats`: scales all the elements down if their sum is too large. Returns the sum that results. -fn scale_stats(table: &mut [u32], last_elt_index: u32, log_target: u32) -> Result { - let prevsum = sum_u32(table.sub(..=last_elt_index as usize)?); - let factor = prevsum >> log_target; - if factor <= 1 { - return Ok(prevsum); - } - downscale_stats( - table, - last_elt_index, - highbit32(factor), - BaseDirective::Base1Guaranteed, - ) -} - -/// Frequencies, out of about `1 << scale_log`, that make each symbol cost the bits `bit_cost` tells. Returns their sum. -fn freqs_from_bit_costs( - freqs: &mut [u32], - last_elt_index: u32, - scale_log: u32, - bit_cost: impl Fn(u32) -> u32, -) -> Result { - let mut sum = 0u32; - for (symbol, freq) in (0u32..).zip(freqs.sub_mut(..=last_elt_index as usize)?) { - let bit_cost = bit_cost(symbol); - // At least 1, to have a cost. - *freq = if bit_cost != 0 { - 1 << (scale_log.wrapping_sub(bit_cost) & 31) - } else { - 1 - }; - sum = sum.wrapping_add(*freq); - } - Ok(sum) -} - -/// `ZSTD_rescaleFreqs`: for the first block, which no statistics tell, starts them from the tables of the dictionary -/// if there is one, else from the literals of `src` and from fixed values for the sequences. For the next ones, scales -/// down what the blocks before have accumulated. -fn rescale_freqs( - opt_ptr: &mut OptState, - symbol_costs: &EntropyCTables, - src: &[u8], - opt_level: i32, -) -> Result<()> { - #[rustfmt::skip] - const BASE_LL_FREQS: [u32; MAX_LL as usize + 1] = [ - 4, 2, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, - ]; - #[rustfmt::skip] - const BASE_OFC_FREQS: [u32; MAX_OFF as usize + 1] = [ - 6, 2, 1, 1, 2, 3, 4, 4, - 4, 3, 2, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - ]; - - let compressed_literals = compressed_literals(opt_ptr); - opt_ptr.price_type = OptPrice::Dynamic; - - if opt_ptr.lit_length_sum == 0 { - if src.len() <= ZSTD_PREDEF_THRESHOLD { - opt_ptr.price_type = OptPrice::Predef; - } - - if symbol_costs.huf.repeat_mode == HufRepeat::Valid { - // A Huffman table that has all the values is presumed to come from a dictionary. - opt_ptr.price_type = OptPrice::Dynamic; - - if compressed_literals { - let huf = &symbol_costs.huf.c_table; - opt_ptr.lit_sum = - freqs_from_bit_costs(&mut opt_ptr.lit_freq, MAX_LIT, 11, |lit| { - huf.get_nb_bits(lit) - })?; - } - let fse = &symbol_costs.fse; - opt_ptr.lit_length_sum = - freqs_from_bit_costs(&mut opt_ptr.lit_length_freq, MAX_LL, 10, |ll| { - fse.litlength_c_table.get_max_nb_bits(ll) - })?; - opt_ptr.match_length_sum = - freqs_from_bit_costs(&mut opt_ptr.match_length_freq, MAX_ML, 10, |ml| { - fse.matchlength_c_table.get_max_nb_bits(ml) - })?; - opt_ptr.off_code_sum = - freqs_from_bit_costs(&mut opt_ptr.off_code_freq, MAX_OFF, 10, |of| { - fse.offcode_c_table.get_max_nb_bits(of) - })?; - } else { - if compressed_literals { - let mut lit = MAX_LIT; - hist::count_simple(&mut opt_ptr.lit_freq, &mut lit, src)?; - opt_ptr.lit_sum = downscale_stats( - &mut opt_ptr.lit_freq, - MAX_LIT, - 8, - BaseDirective::Base0Possible, - )?; - } - - opt_ptr.lit_length_freq.write_at(0, &BASE_LL_FREQS)?; - opt_ptr.lit_length_sum = sum_u32(&BASE_LL_FREQS); - - opt_ptr - .match_length_freq - .sub_mut(..=MAX_ML as usize)? - .fill(1); - opt_ptr.match_length_sum = MAX_ML + 1; - - opt_ptr.off_code_freq.write_at(0, &BASE_OFC_FREQS)?; - opt_ptr.off_code_sum = sum_u32(&BASE_OFC_FREQS); - } - } else { - if compressed_literals { - opt_ptr.lit_sum = scale_stats(&mut opt_ptr.lit_freq, MAX_LIT, 12)?; - } - opt_ptr.lit_length_sum = scale_stats(&mut opt_ptr.lit_length_freq, MAX_LL, 11)?; - opt_ptr.match_length_sum = scale_stats(&mut opt_ptr.match_length_freq, MAX_ML, 11)?; - opt_ptr.off_code_sum = scale_stats(&mut opt_ptr.off_code_freq, MAX_OFF, 11)?; - } - - set_base_prices(opt_ptr, opt_level); - Ok(()) -} - -/// `ZSTD_rawLiteralsCost`: the price of `literals`, without that of their length. -#[inline(always)] -fn raw_literals_cost(literals: &[u8], opt_ptr: &OptState, opt_level: i32) -> Result { - let lit_length = literals.len() as u32; - if lit_length == 0 { - return Ok(0); - } - - if !compressed_literals(opt_ptr) { - // 8 bits per literal. - return Ok((lit_length << 3).wrapping_mul(BITCOST_MULTIPLIER)); - } - - if opt_ptr.price_type == OptPrice::Predef { - // 6 bits per literal. - return Ok(lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER)); - } - - let mut price = opt_ptr.lit_sum_base_price.wrapping_mul(lit_length); - let lit_price_max = opt_ptr.lit_sum_base_price.wrapping_sub(BITCOST_MULTIPLIER); - for &literal in literals { - let lit_price = weight(opt_ptr.lit_freq.at(usize::from(literal))?, opt_level); - price = price.wrapping_sub(lit_price.min(lit_price_max)); - } - Ok(price) -} - -/// `ZSTD_litLengthPrice`: the price of the symbol for a length of literals. -#[inline(always)] -fn lit_length_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - if opt_ptr.price_type == OptPrice::Predef { - return Ok(weight(lit_length, opt_level)); - } - - // The format has no code for the length of a whole block of the largest size, which is then all literals. It is - // made to cost 1 bit more than the length before it. - let (lit_length, extra) = if lit_length == ZSTD_BLOCKSIZE_MAX as u32 { - (ZSTD_BLOCKSIZE_MAX as u32 - 1, BITCOST_MULTIPLIER) - } else { - (lit_length, 0) - }; - - let ll_code = ll_code(lit_length) as usize; - Ok(extra - .wrapping_add(u32::from(LL_BITS.at(ll_code)?) * BITCOST_MULTIPLIER) - .wrapping_add(opt_ptr.lit_length_sum_base_price) - .wrapping_sub(weight(opt_ptr.lit_length_freq.at(ll_code)?, opt_level))) -} - -/// `ZSTD_getMatchPrice`: the price of the offset and of the match length of a sequence. `off_base` is as in -/// [`SeqStore::store_seq`]. Below level 2, it favours small offsets, which are faster to decompress. -#[inline(always)] -fn get_match_price( - off_base: u32, - match_length: u32, - opt_ptr: &OptState, - opt_level: i32, -) -> Result { - let off_code = highbit32(off_base); - let ml_base = match_length.wrapping_sub(MINMATCH); - - if opt_ptr.price_type == OptPrice::Predef { - return Ok(weight(ml_base, opt_level).wrapping_add((16 + off_code) * BITCOST_MULTIPLIER)); - } - - let off_code_weight = weight(opt_ptr.off_code_freq.at(off_code as usize)?, opt_level); - let mut price = (off_code * BITCOST_MULTIPLIER).wrapping_add( - opt_ptr - .off_code_sum_base_price - .wrapping_sub(off_code_weight), - ); - if opt_level < 2 && off_code >= 20 { - price = price.wrapping_add((off_code - 19) * 2 * BITCOST_MULTIPLIER); - } - - let ml_code = ml_code(ml_base) as usize; - let ml_code_weight = weight(opt_ptr.match_length_freq.at(ml_code)?, opt_level); - price = price - .wrapping_add(u32::from(ML_BITS.at(ml_code)?) * BITCOST_MULTIPLIER) - .wrapping_add( - opt_ptr - .match_length_sum_base_price - .wrapping_sub(ml_code_weight), - ); - - // Matches cost a bit more, to have fewer sequences, which is faster to decompress. - Ok(price.wrapping_add(BITCOST_MULTIPLIER / 5)) -} - -/// `ZSTD_updateStats` -#[inline(always)] -fn update_stats( - opt_ptr: &mut OptState, - literals: &[u8], - off_base: u32, - match_length: u32, -) -> Result<()> { - #[inline(always)] - fn increment(freqs: &mut [u32], code: u32, sum: &mut u32) -> Result<()> { - let freq = freqs.at_mut(code as usize)?; - *freq = freq.wrapping_add(1); - *sum = sum.wrapping_add(1); - Ok(()) - } - - let lit_length = literals.len() as u32; - if compressed_literals(opt_ptr) { - for &literal in literals { - let freq = opt_ptr.lit_freq.at_mut(usize::from(literal))?; - *freq = freq.wrapping_add(ZSTD_LITFREQ_ADD); - } - opt_ptr.lit_sum = opt_ptr - .lit_sum - .wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD)); - } - - increment( - &mut opt_ptr.lit_length_freq, - ll_code(lit_length), - &mut opt_ptr.lit_length_sum, - )?; - increment( - &mut opt_ptr.off_code_freq, - highbit32(off_base), - &mut opt_ptr.off_code_sum, - )?; - increment( - &mut opt_ptr.match_length_freq, - ml_code(match_length.wrapping_sub(MINMATCH)), - &mut opt_ptr.match_length_sum, - ) -} - -/// `ZSTD_readMINMATCH`: `length` bytes, 3 or 4, to compare with others. Reads 4. -#[inline(always)] -fn read_minmatch(buf: &[u8], pos: usize, length: u32) -> Result { - let value = read_le32(buf, pos)?; - Ok(if length == 3 { value << 8 } else { value }) -} - -/// `ZSTD_insertAndFindFirstIndexHash3`: indexes the 3-byte hashes up to `ip`, which is not included, and looks it up. -/// All of it is in the current segment. -fn insert_and_find_first_index_hash3( - ms: &mut MatchState, - next_to_update3: &mut u32, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - let hash_table3 = ms.hash_table3.as_mut_slice(); - let hash_log3 = ms.hash_log3; - let target = prefix.index_of(ip); - let hash3 = hash3_ptr(prefix.buf, ip, hash_log3)?; - - for idx in *next_to_update3..target { - hash_table3.put(hash3_ptr(prefix.buf, prefix.pos_of(idx), hash_log3)?, idx)?; - } - - *next_to_update3 = target; - hash_table3.at(hash3) -} - -// ── binary tree search ──────────────────────────────────────────────────── - -/// `*ptr = value`, where `ptr` is an element of `bt`, or `None` for a place that nothing reads. -#[inline(always)] -fn store(bt: &mut [u32], ptr: Option, value: u32) -> Result<()> { - match ptr { - Some(ptr) => bt.put(ptr, value), - None => Ok(()), - } -} - -/// `ZSTD_insertBt1`: adds the position `ip`, which is at most `iend - 8`, to the tree. `target` is up to where the -/// caller is filling it, and `dict` the older segment, which is looked at if `ext_dict`. Returns how many positions -/// to move forward. -#[inline(always)] -fn insert_bt1( - ms: &mut MatchState, - prefix: Prefix<'_>, - dict: Prefix<'_>, - ip: usize, - iend: usize, - target: u32, - mls: u32, - ext_dict: bool, -) -> Result { - let c_params = ms.c_params; - let h = hash_ptr(prefix.buf, ip, c_params.hash_log, mls)?; - let bt_log = c_params.chain_log.wrapping_sub(1); - let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = dict.pos_of(dict_limit); - let prefix_start = prefix.pos_of(dict_limit); - let curr = prefix.index_of(ip); - let bt_low = curr.saturating_sub(bt_mask); - let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); - let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); - // Only the positions that are still in the window once the tree is filled are of use. - let window_low = ms.get_lowest_match_index(target, c_params.window_log); - let mut match_end_idx = curr.wrapping_add(8 + 1); - let mut best_length = 8usize; - let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); - - let bt = ms.chain_table.as_mut_slice(); - let hash_entry = ms.hash_table.at_mut(h)?; - let mut match_index = *hash_entry; - *hash_entry = curr; - - while nb_compares != 0 && match_index >= window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - // What is sure to be common. - let mut match_length = common_length_smaller.min(common_length_larger); - - // A position of the older segment, as one of the current segment, only makes sense with a length that - // brings it there. - let mut match_buf = prefix.buf; - let mut match_pos = prefix.pos_of(match_index); - if !ext_dict || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - prefix.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - match_length += count_2segments( - prefix.buf, - ip + match_length, - dict.buf, - dict.pos_of(match_index) + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - match_buf = dict.buf; - match_pos = dict.pos_of(match_index); - } - } - - if match_length > best_length { - best_length = match_length; - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); - } - } - - if ip + match_length == iend { - // They are equal: there is no telling which is smaller. Dropping it keeps the tree consistent. - break; - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < prefix.buf.at(ip + match_length)? { - // All the smaller ones will have at least this much in common. - store(bt, smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. - smaller_ptr = None; - break; - } - smaller_ptr = Some(next_ptr + 1); - match_index = bt.at(next_ptr + 1)?; - } else { - store(bt, larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = None; - break; - } - larger_ptr = Some(next_ptr); - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - store(bt, smaller_ptr, 0)?; - store(bt, larger_ptr, 0)?; - - let positions = if best_length > 384 { - 192.min((best_length - 384) as u32) - } else { - 0 - }; - Ok(positions.max(match_end_idx.wrapping_sub(curr.wrapping_add(8)))) -} - -/// `ZSTD_updateTree_internal` -#[inline(always)] -fn update_tree_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - dict: Prefix<'_>, - ip: usize, - iend: usize, - mls: u32, - dict_mode: DictMode, -) -> Result<()> { - let target = prefix.index_of(ip); - let mut idx = ms.next_to_update; - - while idx < target { - let forward = insert_bt1( - ms, - prefix, - dict, - prefix.pos_of(idx), - iend, - target, - mls, - dict_mode == DictMode::ExtDict, - )?; - idx = idx.wrapping_add(forward); - } - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. -/// Matches are not compared past the position `iend`. -pub(crate) fn update_tree( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - iend: usize, -) -> Result<()> { - let no_dict = Prefix { buf: &[], base: 0 }; - let mls = ms.c_params.min_match; - update_tree_internal(ms, prefix, no_dict, ip, iend, mls, DictMode::NoDict) -} - -/// `ZSTD_insertBtAndGetAllMatches`: adds the position `ip` to the tree, and stores in `matches` the matches found on -/// the way, each longer than the one before and all at least `length_to_beat` long. `ll0` is whether no literals -/// come before `ip`. Returns how many there are. -#[inline(always)] -fn insert_bt_and_get_all_matches( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - dict_mode: DictMode, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, - mls: u32, -) -> Result { - let buf = src.buf; - let c_params = ms.c_params; - let sufficient_len = c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); - let curr = src.index_of(ip); - let min_match = if mls == 3 { 3 } else { 4 }; - let h = hash_ptr(buf, ip, c_params.hash_log, mls)?; - let bt_log = c_params.chain_log.wrapping_sub(1); - let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt_low = curr.saturating_sub(bt_mask); - let window_low = ms.get_lowest_match_index(curr, c_params.window_log); - let match_low = window_low.max(1); - let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); - let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); - // The farthest position any match refers to, which detects repetitive patterns. - let mut match_end_idx = curr.wrapping_add(8 + 1); - let mut mnum = 0u32; - let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); - - let dms = if dict_mode == DictMode::DictMatchState { - Some(src.dms.ok_or(BUG)?) - } else { - None - }; - let dms_high_limit = dms.map_or(0, |dms| dms.end.wrapping_sub(dms.base) as u32); - let dms_low_limit = dms.map_or(0, |dms| dms.ms.window.low_limit); - let dms_index_delta = dms.map_or(0, |_| window_low.wrapping_sub(dms_high_limit)); - - let mut best_length = length_to_beat.wrapping_sub(1) as usize; - - // The repeat offsets. - let ll0 = u32::from(ll0); - let [rep0, _, _] = *rep; - for rep_code in ll0..ZSTD_REP_NUM as u32 + ll0 { - let rep_offset = match rep.get(rep_code as usize) { - Some(&offset) => offset, - None => rep0.wrapping_sub(1), - }; - let rep_index = curr.wrapping_sub(rep_offset); - let mut rep_len = 0u32; - // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. - if rep_offset.wrapping_sub(1) < curr.wrapping_sub(dict_limit) { - // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. - let rep_match = ip.wrapping_sub(rep_offset as usize); - if rep_index >= window_low - && read_minmatch(buf, ip, min_match)? == read_minmatch(buf, rep_match, min_match)? - { - rep_len = count( - buf, - ip + min_match as usize, - buf, - rep_match + min_match as usize, - i_limit, - )? as u32 - + min_match; - } - } else if dict_mode == DictMode::ExtDict { - // `curr > rep_index >= window_low` - if rep_offset.wrapping_sub(1) < curr.wrapping_sub(window_low) - && index_overlap_check(dict_limit, rep_index) - { - let rep_match = src.dict_pos_of(rep_index); - if read_minmatch(buf, ip, min_match)? - == read_minmatch(src.dict_buf, rep_match, min_match)? - { - rep_len = count_2segments( - buf, - ip + min_match as usize, - src.dict_buf, - rep_match + min_match as usize, - i_limit, - dict_end, - prefix_start, - )? as u32 - + min_match; - } - } - } else if let Some(dms) = dms { - // `curr > rep_index >= dms_low_limit` - if rep_offset.wrapping_sub(1) - < curr.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta)) - && index_overlap_check(dict_limit, rep_index) - { - let rep_match = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); - if read_minmatch(buf, ip, min_match)? - == read_minmatch(dms.buf, rep_match, min_match)? - { - rep_len = count_2segments( - buf, - ip + min_match as usize, - dms.buf, - rep_match + min_match as usize, - i_limit, - dms.end, - prefix_start, - )? as u32 - + min_match; - } - } - } - if rep_len as usize > best_length { - best_length = rep_len as usize; - matches.put( - mnum as usize, - Match { - off: rep_code - ll0 + 1, - len: rep_len, - }, - )?; - mnum += 1; - if rep_len > sufficient_len || ip + rep_len as usize == i_limit { - // The best there can be. - return Ok(mnum); - } - } - } - - // The table of 3-byte hashes. That of a dictionary is not filled. - if mls == 3 && best_length < mls as usize { - let match_index3 = - insert_and_find_first_index_hash3(ms, next_to_update3, src.prefix(), ip)?; - // Farther than that is likely to cost too much. - if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) { - let mlen = if dict_mode != DictMode::ExtDict || match_index3 >= dict_limit { - count(buf, ip, buf, src.pos_of(match_index3), i_limit)? - } else { - count_2segments( - buf, - ip, - src.dict_buf, - src.dict_pos_of(match_index3), - i_limit, - dict_end, - prefix_start, - )? - }; - - if mlen >= mls as usize { - best_length = mlen; - matches.put( - 0, - Match { - off: offset_to_offbase(curr.wrapping_sub(match_index3)), - len: mlen as u32, - }, - )?; - mnum = 1; - if mlen > sufficient_len as usize || ip + mlen == i_limit { - // The best there can be: `ip` is not inserted. - ms.next_to_update = curr.wrapping_add(1); - return Ok(1); - } - } - } - } - - let bt = ms.chain_table.as_mut_slice(); - let hash_entry = ms.hash_table.at_mut(h)?; - let mut match_index = *hash_entry; - *hash_entry = curr; - - while nb_compares != 0 && match_index >= match_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - // What is sure to be common. - let mut match_length = common_length_smaller.min(common_length_larger); - - // A position of the older segment, as one of the current segment, only makes sense with a length that - // brings it there. - let mut match_buf = buf; - let mut match_pos = src.pos_of(match_index); - if dict_mode != DictMode::ExtDict - || match_index as usize + match_length >= dict_limit as usize - { - match_length += count( - buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - i_limit, - )?; - } else { - match_length += count_2segments( - buf, - ip + match_length, - src.dict_buf, - src.dict_pos_of(match_index) + match_length, - i_limit, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - match_buf = src.dict_buf; - match_pos = src.dict_pos_of(match_index); - } - } - - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); - } - best_length = match_length; - matches.put( - mnum as usize, - Match { - off: offset_to_offbase(curr.wrapping_sub(match_index)), - len: match_length as u32, - }, - )?; - mnum += 1; - if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { - // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. - if dict_mode == DictMode::DictMatchState { - // The dictionary is not to be searched either. - nb_compares = 0; - } - break; - } - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < buf.at(ip + match_length)? { - // All the smaller ones will have at least this much in common. - store(bt, smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. - smaller_ptr = None; - break; - } - smaller_ptr = Some(next_ptr + 1); - match_index = bt.at(next_ptr + 1)?; - } else { - store(bt, larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = None; - break; - } - larger_ptr = Some(next_ptr); - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - store(bt, smaller_ptr, 0)?; - store(bt, larger_ptr, 0)?; - - if let Some(dms) = dms - && nb_compares != 0 - { - let dms_c_params = &dms.ms.c_params; - let dms_bt_log = dms_c_params.chain_log.wrapping_sub(1); - let dms_bt_mask = 1u32.wrapping_shl(dms_bt_log).wrapping_sub(1); - let dms_bt_low = if dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit) { - dms_high_limit.wrapping_sub(dms_bt_mask) - } else { - dms_low_limit - }; - let dms_h = hash_ptr(buf, ip, dms_c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(dms_h)?; - let dms_bt = dms.ms.chain_table.as_slice(); - common_length_smaller = 0; - common_length_larger = 0; - while nb_compares != 0 && dict_match_index > dms_low_limit { - let next_ptr = 2 * (dict_match_index & dms_bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - match_length += count_2segments( - buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - i_limit, - dms.end, - prefix_start, - )?; - - if match_length > best_length { - match_index = dict_match_index.wrapping_add(dms_index_delta); - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); - } - best_length = match_length; - matches.put( - mnum as usize, - Match { - off: offset_to_offbase(curr.wrapping_sub(match_index)), - len: match_length as u32, - }, - )?; - mnum += 1; - if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { - break; - } - } - - if dict_match_index <= dms_bt_low { - // Beyond the tree. - break; - } - // A match that reaches the end of the dictionary goes on at the start of the current segment. - let match_byte = if dict_match_index as usize + match_length >= dms_high_limit as usize - { - let match_pos = src.pos_of(dict_match_index.wrapping_add(dms_index_delta)); - buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? - }; - if match_byte < buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dms_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dms_bt.at(next_ptr)?; - } - nb_compares -= 1; - } - } - - // Repetitive patterns are skipped. - ms.next_to_update = match_end_idx.wrapping_sub(8); - Ok(mnum) -} - -/// `ZSTD_getAllMatchesFn`: the arguments are `matches`, `ms`, `next_to_update3`, `src`, `ip`, `i_high_limit`, `rep`, -/// `ll0` and `length_to_beat`. -type GetAllMatchesFn = fn( - &mut [Match], - &mut MatchState, - &mut u32, - &BlockSrc<'_>, - usize, - usize, - &[u32; ZSTD_REP_NUM], - bool, - u32, -) -> Result; - -/// `ZSTD_btGetAllMatches_internal` -#[inline(always)] -fn bt_get_all_matches_internal( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, - dict_mode: DictMode, - mls: u32, -) -> Result { - if src.index_of(ip) < ms.next_to_update { - // An area that is skipped. - return Ok(0); - } - let dict = Prefix { - buf: src.dict_buf, - base: src.dict_base, - }; - update_tree_internal(ms, src.prefix(), dict, ip, i_high_limit, mls, dict_mode)?; - insert_bt_and_get_all_matches( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - dict_mode, - rep, - ll0, - length_to_beat, - mls, - ) -} - -/// `ZSTD_btGetAllMatches_noDict_3` to `ZSTD_btGetAllMatches_noDict_6` -fn bt_get_all_matches_no_dict( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::NoDict, - MLS, - ) -} - -/// `ZSTD_btGetAllMatches_extDict_3` to `ZSTD_btGetAllMatches_extDict_6` -fn bt_get_all_matches_ext_dict( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::ExtDict, - MLS, - ) -} - -/// `ZSTD_btGetAllMatches_dictMatchState_3` to `ZSTD_btGetAllMatches_dictMatchState_6` -fn bt_get_all_matches_dict_match_state( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::DictMatchState, - MLS, - ) -} - -/// `ZSTD_selectBtGetAllMatches` -fn select_bt_get_all_matches(ms: &MatchState, dict_mode: DictMode) -> GetAllMatchesFn { - match (dict_mode, ms.c_params.min_match) { - (DictMode::NoDict, ..=3) => bt_get_all_matches_no_dict::<3>, - (DictMode::NoDict, 4) => bt_get_all_matches_no_dict::<4>, - (DictMode::NoDict, 5) => bt_get_all_matches_no_dict::<5>, - (DictMode::NoDict, 6..) => bt_get_all_matches_no_dict::<6>, - (DictMode::ExtDict, ..=3) => bt_get_all_matches_ext_dict::<3>, - (DictMode::ExtDict, 4) => bt_get_all_matches_ext_dict::<4>, - (DictMode::ExtDict, 5) => bt_get_all_matches_ext_dict::<5>, - (DictMode::ExtDict, 6..) => bt_get_all_matches_ext_dict::<6>, - (DictMode::DictMatchState, ..=3) => bt_get_all_matches_dict_match_state::<3>, - (DictMode::DictMatchState, 4) => bt_get_all_matches_dict_match_state::<4>, - (DictMode::DictMatchState, 5) => bt_get_all_matches_dict_match_state::<5>, - (DictMode::DictMatchState, 6..) => bt_get_all_matches_dict_match_state::<6>, - } -} - -// ── candidates from long distance matching ──────────────────────────────── - -/// `ZSTD_optLdm_t`: what it takes to decide whether to use a long distance match. -struct OptLdm<'a> { - /// `seqStore.seq`: the `seqStore.size` candidates for this block and the next ones. - seq: &'a [RawSeq], - /// `seqStore.pos` - pos: usize, - /// `seqStore.posInSequence` - pos_in_sequence: usize, - /// Where the current candidate starts. - start_pos_in_block: u32, - /// Where it ends. - end_pos_in_block: u32, - offset: u32, -} - -impl<'a> OptLdm<'a> { - fn new(seq_store: Option<&'a RawSeqStore>) -> Self { - Self { - seq: seq_store - .and_then(|store| store.seq.get(..store.size)) - .unwrap_or_default(), - pos: seq_store.map_or(0, |store| store.pos), - pos_in_sequence: seq_store.map_or(0, |store| store.pos_in_sequence), - start_pos_in_block: 0, - end_pos_in_block: 0, - offset: 0, - } - } - - /// `ZSTD_optLdm_skipRawSeqStoreBytes`: moves forward by `nb_bytes`. - fn skip_raw_seq_store_bytes(&mut self, nb_bytes: usize) { - let mut curr_pos = self.pos_in_sequence.wrapping_add(nb_bytes) as u32; - while curr_pos != 0 { - let Some(curr_seq) = self.seq.get(self.pos) else { - break; - }; - let seq_length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); - if curr_pos >= seq_length { - curr_pos -= seq_length; - self.pos += 1; - } else { - self.pos_in_sequence = curr_pos as usize; - break; - } - } - if curr_pos == 0 || self.pos == self.seq.len() { - self.pos_in_sequence = 0; - } - } - - /// `ZSTD_opt_getNextMatchAndUpdateSeqStore`: finds where the next candidate starts and ends in the block, and - /// moves past it. - fn get_next_match_and_update_seq_store( - &mut self, - curr_pos_in_block: u32, - block_bytes_remaining: u32, - ) { - let Some(&curr_seq) = self.seq.get(self.pos) else { - // No position gets there, so that there is no candidate in this block. - self.start_pos_in_block = u32::MAX; - self.end_pos_in_block = u32::MAX; - return; - }; - // What is left of the literals and of the match. - let pos_in_sequence = self.pos_in_sequence as u32; - let curr_block_end_pos = curr_pos_in_block.wrapping_add(block_bytes_remaining); - let literals_bytes_remaining = curr_seq.lit_length.saturating_sub(pos_in_sequence); - let match_bytes_remaining = if literals_bytes_remaining == 0 { - curr_seq - .match_length - .wrapping_sub(pos_in_sequence.wrapping_sub(curr_seq.lit_length)) - } else { - curr_seq.match_length - }; - - if literals_bytes_remaining >= block_bytes_remaining { - // The match starts after the block. - self.start_pos_in_block = u32::MAX; - self.end_pos_in_block = u32::MAX; - self.skip_raw_seq_store_bytes(block_bytes_remaining as usize); - return; - } - - // What is left of a match can be too short to use, which is checked later. - self.start_pos_in_block = curr_pos_in_block.wrapping_add(literals_bytes_remaining); - self.end_pos_in_block = self.start_pos_in_block.wrapping_add(match_bytes_remaining); - self.offset = curr_seq.offset; - - if self.end_pos_in_block > curr_block_end_pos { - // The match ends after the block, which can only use its start. - self.end_pos_in_block = curr_block_end_pos; - self.skip_raw_seq_store_bytes( - curr_block_end_pos.wrapping_sub(curr_pos_in_block) as usize - ); - } else { - self.skip_raw_seq_store_bytes( - literals_bytes_remaining.wrapping_add(match_bytes_remaining) as usize, - ); - } - } - - /// `ZSTD_optLdm_maybeAddMatch`: adds what is left of the candidate at `curr_pos_in_block` to `matches`, if it is - /// long enough and longer than all of them, since they are in order of length. - fn maybe_add_match( - &self, - matches: &mut [Match], - nb_matches: &mut u32, - curr_pos_in_block: u32, - min_match: u32, - ) -> Result<()> { - let pos_diff = curr_pos_in_block.wrapping_sub(self.start_pos_in_block); - let candidate_match_length = self - .end_pos_in_block - .wrapping_sub(self.start_pos_in_block) - .wrapping_sub(pos_diff); - - if curr_pos_in_block < self.start_pos_in_block - || curr_pos_in_block >= self.end_pos_in_block - || candidate_match_length < min_match - { - return Ok(()); - } - - if *nb_matches == 0 - || (candidate_match_length > matches.at(*nb_matches as usize - 1)?.len - && (*nb_matches as usize) < ZSTD_OPT_NUM) - { - matches.put( - *nb_matches as usize, - Match { - off: offset_to_offbase(self.offset), - len: candidate_match_length, - }, - )?; - *nb_matches += 1; - } - Ok(()) - } - - /// `ZSTD_optLdm_processMatchCandidate` - #[inline(always)] - fn process_match_candidate( - &mut self, - matches: &mut [Match], - nb_matches: &mut u32, - curr_pos_in_block: u32, - remaining_bytes: u32, - min_match: u32, - ) -> Result<()> { - if self.pos >= self.seq.len() { - return Ok(()); - } - - if curr_pos_in_block >= self.end_pos_in_block { - if curr_pos_in_block > self.end_pos_in_block { - // The parser does not stop at the end of each candidate: it is often some bytes past it. - let pos_overshoot = curr_pos_in_block - self.end_pos_in_block; - self.skip_raw_seq_store_bytes(pos_overshoot as usize); - } - self.get_next_match_and_update_seq_store(curr_pos_in_block, remaining_bytes); - } - self.maybe_add_match(matches, nb_matches, curr_pos_in_block, min_match) - } -} - -// ── the optimal parser ──────────────────────────────────────────────────── - -/// `LIT_PRICE`: the price of the literal at `pos`. -#[inline(always)] -fn lit_price(buf: &[u8], pos: usize, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok(raw_literals_cost(buf.span(pos, 1)?, opt_ptr, opt_level)? as i32) -} - -/// `LL_PRICE` -#[inline(always)] -fn ll_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok(lit_length_price(lit_length, opt_ptr, opt_level)? as i32) -} - -/// `LL_INCPRICE`: what one more literal, which makes `lit_length` of them, adds to the price of their length. -#[inline(always)] -fn ll_inc_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok( - ll_price(lit_length, opt_ptr, opt_level)?.wrapping_sub(ll_price( - lit_length.wrapping_sub(1), - opt_ptr, - opt_level, - )?), - ) -} - -/// `ZSTD_compressBlock_opt_generic` -#[inline(always)] -fn compress_block_opt_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - opt_level: i32, - dict_mode: DictMode, -) -> Result { - // The match finders borrow the whole of `ms`. - let mut price_table = mem::take(&mut ms.opt.price_table); - let mut match_table = mem::take(&mut ms.opt.match_table); - let result = parse_block( - ms, - seq_store, - rep, - src, - &mut price_table, - &mut match_table, - opt_level, - dict_mode, - ); - ms.opt.price_table = price_table; - ms.opt.match_table = match_table; - result -} - -/// The body of `ZSTD_compressBlock_opt_generic`. `opt` is `ms.opt.price_table` and `matches` is `ms.opt.match_table`. -#[inline(always)] -fn parse_block( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - opt: &mut [Optimal], - matches: &mut [Match], - opt_level: i32, - dict_mode: DictMode, -) -> Result { - let buf = src.buf; - let istart = src.start; - let mut ip = istart; - let mut anchor = istart; - let iend = src.end; - let ilimit = iend.saturating_sub(8); - let prefix_start = src.pos_of(ms.window.dict_limit); - - let get_all_matches = select_bt_get_all_matches(ms, dict_mode); - - let sufficient_len = ms.c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); - let min_match = if ms.c_params.min_match == 3 { 3 } else { 4 }; - let mut next_to_update3 = ms.next_to_update; - - let mut last_stretch = Optimal::default(); - let mut opt_ldm = OptLdm::new(src.ldm_seq_store); - opt_ldm.get_next_match_and_update_seq_store(0, (iend - ip) as u32); - - rescale_freqs( - &mut ms.opt, - src.symbol_costs, - buf.sub(istart..iend)?, - opt_level, - )?; - ip += usize::from(ip == prefix_start); - - 'series: while ip < ilimit { - let mut last_pos: u32; - - // The forward pass. `opt` holds stretches, each a match followed by literals, unlike a sequence, which is - // literals followed by a match: that way each position of a run of literals can have another match before it. - // It ends with the last stretch in `last_stretch`, which starts at `cur` and ends at `last_pos`. - let mut cur = 'forward: { - // Find a first match. - { - let litlen = (ip - anchor) as u32; - let mut nb_matches = get_all_matches( - matches, - ms, - &mut next_to_update3, - src, - ip, - iend, - rep, - litlen == 0, - min_match, - )?; - opt_ldm.process_match_candidate( - matches, - &mut nb_matches, - (ip - istart) as u32, - (iend - ip) as u32, - min_match, - )?; - if nb_matches == 0 { - ip += 1; - continue 'series; - } - let matches = matches.sub(..nb_matches as usize)?; - - // The price of the literals before the first match is the same whatever comes next, so it is left - // out. But how the price of their length changes depends on their length. - let first_price = ll_price(litlen, &ms.opt, opt_level)?; - { - let first = opt.at_mut(0)?; - first.mlen = 0; - first.litlen = litlen; - first.price = first_price; - first.rep = *rep; - } - - // A large match is encoded at once. - let longest = matches.at(matches.len() - 1)?; - if longest.len > sufficient_len { - last_stretch.litlen = 0; - last_stretch.mlen = longest.len; - last_stretch.off = longest.off; - last_pos = longest.len; - break 'forward 0; - } - - // Set the prices of the first matches. - let ll0_price = ll_price(0, &ms.opt, opt_level)?; - let mut pos = 1u32; - while pos < min_match { - let stretch = opt.at_mut(pos as usize)?; - stretch.price = ZSTD_MAX_PRICE; - stretch.mlen = 0; - stretch.litlen = litlen.wrapping_add(pos); - pos += 1; - } - for candidate in matches { - while pos <= candidate.len { - let match_price = - get_match_price(candidate.off, pos, &ms.opt, opt_level)? as i32; - let sequence_price = first_price.wrapping_add(match_price); - let stretch = opt.at_mut(pos as usize)?; - stretch.mlen = pos; - stretch.off = candidate.off; - stretch.litlen = 0; - stretch.price = sequence_price.wrapping_add(ll0_price); - pos += 1; - } - } - last_pos = pos - 1; - opt.at_mut(pos as usize)?.price = ZSTD_MAX_PRICE; - } - - // Check the positions that follow. - let mut cur = 0u32; - while cur < last_pos { - cur += 1; - let inr = ip + cur as usize; - - // Get to this position by a literal, if it is cheaper. - { - let before = opt.at(cur as usize - 1)?; - let litlen = before.litlen.wrapping_add(1); - let price = before - .price - .wrapping_add(lit_price(buf, inr - 1, &ms.opt, opt_level)?) - .wrapping_add(ll_inc_price(litlen, &ms.opt, opt_level)?); - let prev_match = opt.at(cur as usize)?; - if price <= prev_match.price { - opt.put( - cur as usize, - Optimal { - litlen, - price, - ..before - }, - )?; - // If this replaces a match, and 1 literal is cheaper than none, the match followed by a - // literal may be a cheaper way to the next position. - if opt_level >= 1 - && prev_match.litlen == 0 - && ll_inc_price(1, &ms.opt, opt_level)? < 0 - && inr < iend - { - let next_lit_price = lit_price(buf, inr, &ms.opt, opt_level)?; - let with_1_literal = prev_match - .price - .wrapping_add(next_lit_price) - .wrapping_add(ll_inc_price(1, &ms.opt, opt_level)?); - let with_more_literals = price - .wrapping_add(next_lit_price) - .wrapping_add(ll_inc_price( - litlen.wrapping_add(1), - &ms.opt, - opt_level, - )?); - if with_1_literal < with_more_literals - && with_1_literal < opt.at(cur as usize + 1)?.price - { - let prev = opt.at(cur.wrapping_sub(prev_match.mlen) as usize)?; - opt.put( - cur as usize + 1, - Optimal { - rep: new_rep(&prev.rep, prev_match.off, prev.litlen == 0), - litlen: 1, - price: with_1_literal, - ..prev_match - }, - )?; - last_pos = last_pos.max(cur + 1); - } - } - } - } - - // The repeat offsets are not updated while matches are compared, but here, now that one is selected. - let mut here = opt.at(cur as usize)?; - if here.litlen == 0 { - let prev = opt.at(cur.wrapping_sub(here.mlen) as usize)?; - here.rep = new_rep(&prev.rep, here.off, prev.litlen == 0); - opt.at_mut(cur as usize)?.rep = here.rep; - } - - // The last match starts at least 8 bytes before the end. - if inr > ilimit { - continue; - } - - if cur == last_pos { - break; - } - - if opt_level == 0 - && opt.at(cur as usize + 1)?.price - <= here.price.wrapping_add(BITCOST_MULTIPLIER as i32 / 2) - { - // The next position is cheaper to get to: this one is not promising. - continue; - } - - { - let base_price = here.price.wrapping_add(ll_price(0, &ms.opt, opt_level)?); - let mut nb_matches = get_all_matches( - matches, - ms, - &mut next_to_update3, - src, - inr, - iend, - &here.rep, - here.litlen == 0, - min_match, - )?; - opt_ldm.process_match_candidate( - matches, - &mut nb_matches, - (inr - istart) as u32, - (iend - inr) as u32, - min_match, - )?; - if nb_matches == 0 { - continue; - } - let matches = matches.sub(..nb_matches as usize)?; - - let longest = matches.at(matches.len() - 1)?; - if longest.len > sufficient_len - || cur.wrapping_add(longest.len) as usize >= ZSTD_OPT_NUM - || inr + longest.len as usize >= iend - { - last_stretch.mlen = longest.len; - last_stretch.off = longest.off; - last_stretch.litlen = 0; - last_pos = cur.wrapping_add(longest.len); - break 'forward cur; - } - - // Set the prices that the matches found here give. - let mut start_ml = min_match; - for candidate in matches { - let mut mlen = candidate.len; - while mlen >= start_ml { - let pos = cur + mlen; - let price = base_price.wrapping_add(get_match_price( - candidate.off, - mlen, - &ms.opt, - opt_level, - )? - as i32); - - if pos > last_pos || price < opt.at(pos as usize)?.price { - while last_pos < pos { - // Positions that nothing leads to yet. - last_pos += 1; - let stretch = opt.at_mut(last_pos as usize)?; - stretch.price = ZSTD_MAX_PRICE; - // Not 0, which would be the end of a match. - stretch.litlen = 1; - } - let stretch = opt.at_mut(pos as usize)?; - stretch.mlen = mlen; - stretch.off = candidate.off; - stretch.litlen = 0; - stretch.price = price; - } else if opt_level == 0 { - // Shorter lengths are not tried. - break; - } - mlen -= 1; - } - start_ml = candidate.len + 1; - } - } - opt.at_mut(last_pos as usize + 1)?.price = ZSTD_MAX_PRICE; - } - - last_stretch = opt.at(last_pos as usize)?; - last_pos.wrapping_sub(last_stretch.mlen) - }; - - if last_stretch.mlen == 0 { - // All the matches have been turned into literals. - ip += last_pos as usize; - continue; - } - - if last_stretch.litlen == 0 { - // It ends with a match. - let prev = opt.at(cur as usize)?; - *rep = new_rep(&prev.rep, last_stretch.off, prev.litlen == 0); - } else { - *rep = last_stretch.rep; - cur = cur.wrapping_sub(last_stretch.litlen); - } - - // Write the cheapest path in `opt`, backwards from `store_end`, over the stretches that are not on it, and as - // sequences. The literals that end the last stretch are in no sequence: they are parsed again. (The C code - // starts by making them a last sequence without a match, and overwrites it at once.) - let store_end = cur.wrapping_add(2) as usize; - let mut store_start = store_end; - let mut stretch_pos = cur; - opt.put(store_end, last_stretch)?; - loop { - let next_stretch = opt.at(stretch_pos as usize)?; - opt.at_mut(store_start)?.litlen = next_stretch.litlen; - if next_stretch.mlen == 0 { - // The start of the series. - break; - } - store_start = store_start.wrapping_sub(1); - opt.put(store_start, next_stretch)?; - stretch_pos = - stretch_pos.wrapping_sub(next_stretch.litlen.wrapping_add(next_stretch.mlen)); - } - - for sequence in opt.sub(store_start..=store_end)? { - let llen = sequence.litlen as usize; - let mlen = sequence.mlen as usize; - update_stats( - &mut ms.opt, - buf.span(anchor, llen)?, - sequence.off, - sequence.mlen, - )?; - seq_store.store_seq(llen, buf, anchor, sequence.off, mlen)?; - anchor += llen + mlen; - ip = anchor; - } - - set_base_prices(&mut ms.opt, opt_level); - } - - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_opt0` -fn compress_block_opt0( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode) -} - -/// `ZSTD_compressBlock_opt2` -fn compress_block_opt2( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode) -} - -/// `ZSTD_initStats_ultra`: a first pass over the block, only to have better statistics to start from. It has to be the -/// first block, with no dictionary and no long distance matching. Returns the block as the window then sees it, since -/// its indices change. -fn init_stats_ultra<'a>( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &[u32; ZSTD_REP_NUM], - src: &BlockSrc<'a>, -) -> Result> { - // The repeat offsets it leaves are of no use. - let mut tmp_rep = *rep; - compress_block_opt2(ms, seq_store, &mut tmp_rep, src, DictMode::NoDict)?; - - // Of the first pass, only the statistics are kept: what the tables have is made out of reach. - let src_size = src.end - src.start; - seq_store.reset(); - ms.window.base = ms.window.base.wrapping_sub(src_size); - ms.window.dict_limit = ms.window.dict_limit.wrapping_add(src_size as u32); - ms.window.low_limit = ms.window.dict_limit; - ms.next_to_update = ms.window.dict_limit; - Ok(BlockSrc { - base: src.base.wrapping_sub(src_size), - ..*src - }) -} - -/// `ZSTD_compressBlock_btultra2` -fn compress_block_btultra2( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let curr = src.index_of(src.start); - - // The first block is parsed twice, the first time to collect statistics for the second. Since the first pass is - // then forgotten, nothing must be in the tables before it: no dictionary, no prefix, no long distance matching. - if ms.opt.lit_length_sum == 0 - && seq_store.sequences.is_empty() - && ms.window.dict_limit == ms.window.low_limit - && curr == ms.window.dict_limit - && src.end - src.start > ZSTD_PREDEF_THRESHOLD - { - let src = init_stats_ultra(ms, seq_store, rep, src)?; - return compress_block_opt2(ms, seq_store, rep, &src, DictMode::NoDict); - } - - compress_block_opt2(ms, seq_store, rep, src, DictMode::NoDict) -} - -/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` -/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. -/// -/// `btultra2` can move `ms.window.base`, which `src.base` comes from. -pub(crate) fn compress_block_opt( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - dict_mode: DictMode, -) -> Result { - match (strategy, dict_mode) { - (Strategy::BtUltra2, DictMode::NoDict) => compress_block_btultra2(ms, seq_store, rep, src), - (Strategy::BtUltra | Strategy::BtUltra2, _) => { - compress_block_opt2(ms, seq_store, rep, src, dict_mode) - } - _ => compress_block_opt0(ms, seq_store, rep, src, dict_mode), - } -} diff --git a/src/zstd/wip/work/ldm/zstd/compress/presplit.rs b/src/zstd/wip/work/ldm/zstd/compress/presplit.rs deleted file mode 100644 index 6676175f0c1f..000000000000 --- a/src/zstd/wip/work/ldm/zstd/compress/presplit.rs +++ /dev/null @@ -1,184 +0,0 @@ -//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. - -use super::hist; -use crate::error::Result; -use crate::util::SliceExt; - -/// What the C code needs of workspace, which counts in the size of a context. -pub(crate) const ZSTD_SLIPBLOCK_WORKSPACESIZE: usize = 8208; - -const THRESHOLD_PENALTY_RATE: u64 = 16; -const THRESHOLD_BASE: i32 = THRESHOLD_PENALTY_RATE as i32 - 2; -const THRESHOLD_PENALTY: i32 = 3; - -const HASHLENGTH: usize = 2; -const KNUTH: u32 = 0x9e37_79b9; - -const CHUNKSIZE: usize = 8 << 10; -const SEGMENT_SIZE: usize = 512; - -/// `Fingerprint`, with a table of `SIZE` events, which is `1 << hashLog`: the C code leaves the rest of its table at 0. -struct Fingerprint { - events: [u32; SIZE], - nb_events: usize, -} - -impl Fingerprint { - const fn new() -> Self { - Self { - events: [0; SIZE], - nb_events: 0, - } - } - - /// `hash2`: a table of 256 takes the first byte as it is, a larger one a hash of the 2 bytes. - #[inline(always)] - const fn hash2(pair: [u8; HASHLENGTH]) -> usize { - if SIZE == 256 { - let [first, _] = pair; - return first as usize; - } - ((u16::from_le_bytes(pair) as u32).wrapping_mul(KNUTH) >> (32 - SIZE.trailing_zeros())) - as usize - } - - /// `addEvents_generic`: `src` has at least 2 bytes. - #[inline(always)] - fn add_events(&mut self, src: &[u8]) { - let limit = src.len().saturating_sub(HASHLENGTH - 1); - let mut n = 0; - while n < limit { - if let Some(pair) = src.get(n..).and_then(<[u8]>::first_chunk) - && let Some(event) = self.events.get_mut(Self::hash2(*pair) & (SIZE - 1)) - { - *event += 1; - } - n += SAMPLING_RATE; - } - self.nb_events += limit / SAMPLING_RATE; - } - - /// `recordFingerprint_generic` - #[inline(always)] - fn record(&mut self, src: &[u8]) { - self.events.fill(0); - self.nb_events = 0; - self.add_events::(src); - } - - /// `fpDistance` - fn distance(&self, other: &Self) -> u64 { - let nb_events1 = self.nb_events as i64; - let nb_events2 = other.nb_events as i64; - self.events - .iter() - .zip(&other.events) - .map(|(&event1, &event2)| { - (i64::from(event1) * nb_events2 - i64::from(event2) * nb_events1).unsigned_abs() - }) - .sum() - } - - /// `compareFingerprints`: whether `newfp` is too different from this one. - fn differs_from(&self, newfp: &Self, penalty: i32) -> bool { - let p50 = self.nb_events as u64 * newfp.nb_events as u64; - let deviation = self.distance(newfp); - let threshold = p50 * (THRESHOLD_BASE + penalty) as u64 / THRESHOLD_PENALTY_RATE; - deviation >= threshold - } - - /// `mergeEvents` - fn merge(&mut self, newfp: &Self) { - for (acc, &event) in self.events.iter_mut().zip(&newfp.events) { - *acc += event; - } - self.nb_events += newfp.nb_events; - } -} - -/// `ZSTD_splitBlock_byChunks`: cuts before the first chunk that is too different from all there is before it. -#[inline(always)] -fn split_block_by_chunks(block: &[u8]) -> usize { - let mut past_events = Fingerprint::::new(); - let mut new_events = Fingerprint::::new(); - let mut penalty = THRESHOLD_PENALTY; - let (chunks, _) = block.as_chunks::(); - let Some((first, chunks)) = chunks.split_first() else { - return block.len(); - }; - - past_events.record::(first); - let mut pos = CHUNKSIZE; - for chunk in chunks { - new_events.record::(chunk); - if past_events.differs_from(&new_events, penalty) { - return pos; - } - past_events.merge(&new_events); - if penalty > 0 { - penalty -= 1; - } - pos += CHUNKSIZE; - } - block.len() -} - -fn split_block_by_chunks_43(block: &[u8]) -> usize { - split_block_by_chunks::<43, { 1 << 8 }>(block) -} - -fn split_block_by_chunks_11(block: &[u8]) -> usize { - split_block_by_chunks::<11, { 1 << 9 }>(block) -} - -fn split_block_by_chunks_5(block: &[u8]) -> usize { - split_block_by_chunks::<5, { 1 << 10 }>(block) -} - -fn split_block_by_chunks_1(block: &[u8]) -> usize { - split_block_by_chunks::<1, { 1 << 10 }>(block) -} - -/// `ZSTD_splitBlock_fromBorders`: cuts if the start and the end of the block are too different. In the middle, or at a -/// quarter from the border that the middle is clearly the most different from. -fn split_block_from_borders(block: &[u8]) -> Result { - fn segment(block: &[u8], start: usize) -> Result> { - let mut fingerprint = Fingerprint::new(); - hist::add(&mut fingerprint.events, block.span(start, SEGMENT_SIZE)?)?; - fingerprint.nb_events = SEGMENT_SIZE; - Ok(fingerprint) - } - - let block_size = block.len(); - let past_events = segment(block, 0)?; - let new_events = segment(block, block_size.wrapping_sub(SEGMENT_SIZE))?; - if !past_events.differs_from(&new_events, 0) { - return Ok(block_size); - } - - let middle_events = segment(block, (block_size / 2).wrapping_sub(SEGMENT_SIZE / 2))?; - let dist_from_begin = past_events.distance(&middle_events); - let dist_from_end = new_events.distance(&middle_events); - let min_distance = (SEGMENT_SIZE * SEGMENT_SIZE / 3) as u64; - if dist_from_begin.abs_diff(dist_from_end) < min_distance { - return Ok(64 << 10); - } - Ok(if dist_from_begin > dist_from_end { - 32 << 10 - } else { - 96 << 10 - }) -} - -/// `ZSTD_splitBlock`: `block` is 128 KB, and `level` is from 0 to 4: the higher, the more work is put in. Returns the -/// size of the first part, which is the whole if the block is better left in one piece. -pub(crate) fn split_block(block: &[u8], level: i32) -> Result { - debug_assert!(block.len() == 128 << 10); - Ok(match level { - 0 => split_block_from_borders(block)?, - 1 => split_block_by_chunks_43(block), - 2 => split_block_by_chunks_11(block), - 3 => split_block_by_chunks_5(block), - _ => split_block_by_chunks_1(block), - }) -} diff --git a/src/zstd/wip/work/opt/STATUS.md b/src/zstd/wip/work/opt/STATUS.md new file mode 100644 index 000000000000..8eec4032674e --- /dev/null +++ b/src/zstd/wip/work/opt/STATUS.md @@ -0,0 +1,38 @@ +# opt (zstd_opt.c -> compress/opt.rs): status at 12:30 UTC + +Deliverable: /root/zstd-ref/work/opt/zstd/compress/opt.rs. Complete port, public API of the stub unchanged. +No clippy finding in opt.rs, rustfmt clean (the fmt diffs of `cargo fmt --check` are all in the scratch-only testing.rs). + +## Verified against C (no difference found in any run of the final code) +Harness = zstd-lazy's fixed one, extended. Whole frames (several blocks, up to 400 KB), strategies 7,8,9. Compared: all +sequences, and after the frame hashTable, chainTable, hashTable3, window limits, nextToUpdate, rep, the 4 frequency +tables and the 8 sums / base prices of optState_t. +- Long runs, 100,000 iterations each, still running when this was written, no failure so far. Last progress lines: + seqs iter 84000 (seed 101084303);seqs iter 82000 (seed 102082306);seqs iter 84000 (seed 201084603);seqs iter 82000 (seed 202082606);seqs iter 88000 (seed 301088903); + (r0a, r0b: no dict; r1a, r1b: none/prefix = extDict; r2a: attached CDict = dictMatchState). Logs: runs/*.log +- Finished runs: modes 0, 1, 2: 3000 + 1000 + 800 + 1500 iterations each. +- "x" mode (shim_frame2 in harness/shim.c: block-level API, exact unadjusted cParams incl. targetLength 0, random LDM + rawSeqs through ZSTD_referenceExternalSequences, poked dictionary entropy tables with HUF_repeat_valid, + literalCompressionMode 0/1/2, input in two non-contiguous chunks, none/prefix/attached CDict): + 3000 + 3000 + 800 + 1500 iterations, about 2/3 with LDM sequences and 1/2 with entropy tables. +- Line + branch coverage of opt.rs under those runs: every line is hit but the ones listed below. + +## Not covered by any test +- zop_predef prices (3 early returns): cannot be reached, a first block of <= 8 bytes never enters the match loop. +- raw_literals_cost with 0 literals: never called so. +- lit_length_price(ZSTD_BLOCKSIZE_MAX): reachable (128 KB block without a sequence whose last 9+ bytes match). A data + kind for it was added to seqs.rs (noise + copied tail) but did not hit it in 6000 iterations. +- `inr < iend` false, `nb_matches < ZSTD_OPT_NUM` false, and btultra2's 2nd to 4th conditions false. +- Index overflow correction, windows > 22 bits. + +## How to run +cd harness && RUSTFLAGS="--cap-lints=warn" cargo build --release +./target/release/zstd_harness seqs 7,8,9 <0|1|2|3> [x] + 0 no dict, 1 none/prefix, 2 attached CDict, 3 all three; x = shim_frame2 extras +ZSTD_REF=asserts|trace picks a C lib built by cbuild/build.sh (asserts on / DEBUGLEVEL 8 in zstd_opt.c). + +## Notes for the integrator +- btultra2 (ZSTD_initStats_ultra) moves ms.window.base: BlockSrc.base must be derived from ms.window again for each block. +- The template seqs.rs sets cParams by 101+i, which swaps chainLog and hashLog (ids are windowLog, hashLog, chainLog). + The fixed harness is immune because it reads the applied cParams back from C. +- MAX_LIT (255) is missing from common/mod.rs: private const in opt.rs. diff --git a/src/zstd/wip/work/opt/zstd/compress/double_fast.rs b/src/zstd/wip/work/opt/zstd/compress/double_fast.rs deleted file mode 100644 index 65d401675d1b..000000000000 --- a/src/zstd/wip/work/opt/zstd/compress/double_fast.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. - -use super::internal::{ - BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_double_hash_table( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _end: usize, - _dtlm: DictTableLoadMethod, - _tfp: TableFillPurpose, -) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast` -pub(crate) fn compress_block_double_fast( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast_dictMatchState` -pub(crate) fn compress_block_double_fast_dict_match_state( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_doubleFast_extDict` -pub(crate) fn compress_block_double_fast_ext_dict( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/opt/zstd/compress/fast.rs b/src/zstd/wip/work/opt/zstd/compress/fast.rs deleted file mode 100644 index 1f4788af28b4..000000000000 --- a/src/zstd/wip/work/opt/zstd/compress/fast.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. - -use super::internal::{ - BlockSrc, DictTableLoadMethod, MatchState, Prefix, SeqStore, TableFillPurpose, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. -pub(crate) fn fill_hash_table( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _end: usize, - _dtlm: DictTableLoadMethod, - _tfp: TableFillPurpose, -) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast` -pub(crate) fn compress_block_fast( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast_dictMatchState` -pub(crate) fn compress_block_fast_dict_match_state( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_fast_extDict` -pub(crate) fn compress_block_fast_ext_dict( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/opt/zstd/compress/fse.rs b/src/zstd/wip/work/opt/zstd/compress/fse.rs deleted file mode 100644 index 1b8f7cbf3173..000000000000 --- a/src/zstd/wip/work/opt/zstd/compress/fse.rs +++ /dev/null @@ -1,70 +0,0 @@ -//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. - -use crate::common::{MAX_SEQ, ML_FSE_LOG}; -use crate::util::SliceExt; - -/// `FSE_repeat` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum FseRepeat { - /// The table cannot be used again. - None, - /// It can, if it has a code for every symbol to encode. - Check, - /// It can. - Valid, -} - -/// `FSE_symbolCompressionTransform` -#[derive(Clone, Copy, Default)] -pub(crate) struct FseSymbolTransform { - pub(crate) delta_find_state: i32, - pub(crate) delta_nb_bits: u32, -} - -/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. -#[derive(Clone)] -pub(crate) struct FseCTable { - pub(crate) table_log: u32, - pub(crate) max_symbol_value: u32, - pub(crate) state_table: [u16; 1 << ML_FSE_LOG], - pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], -} - -impl FseCTable { - pub(crate) const fn new() -> Self { - Self { - table_log: 0, - max_symbol_value: 0, - state_table: [0; 1 << ML_FSE_LOG], - symbol_tt: [FseSymbolTransform { - delta_find_state: 0, - delta_nb_bits: 0, - }; MAX_SEQ as usize + 1], - } - } - - /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log - /// if the table has no code for it. - #[inline(always)] - pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { - let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); - transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 - } - - /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. - /// The table has a code for it. - #[inline(always)] - pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { - let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); - let min_nb_bits = transform.delta_nb_bits >> 16; - let threshold = (min_nb_bits + 1) << 16; - let table_size = 1u32 << self.table_log; - let delta_from_threshold = - threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); - // A linear interpolation, which is very approximate. - let normalized_delta_from_threshold = - (delta_from_threshold << accuracy_log) >> self.table_log; - let bit_multiplier = 1u32 << accuracy_log; - ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) - } -} diff --git a/src/zstd/wip/work/opt/zstd/compress/hist.rs b/src/zstd/wip/work/opt/zstd/compress/hist.rs deleted file mode 100644 index becf42516867..000000000000 --- a/src/zstd/wip/work/opt/zstd/compress/hist.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! `hist.c`: how many times each byte value occurs. - -use crate::error::Result; -use crate::util::SliceExt; - -/// `HIST_add`: `count` has room for every value in `src`. -pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { - for &byte in src { - *count.at_mut(usize::from(byte))? += 1; - } - Ok(()) -} - -/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. -/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. -pub(crate) fn count_simple( - count: &mut [u32], - max_symbol_value: &mut u32, - src: &[u8], -) -> Result { - let mut max_symbol = *max_symbol_value as usize; - count.sub_mut(..=max_symbol)?.fill(0); - if src.is_empty() { - *max_symbol_value = 0; - return Ok(0); - } - add(count, src)?; - while count.at(max_symbol)? == 0 { - max_symbol -= 1; - } - *max_symbol_value = max_symbol as u32; - Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) -} diff --git a/src/zstd/wip/work/opt/zstd/compress/huf.rs b/src/zstd/wip/work/opt/zstd/compress/huf.rs deleted file mode 100644 index 78373a89ebf0..000000000000 --- a/src/zstd/wip/work/opt/zstd/compress/huf.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! `huf_compress.c`: Huffman coding of literals. - -use crate::common::HUF_SYMBOLVALUE_MAX; -use crate::util::SliceExt; - -/// `HUF_repeat` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum HufRepeat { - /// The table cannot be used again. - None, - /// It can, if it has a code for every symbol to encode. - Check, - /// It can. - Valid, -} - -/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the -/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in -/// its highest bits. -#[derive(Clone)] -pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); - -impl HufCTable { - pub(crate) const fn new() -> Self { - Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) - } - - /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. - #[inline] - pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { - let [header, elements @ ..] = &self.0; - let max_symbol_value = (header >> 8) as u8; - if symbol_value > u32::from(max_symbol_value) { - return 0; - } - (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 - } -} diff --git a/src/zstd/wip/work/opt/zstd/compress/internal.rs b/src/zstd/wip/work/opt/zstd/compress/internal.rs deleted file mode 100644 index 51157ea6c1ce..000000000000 --- a/src/zstd/wip/work/opt/zstd/compress/internal.rs +++ /dev/null @@ -1,1212 +0,0 @@ -//! `zstd_compress_internal.h`: what the parts of the compressor share. - -use super::fse::{FseCTable, FseRepeat}; -use super::huf::{HufCTable, HufRepeat}; -use crate::common::{ - BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, - ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, -}; -use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; - -pub(crate) const K_SEARCH_STRENGTH: u32 = 8; -pub(crate) const HASH_READ_SIZE: usize = 8; -/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. -pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; - -pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; -pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; - -pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; -pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53 * 9 + 36 * 9 + 32 * 8 + 7) / 8; - -/// `ZSTD_strategy` -#[repr(u8)] -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] -pub(crate) enum Strategy { - Fast = 1, - DFast = 2, - Greedy = 3, - Lazy = 4, - Lazy2 = 5, - BtLazy2 = 6, - BtOpt = 7, - BtUltra = 8, - BtUltra2 = 9, -} - -impl Strategy { - pub(crate) const fn from_int(value: i32) -> Option { - Some(match value { - 1 => Self::Fast, - 2 => Self::DFast, - 3 => Self::Greedy, - 4 => Self::Lazy, - 5 => Self::Lazy2, - 6 => Self::BtLazy2, - 7 => Self::BtOpt, - 8 => Self::BtUltra, - 9 => Self::BtUltra2, - _ => return None, - }) - } -} - -/// `ZSTD_ParamSwitch_e` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum ParamSwitch { - Auto = 0, - Enable = 1, - Disable = 2, -} - -/// `ZSTD_compressionParameters` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) struct CompressionParameters { - pub(crate) window_log: u32, - pub(crate) chain_log: u32, - pub(crate) hash_log: u32, - pub(crate) search_log: u32, - pub(crate) min_match: u32, - pub(crate) target_length: u32, - /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. - pub(crate) strategy: Option, -} - -impl CompressionParameters { - /// The strategy of parameters that are ready to be applied. - #[inline(always)] - pub(crate) fn strat(&self) -> Strategy { - self.strategy.unwrap_or(Strategy::Fast) - } -} - -/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum DictMode { - NoDict, - ExtDict, - DictMatchState, -} - -/// `ZSTD_dictTableLoadMethod_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum DictTableLoadMethod { - Fast, - Full, -} - -/// `ZSTD_tableFillPurpose_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum TableFillPurpose { - ForCCtx, - ForCDict, -} - -// ── entropy tables ──────────────────────────────────────────────────────── - -/// `ZSTD_hufCTables_t` -#[derive(Clone)] -pub(crate) struct HufCTables { - pub(crate) c_table: HufCTable, - pub(crate) repeat_mode: HufRepeat, -} - -/// `ZSTD_fseCTables_t` -#[derive(Clone)] -pub(crate) struct FseCTables { - pub(crate) offcode_c_table: FseCTable, - pub(crate) matchlength_c_table: FseCTable, - pub(crate) litlength_c_table: FseCTable, - pub(crate) offcode_repeat_mode: FseRepeat, - pub(crate) matchlength_repeat_mode: FseRepeat, - pub(crate) litlength_repeat_mode: FseRepeat, -} - -/// `ZSTD_entropyCTables_t` -#[derive(Clone)] -pub(crate) struct EntropyCTables { - pub(crate) huf: HufCTables, - pub(crate) fse: FseCTables, -} - -/// `ZSTD_compressedBlockState_t` -#[derive(Clone)] -pub(crate) struct CompressedBlockState { - pub(crate) entropy: EntropyCTables, - pub(crate) rep: [u32; ZSTD_REP_NUM], -} - -impl CompressedBlockState { - /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. - pub(crate) fn new() -> Self { - Self { - entropy: EntropyCTables { - huf: HufCTables { - c_table: HufCTable::new(), - repeat_mode: HufRepeat::None, - }, - fse: FseCTables { - offcode_c_table: FseCTable::new(), - matchlength_c_table: FseCTable::new(), - litlength_c_table: FseCTable::new(), - offcode_repeat_mode: FseRepeat::None, - matchlength_repeat_mode: FseRepeat::None, - litlength_repeat_mode: FseRepeat::None, - }, - }, - rep: REP_START_VALUE, - } - } - - /// `ZSTD_reset_compressedBlockState` - pub(crate) fn reset(&mut self) { - self.rep = REP_START_VALUE; - self.entropy.huf.repeat_mode = HufRepeat::None; - self.entropy.fse.offcode_repeat_mode = FseRepeat::None; - self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; - self.entropy.fse.litlength_repeat_mode = FseRepeat::None; - } -} - -/// `ZSTD_hufCTablesMetadata_t` -pub(crate) struct HufCTablesMetadata { - pub(crate) h_type: SymbolEncodingType, - pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], - pub(crate) huf_des_size: usize, -} - -/// `ZSTD_fseCTablesMetadata_t` -pub(crate) struct FseCTablesMetadata { - pub(crate) ll_type: SymbolEncodingType, - pub(crate) of_type: SymbolEncodingType, - pub(crate) ml_type: SymbolEncodingType, - pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], - pub(crate) fse_tables_size: usize, - /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. - pub(crate) last_count_size: usize, -} - -/// `ZSTD_entropyCTablesMetadata_t` -pub(crate) struct EntropyCTablesMetadata { - pub(crate) huf_metadata: HufCTablesMetadata, - pub(crate) fse_metadata: FseCTablesMetadata, -} - -impl EntropyCTablesMetadata { - pub(crate) const fn new() -> Self { - Self { - huf_metadata: HufCTablesMetadata { - h_type: SymbolEncodingType::Basic, - huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], - huf_des_size: 0, - }, - fse_metadata: FseCTablesMetadata { - ll_type: SymbolEncodingType::Basic, - of_type: SymbolEncodingType::Basic, - ml_type: SymbolEncodingType::Basic, - fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], - fse_tables_size: 0, - last_count_size: 0, - }, - } - } -} - -// ── sequences ───────────────────────────────────────────────────────────── - -/// `SeqDef` -#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] -pub(crate) struct SeqDef { - /// A repeat code (1 to 3), or an offset plus 3. - pub(crate) off_base: u32, - pub(crate) lit_length: u16, - /// The match length minus [`MINMATCH`]. - pub(crate) ml_base: u16, -} - -/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum LongLengthType { - None, - LiteralLength, - MatchLength, -} - -/// `SeqStore_t`: the sequences and the literals of a block. -pub(crate) struct SeqStore { - /// Never grows beyond the capacity it is given by [`Self::reserve`]. - pub(crate) sequences: Vec, - /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. - pub(crate) lit_buffer: Vec, - /// How much of `lit_buffer` is literals. - pub(crate) lit_len: usize, - pub(crate) ll_code: Vec, - pub(crate) ml_code: Vec, - pub(crate) of_code: Vec, - pub(crate) max_nb_seq: usize, - pub(crate) max_nb_lit: usize, - pub(crate) long_length_type: LongLengthType, - /// Which sequence has the long length. - pub(crate) long_length_pos: u32, -} - -/// `ZSTD_SequenceLength` -#[derive(Clone, Copy)] -pub(crate) struct SequenceLength { - pub(crate) lit_length: u32, - pub(crate) match_length: u32, -} - -impl SeqStore { - pub(crate) const fn new() -> Self { - Self { - sequences: Vec::new(), - lit_buffer: Vec::new(), - lit_len: 0, - ll_code: Vec::new(), - ml_code: Vec::new(), - of_code: Vec::new(), - max_nb_seq: 0, - max_nb_lit: 0, - long_length_type: LongLengthType::None, - long_length_pos: 0, - } - } - - /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. - pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { - fn grow(vec: &mut Vec, len: usize) -> Result<()> { - vec.clear(); - vec.try_reserve_exact(len) - .map_err(|_| ErrorCode::MemoryAllocation)?; - vec.resize(len, T::default()); - Ok(()) - } - grow(&mut self.sequences, max_nb_seq)?; - grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; - grow(&mut self.ll_code, max_nb_seq)?; - grow(&mut self.ml_code, max_nb_seq)?; - grow(&mut self.of_code, max_nb_seq)?; - self.max_nb_seq = max_nb_seq; - self.max_nb_lit = max_nb_lit; - self.reset(); - Ok(()) - } - - /// `ZSTD_resetSeqStore` - #[inline] - pub(crate) fn reset(&mut self) { - self.lit_len = 0; - self.sequences.clear(); - self.long_length_type = LongLengthType::None; - } - - #[inline] - pub(crate) fn literals(&self) -> &[u8] { - self.lit_buffer.get(..self.lit_len).unwrap_or_default() - } - - /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. - #[inline(always)] - pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { - debug_assert!(self.sequences.len() < self.max_nb_seq); - if lit_length > 0xFFFF { - self.long_length_type = LongLengthType::LiteralLength; - self.long_length_pos = self.sequences.len() as u32; - } - let ml_base = match_length.wrapping_sub(MINMATCH as usize); - if ml_base > 0xFFFF { - self.long_length_type = LongLengthType::MatchLength; - self.long_length_pos = self.sequences.len() as u32; - } - self.sequences.push(SeqDef { - off_base, - lit_length: lit_length as u16, - ml_base: ml_base as u16, - }); - } - - /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. - #[inline(always)] - pub(crate) fn store_seq( - &mut self, - lit_length: usize, - buf: &[u8], - literals: usize, - off_base: u32, - match_length: usize, - ) -> Result<()> { - match ( - buf.array_at::<16>(literals), - self.lit_buffer.array_at_mut::<16>(self.lit_len), - ) { - (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, - _ => self - .lit_buffer - .write_at(self.lit_len, buf.span(literals, lit_length)?)?, - } - self.lit_len += lit_length; - self.store_seq_only(lit_length, off_base, match_length); - Ok(()) - } - - /// Appends the literals that follow the last sequence of a block. - #[inline] - pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { - self.lit_buffer.write_at(self.lit_len, literals)?; - self.lit_len += literals.len(); - Ok(()) - } -} - -pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; -pub(crate) const REPCODE2_TO_OFFBASE: u32 = 2; -pub(crate) const REPCODE3_TO_OFFBASE: u32 = 3; - -/// `OFFSET_TO_OFFBASE`: `offset` is not 0. -#[inline(always)] -pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { - offset.wrapping_add(ZSTD_REP_NUM as u32) -} - -/// `OFFBASE_IS_OFFSET` -#[inline(always)] -pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { - off_base > ZSTD_REP_NUM as u32 -} - -/// `OFFBASE_IS_REPCODE` -#[inline(always)] -pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { - 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 -} - -/// `OFFBASE_TO_OFFSET` -#[inline(always)] -pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { - off_base.wrapping_sub(ZSTD_REP_NUM as u32) -} - -/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. -#[inline(always)] -pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { - let [rep0, rep1, rep2] = *rep; - if offbase_is_offset(off_base) { - *rep = [offbase_to_offset(off_base), rep0, rep1]; - } else { - match off_base - 1 + u32::from(ll0) { - 0 => {} - 1 => *rep = [rep1, rep0, rep2], - 2 => *rep = [rep2, rep0, rep1], - _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], - } - } -} - -/// `ZSTD_newRep` -#[inline(always)] -pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { - let mut new_reps = *rep; - update_rep(&mut new_reps, off_base, ll0); - new_reps -} - -/// `ZSTD_LLcode` -#[inline(always)] -pub(crate) fn ll_code(lit_length: u32) -> u32 { - #[rustfmt::skip] - const LL_CODE: [u8; 64] = [ - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, - ]; - const LL_DELTA_CODE: u32 = 19; - match LL_CODE.get(lit_length as usize) { - Some(&code) => u32::from(code), - None => highbit32(lit_length) + LL_DELTA_CODE, - } -} - -/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. -#[inline(always)] -pub(crate) fn ml_code(ml_base: u32) -> u32 { - #[rustfmt::skip] - const ML_CODE: [u8; 128] = [ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - ]; - const ML_DELTA_CODE: u32 = 36; - match ML_CODE.get(ml_base as usize) { - Some(&code) => u32::from(code), - None => highbit32(ml_base) + ML_DELTA_CODE, - } -} - -/// `ZSTD_noCompressBlock` -pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { - let c_block_header24 = - u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); - if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { - return Err(ErrorCode::DstSizeTooSmall); - } - write_le24(dst, 0, c_block_header24)?; - dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; - Ok(ZSTD_BLOCKHEADERSIZE + src.len()) -} - -/// `ZSTD_rleCompressBlock` -pub(crate) fn rle_compress_block( - dst: &mut [u8], - byte: u8, - src_size: usize, - last_block: bool, -) -> Result { - let c_block_header = - u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); - if dst.len() < 4 { - return Err(ErrorCode::DstSizeTooSmall); - } - write_le24(dst, 0, c_block_header)?; - dst.put(3, byte)?; - Ok(4) -} - -/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. -#[inline] -pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { - let minlog = if strat >= Strategy::BtUltra { - strat as u32 - 1 - } else { - 6 - }; - (src_size >> minlog) + 2 -} - -// ── matching ────────────────────────────────────────────────────────────── - -/// How many bytes `a` and `b` have in common at their start. -#[inline(always)] -pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { - let (a_words, _) = a.as_chunks::<8>(); - let (b_words, _) = b.as_chunks::<8>(); - let mut len = 0usize; - for (a_word, b_word) in a_words.iter().zip(b_words) { - let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); - if diff != 0 { - return len + (diff.trailing_zeros() >> 3) as usize; - } - len += 8; - } - let a_tail = a.get(len..).unwrap_or_default(); - let b_tail = b.get(len..).unwrap_or_default(); - len + a_tail - .iter() - .zip(b_tail) - .take_while(|(a_byte, b_byte)| a_byte == b_byte) - .count() -} - -/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. -#[inline(always)] -pub(crate) fn count( - buf: &[u8], - p_in: usize, - match_buf: &[u8], - p_match: usize, - p_in_limit: usize, -) -> Result { - Ok(common_prefix( - buf.sub(p_in..p_in_limit)?, - match_buf.sub(p_match..)?, - )) -} - -/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at -/// `i_start` in `buf`. -#[inline(always)] -pub(crate) fn count_2segments( - buf: &[u8], - ip: usize, - match_buf: &[u8], - match_pos: usize, - i_end: usize, - m_end: usize, - i_start: usize, -) -> Result { - let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); - if match_pos + match_length != m_end { - return Ok(match_length); - } - Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) -} - -const PRIME_3_BYTES: u32 = 506_832_829; -const PRIME_4_BYTES: u32 = 2_654_435_761; -const PRIME_5_BYTES: u64 = 889_523_592_379; -const PRIME_6_BYTES: u64 = 227_718_039_650_203; -const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; -pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; - -/// `ZSTD_hash3` -#[inline(always)] -const fn hash3(u: u32, h: u32, s: u32) -> u32 { - (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash4(u: u32, h: u32, s: u32) -> u32 { - ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash5(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash6(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash7(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -/// `ZSTD_hash8` -#[inline(always)] -pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { - (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} - -/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. -#[inline(always)] -pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { - Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) -} - -/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, -/// else 8. -#[inline(always)] -pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { - hash_ptr_salted(buf, pos, h_bits, mls, 0) -} - -/// `ZSTD_hashPtrSalted` -#[inline(always)] -pub(crate) fn hash_ptr_salted( - buf: &[u8], - pos: usize, - h_bits: u32, - mls: u32, - hash_salt: u64, -) -> Result { - Ok(match mls { - 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), - 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), - 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), - 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), - _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, - }) -} - -// ── the window ──────────────────────────────────────────────────────────── - -/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are -/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the -/// others, and an address is turned into a position in the buffer only to read. -pub(crate) type Addr = usize; - -/// The buffers that a window can have a segment in. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum BufId { - /// Where an empty window points. - None = 0, - /// The context's own buffer of input. - InBuff = 1, - /// What the caller gave to the call in progress. - User = 2, - /// The content of the dictionary, or the prefix. - Dict = 3, -} - -impl BufId { - /// The address of the first byte of the buffer. - #[inline(always)] - pub(crate) const fn start(self) -> Addr { - (self as usize + 1) << 44 - } -} - -/// The buffers themselves, for as long as a call lasts. -#[derive(Clone, Copy)] -pub(crate) struct Buffers<'a> { - pub(crate) in_buff: &'a [u8], - pub(crate) user: &'a [u8], - pub(crate) dict: &'a [u8], -} - -impl<'a> Buffers<'a> { - #[inline] - pub(crate) fn get(&self, id: BufId) -> &'a [u8] { - match id { - BufId::None => &[], - BufId::InBuff => self.in_buff, - BufId::User => self.user, - BufId::Dict => self.dict, - } - } -} - -/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. -pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; - -/// The index past which indices are brought down. -pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; -/// The most input that can be indexed at once. -pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; - -/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the -/// index is less than `dict_limit`. -#[derive(Clone, Copy, Debug)] -pub(crate) struct Window { - /// Where input has to continue to be contiguous. - pub(crate) next_src: Addr, - pub(crate) base: Addr, - pub(crate) dict_base: Addr, - /// The buffer of the segment that starts at `dict_limit`. - pub(crate) base_buf: BufId, - /// The buffer of the segment from `low_limit` to `dict_limit`. - pub(crate) dict_buf: BufId, - pub(crate) dict_limit: u32, - /// Below this, there is nothing valid. - pub(crate) low_limit: u32, - pub(crate) nb_overflow_corrections: u32, -} - -impl Window { - /// `ZSTD_window_init` - pub(crate) const fn new() -> Self { - let base = BufId::None.start(); - Self { - next_src: base + ZSTD_WINDOW_START_INDEX as usize, - base, - dict_base: base, - base_buf: BufId::None, - dict_buf: BufId::None, - dict_limit: ZSTD_WINDOW_START_INDEX, - low_limit: ZSTD_WINDOW_START_INDEX, - nb_overflow_corrections: 0, - } - } - - /// The index of `addr`, which is in the current segment or at its end. - #[inline(always)] - pub(crate) const fn index_of(&self, addr: Addr) -> u32 { - addr.wrapping_sub(self.base) as u32 - } - - /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. - pub(crate) fn clear(&mut self) { - let end = self.index_of(self.next_src); - self.low_limit = end; - self.dict_limit = end; - } - - /// `ZSTD_window_isEmpty` - pub(crate) fn is_empty(&self) -> bool { - self.dict_limit == ZSTD_WINDOW_START_INDEX - && self.low_limit == ZSTD_WINDOW_START_INDEX - && self.next_src.wrapping_sub(self.base) == ZSTD_WINDOW_START_INDEX as usize - } - - /// `ZSTD_window_hasExtDict` - #[inline] - pub(crate) fn has_ext_dict(&self) -> bool { - self.low_limit < self.dict_limit - } - - /// `ZSTD_window_needOverflowCorrection` - #[inline] - pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { - self.index_of(src_end) > ZSTD_CURRENT_MAX - } - - /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns - /// what to subtract from every index that is stored. - pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { - let cycle_size = 1u32 << cycle_log; - let cycle_mask = cycle_size - 1; - let curr = self.index_of(src); - let current_cycle = curr & cycle_mask; - // Indices must not fall below the start index. - let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { - cycle_size.max(ZSTD_WINDOW_START_INDEX) - } else { - 0 - }; - let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); - let correction = curr.wrapping_sub(new_current); - - self.base = self.base.wrapping_add(correction as usize); - self.dict_base = self.dict_base.wrapping_add(correction as usize); - self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { - ZSTD_WINDOW_START_INDEX - } else { - self.low_limit - correction - }; - self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { - ZSTD_WINDOW_START_INDEX - } else { - self.dict_limit - correction - }; - self.nb_overflow_corrections += 1; - correction - } - - /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the - /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. - pub(crate) fn enforce_max_dist( - &mut self, - block_end: Addr, - max_dist: u32, - loaded_dict_end: u32, - ) -> bool { - let block_end_idx = self.index_of(block_end); - if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { - let new_low_limit = block_end_idx - max_dist; - if self.low_limit < new_low_limit { - self.low_limit = new_low_limit; - } - if self.dict_limit < self.low_limit { - self.dict_limit = self.low_limit; - } - return true; - } - false - } - - /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not - /// right before the current segment, and is to be forgotten. - pub(crate) fn dict_is_invalid( - &self, - block_end: Addr, - max_dist: u32, - loaded_dict_end: u32, - ) -> bool { - let block_end_idx = self.index_of(block_end); - block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit - } - - /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not - /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. - pub(crate) fn update( - &mut self, - src: Addr, - src_buf: BufId, - src_size: usize, - force_non_contiguous: bool, - ) -> bool { - let mut contiguous = true; - if src_size == 0 { - return contiguous; - } - if src != self.next_src || force_non_contiguous { - let distance_from_base = self.next_src.wrapping_sub(self.base); - self.low_limit = self.dict_limit; - self.dict_limit = distance_from_base as u32; - self.dict_base = self.base; - self.dict_buf = self.base_buf; - self.base = src.wrapping_sub(distance_from_base); - self.base_buf = src_buf; - if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { - // Too small to be of use. - self.low_limit = self.dict_limit; - } - contiguous = false; - } - self.next_src = src + src_size; - // Where the input overlaps the older segment, it has overwritten it. - if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) - & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) - { - let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); - self.low_limit = if high_input_idx > self.dict_limit as usize { - self.dict_limit - } else { - high_input_idx as u32 - }; - } - contiguous - } -} - -/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes -/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. -#[inline(always)] -pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { - prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 -} - -pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; -pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; - -/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. -#[inline(always)] -pub(crate) fn write_tagged_index( - hash_table: &mut [u32], - hash_and_tag: usize, - index: u32, -) -> Result<()> { - let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; - hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) -} - -/// `ZSTD_comparePackedTags` -#[inline(always)] -pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { - (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) - == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) -} - -// ── the state of the match finders ──────────────────────────────────────── - -/// `ZSTD_match_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct Match { - /// As [`SeqDef::off_base`]. - pub(crate) off: u32, - pub(crate) len: u32, -} - -/// `rawSeq`: a sequence found by long distance matching. -#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] -pub(crate) struct RawSeq { - pub(crate) offset: u32, - pub(crate) lit_length: u32, - pub(crate) match_length: u32, -} - -/// `RawSeqStore_t` -#[derive(Clone, Default)] -pub(crate) struct RawSeqStore { - /// `capacity` of them, of which `size` are in use. - pub(crate) seq: Vec, - /// The sequence that is being read. - pub(crate) pos: usize, - /// How many of its bytes, literals first, are read. - pub(crate) pos_in_sequence: usize, - pub(crate) size: usize, - pub(crate) capacity: usize, -} - -/// `ZSTD_optimal_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct Optimal { - /// The cost of getting to this position. - pub(crate) price: i32, - pub(crate) off: u32, - pub(crate) mlen: u32, - pub(crate) litlen: u32, - pub(crate) rep: [u32; ZSTD_REP_NUM], -} - -/// `ZSTD_OptPrice_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum OptPrice { - Dynamic, - Predef, -} - -/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. -pub(crate) struct OptState { - pub(crate) lit_freq: Vec, - pub(crate) lit_length_freq: Vec, - pub(crate) match_length_freq: Vec, - pub(crate) off_code_freq: Vec, - pub(crate) match_table: Vec, - pub(crate) price_table: Vec, - pub(crate) lit_sum: u32, - pub(crate) lit_length_sum: u32, - pub(crate) match_length_sum: u32, - pub(crate) off_code_sum: u32, - pub(crate) lit_sum_base_price: u32, - pub(crate) lit_length_sum_base_price: u32, - pub(crate) match_length_sum_base_price: u32, - pub(crate) off_code_sum_base_price: u32, - pub(crate) price_type: OptPrice, - pub(crate) literal_compression_mode: ParamSwitch, -} - -impl OptState { - pub(crate) const fn new() -> Self { - Self { - lit_freq: Vec::new(), - lit_length_freq: Vec::new(), - match_length_freq: Vec::new(), - off_code_freq: Vec::new(), - match_table: Vec::new(), - price_table: Vec::new(), - lit_sum: 0, - lit_length_sum: 0, - match_length_sum: 0, - off_code_sum: 0, - lit_sum_base_price: 0, - lit_length_sum_base_price: 0, - match_length_sum_base_price: 0, - off_code_sum_base_price: 0, - price_type: OptPrice::Dynamic, - literal_compression_mode: ParamSwitch::Auto, - } - } -} - -pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; - -/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures -/// own, are in [`BlockSrc`]. -pub(crate) struct MatchState { - pub(crate) window: Window, - /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. - pub(crate) loaded_dict_end: u32, - /// The index from which the tables are still to be filled. - pub(crate) next_to_update: u32, - /// 0 if there is no table of 3-byte hashes. - pub(crate) hash_log3: u32, - pub(crate) row_hash_log: u32, - /// For the row-based match finder: a byte of hash for each entry of `hash_table`. - pub(crate) tag_table: Vec, - pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], - /// Makes what a previous frame left in `tag_table` meaningless. - pub(crate) hash_salt: u64, - pub(crate) hash_salt_entropy: u32, - pub(crate) hash_table: Vec, - pub(crate) hash_table3: Vec, - pub(crate) chain_table: Vec, - pub(crate) force_non_contiguous: bool, - pub(crate) opt: OptState, - /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. - pub(crate) has_dict_match_state: bool, - pub(crate) c_params: CompressionParameters, - pub(crate) lazy_skipping: bool, -} - -impl MatchState { - pub(crate) const fn new() -> Self { - Self { - window: Window::new(), - loaded_dict_end: 0, - next_to_update: 0, - hash_log3: 0, - row_hash_log: 0, - tag_table: Vec::new(), - hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], - hash_salt: 0, - hash_salt_entropy: 0, - hash_table: Vec::new(), - hash_table3: Vec::new(), - chain_table: Vec::new(), - force_non_contiguous: false, - opt: OptState::new(), - has_dict_match_state: false, - c_params: CompressionParameters { - window_log: 0, - chain_log: 0, - hash_log: 0, - search_log: 0, - min_match: 0, - target_length: 0, - strategy: None, - }, - lazy_skipping: false, - } - } - - /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. - pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { - if self.window.has_ext_dict() { - return DictMode::ExtDict; - } - if dms.is_some() { - DictMode::DictMatchState - } else { - DictMode::NoDict - } - } - - /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. - #[inline(always)] - pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { - let max_distance = 1u32 << window_log; - let lowest_valid = self.window.low_limit; - let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { - curr - max_distance - } else { - lowest_valid - }; - // A dictionary is valid as a whole until it is invalidated as a whole. - if self.loaded_dict_end != 0 { - lowest_valid - } else { - within_window - } - } - - /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. - #[inline(always)] - pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { - let max_distance = 1u32 << window_log; - let lowest_valid = self.window.dict_limit; - let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { - curr - max_distance - } else { - lowest_valid - }; - if self.loaded_dict_end != 0 { - lowest_valid - } else { - within_window - } - } -} - -/// The match state of a dictionary, and where its indices lead. -#[derive(Clone, Copy)] -pub(crate) struct DictMatchState<'a> { - pub(crate) ms: &'a MatchState, - /// The buffer its window is in, which is one segment. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, - /// `window.nextSrc`, as a position in `buf`. - pub(crate) end: usize, -} - -/// The current segment of a window, for the functions that index a dictionary. -#[derive(Clone, Copy)] -pub(crate) struct Prefix<'a> { - /// The buffer that the segment is in. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, -} - -impl Prefix<'_> { - /// The index of the position `pos` of `buf`. - #[inline(always)] - pub(crate) const fn index_of(&self, pos: usize) -> u32 { - pos.wrapping_sub(self.base) as u32 - } - - /// The position in `buf` of `index`. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } -} - -/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. -#[derive(Clone, Copy)] -pub(crate) struct BlockSrc<'a> { - /// The buffer that the current segment of the window is in, and so the block. - pub(crate) buf: &'a [u8], - /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. - pub(crate) base: usize, - /// The buffer that the older segment of the window is in. - pub(crate) dict_buf: &'a [u8], - /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. - pub(crate) dict_base: usize, - /// Where the block starts in `buf`. - pub(crate) start: usize, - /// Where it ends. - pub(crate) end: usize, - pub(crate) dms: Option>, - /// `opt.symbolCosts`: the tables of the previous block. - pub(crate) symbol_costs: &'a EntropyCTables, - /// `ldmSeqStore`: matches that the optimal parser is to consider. - pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, -} - -impl<'a> BlockSrc<'a> { - #[inline(always)] - pub(crate) const fn prefix(&self) -> Prefix<'a> { - Prefix { - buf: self.buf, - base: self.base, - } - } - - /// The index of the position `pos` of `buf`. - #[inline(always)] - pub(crate) const fn index_of(&self, pos: usize) -> u32 { - pos.wrapping_sub(self.base) as u32 - } - - /// The position in `buf` of `index`, which is in the current segment. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } - - /// The position in `dict_buf` of `index`, which is in the older segment. - #[inline(always)] - pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { - self.dict_base.wrapping_add(index as usize) - } -} - -impl DictMatchState<'_> { - /// The position in `buf` of `index`. - #[inline(always)] - pub(crate) const fn pos_of(&self, index: u32) -> usize { - self.base.wrapping_add(index as usize) - } -} - -// ── long distance matching ──────────────────────────────────────────────── - -/// `ldmEntry_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct LdmEntry { - pub(crate) offset: u32, - pub(crate) checksum: u32, -} - -/// `ldmParams_t` -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) struct LdmParams { - pub(crate) enable_ldm: ParamSwitch, - pub(crate) hash_log: u32, - pub(crate) bucket_size_log: u32, - pub(crate) min_match_length: u32, - pub(crate) hash_rate_log: u32, - pub(crate) window_log: u32, -} - -/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. -pub(crate) struct LdmState { - pub(crate) window: Window, - pub(crate) hash_table: Vec, - pub(crate) loaded_dict_end: u32, - /// The next entry to replace in each bucket. - pub(crate) bucket_offsets: Vec, -} - -impl LdmState { - pub(crate) const fn new() -> Self { - Self { - window: Window::new(), - hash_table: Vec::new(), - loaded_dict_end: 0, - bucket_offsets: Vec::new(), - } - } -} diff --git a/src/zstd/wip/work/opt/zstd/compress/lazy.rs b/src/zstd/wip/work/opt/zstd/compress/lazy.rs deleted file mode 100644 index 9f653fd684f3..000000000000 --- a/src/zstd/wip/work/opt/zstd/compress/lazy.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. - -use super::internal::{BlockSrc, DictMode, MatchState, Prefix, SeqStore, Strategy}; -use crate::common::ZSTD_REP_NUM; -use crate::error::{ErrorCode, Result}; - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - _ms: &mut MatchState, - _prefix: Prefix<'_>, - _ip: usize, -) -> Result { - Err(ErrorCode::Generic) -} - -/// `ZSTD_row_update`: the same, in the rows of the row-based match finder. -pub(crate) fn row_update(_ms: &mut MatchState, _prefix: Prefix<'_>, _ip: usize) -> Result<()> { - Err(ErrorCode::Generic) -} - -/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: -/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. -pub(crate) fn compress_block_lazy( - _ms: &mut MatchState, - _seq_store: &mut SeqStore, - _rep: &mut [u32; ZSTD_REP_NUM], - _src: &BlockSrc<'_>, - _strategy: Strategy, - _use_row: bool, - _dict_mode: DictMode, -) -> Result { - Err(ErrorCode::Generic) -} diff --git a/src/zstd/wip/work/opt/zstd/compress/ldm.rs b/src/zstd/wip/work/opt/zstd/compress/ldm.rs deleted file mode 100644 index 2177fd945e10..000000000000 --- a/src/zstd/wip/work/opt/zstd/compress/ldm.rs +++ /dev/null @@ -1 +0,0 @@ -//! `zstd_ldm.c`: long distance matching. diff --git a/src/zstd/wip/work/opt/zstd/compress/mod.rs b/src/zstd/wip/work/opt/zstd/compress/mod.rs deleted file mode 100644 index 13ddecdf82e4..000000000000 --- a/src/zstd/wip/work/opt/zstd/compress/mod.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! `lib/compress` - -pub(crate) mod double_fast; -pub(crate) mod fast; -pub(crate) mod fse; -pub(crate) mod hist; -pub(crate) mod huf; -pub(crate) mod internal; -pub(crate) mod lazy; -pub(crate) mod ldm; -pub(crate) mod opt; -pub(crate) mod presplit; - -use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; - -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; - -/// `ZSTD_rowMatchFinderSupported` -pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { - strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 -} - -/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to -/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies -/// that support it look at. Returns how many bytes at the end of the block are in no sequence. -pub(crate) fn compress_block( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row_match_finder: bool, - dict_mode: DictMode, -) -> Result { - match strategy { - Strategy::Fast => match dict_mode { - DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), - DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), - DictMode::DictMatchState => { - fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) - } - }, - Strategy::DFast => match dict_mode { - DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), - DictMode::ExtDict => { - double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) - } - }, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { - lazy::compress_block_lazy( - ms, - seq_store, - rep, - src, - strategy, - use_row_match_finder && row_match_finder_supported(strategy), - dict_mode, - ) - } - Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { - opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) - } - } -} diff --git a/src/zstd/wip/work/opt/zstd/compress/opt.rs b/src/zstd/wip/work/opt/zstd/compress/opt.rs deleted file mode 100644 index a7a6022a3a45..000000000000 --- a/src/zstd/wip/work/opt/zstd/compress/opt.rs +++ /dev/null @@ -1,1721 +0,0 @@ -//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. - -use core::mem; - -use super::hist; -use super::huf::HufRepeat; -use super::internal::{ - BlockSrc, DictMode, EntropyCTables, Match, MatchState, OptPrice, OptState, Optimal, - ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, ZSTD_OPT_NUM, count, - count_2segments, hash_ptr, hash3_ptr, index_overlap_check, ll_code, ml_code, new_rep, - offset_to_offbase, -}; -use crate::common::{ - LL_BITS, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, -}; -use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32}; - -/// How much a literal adds to its frequency, so that frequencies adapt fast to new statistics. -const ZSTD_LITFREQ_ADD: u32 = 2; -const ZSTD_MAX_PRICE: i32 = 1 << 30; -/// A first block of at most this size is priced by fixed rules, not by statistics. -const ZSTD_PREDEF_THRESHOLD: usize = 8; - -const MAX_LIT: u32 = 255; - -// ── prices ──────────────────────────────────────────────────────────────── - -/// Prices are in units of `1 / BITCOST_MULTIPLIER` of a bit. -const BITCOST_ACCURACY: u32 = 8; -const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY; - -/// `WEIGHT`: approximate for `btopt`, accurate for `btultra`. -#[inline(always)] -const fn weight(stat: u32, opt_level: i32) -> u32 { - if opt_level != 0 { - frac_weight(stat) - } else { - bit_weight(stat) - } -} - -/// `ZSTD_bitWeight`: the cost of a statistic, in whole bits. -#[inline(always)] -const fn bit_weight(stat: u32) -> u32 { - highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER -} - -/// `ZSTD_fracWeight`: the cost of a statistic, with a fraction of a bit that is interpolated linearly. -#[inline(always)] -const fn frac_weight(raw_stat: u32) -> u32 { - let stat = raw_stat.wrapping_add(1); - let hb = highbit32(stat); - let b_weight = hb * BITCOST_MULTIPLIER; - // From 1 to 2, in fixed point. - let f_weight = (stat << BITCOST_ACCURACY) >> hb; - b_weight.wrapping_add(f_weight) -} - -/// `ZSTD_compressedLiterals` -#[inline(always)] -fn compressed_literals(opt_ptr: &OptState) -> bool { - opt_ptr.literal_compression_mode != ParamSwitch::Disable -} - -/// `ZSTD_setBasePrices` -#[inline(always)] -fn set_base_prices(opt_ptr: &mut OptState, opt_level: i32) { - if compressed_literals(opt_ptr) { - opt_ptr.lit_sum_base_price = weight(opt_ptr.lit_sum, opt_level); - } - opt_ptr.lit_length_sum_base_price = weight(opt_ptr.lit_length_sum, opt_level); - opt_ptr.match_length_sum_base_price = weight(opt_ptr.match_length_sum, opt_level); - opt_ptr.off_code_sum_base_price = weight(opt_ptr.off_code_sum, opt_level); -} - -/// `sum_u32` -fn sum_u32(table: &[u32]) -> u32 { - table.iter().fold(0, |total, &elt| total.wrapping_add(elt)) -} - -/// `base_directive_e`: the least a scaled down statistic can be. -#[derive(Clone, Copy, PartialEq, Eq)] -enum BaseDirective { - /// 0, if it is 0 already. - Base0Possible, - Base1Guaranteed, -} - -/// `ZSTD_downscaleStats`: returns the new sum of the elements. -fn downscale_stats( - table: &mut [u32], - last_elt_index: u32, - shift: u32, - base1: BaseDirective, -) -> Result { - let mut sum = 0u32; - for elt in table.sub_mut(..=last_elt_index as usize)? { - let base = u32::from(base1 == BaseDirective::Base1Guaranteed || *elt > 0); - let new_stat = base.wrapping_add(*elt >> (shift & 31)); - sum = sum.wrapping_add(new_stat); - *elt = new_stat; - } - Ok(sum) -} - -/// `ZSTD_scaleStats`: scales all the elements down if their sum is too large. Returns the sum that results. -fn scale_stats(table: &mut [u32], last_elt_index: u32, log_target: u32) -> Result { - let prevsum = sum_u32(table.sub(..=last_elt_index as usize)?); - let factor = prevsum >> log_target; - if factor <= 1 { - return Ok(prevsum); - } - downscale_stats( - table, - last_elt_index, - highbit32(factor), - BaseDirective::Base1Guaranteed, - ) -} - -/// Frequencies, out of about `1 << scale_log`, that make each symbol cost the bits `bit_cost` tells. Returns their sum. -fn freqs_from_bit_costs( - freqs: &mut [u32], - last_elt_index: u32, - scale_log: u32, - bit_cost: impl Fn(u32) -> u32, -) -> Result { - let mut sum = 0u32; - for (symbol, freq) in (0u32..).zip(freqs.sub_mut(..=last_elt_index as usize)?) { - let bit_cost = bit_cost(symbol); - // At least 1, to have a cost. - *freq = if bit_cost != 0 { - 1 << (scale_log.wrapping_sub(bit_cost) & 31) - } else { - 1 - }; - sum = sum.wrapping_add(*freq); - } - Ok(sum) -} - -/// `ZSTD_rescaleFreqs`: for the first block, which no statistics tell, starts them from the tables of the dictionary -/// if there is one, else from the literals of `src` and from fixed values for the sequences. For the next ones, scales -/// down what the blocks before have accumulated. -fn rescale_freqs( - opt_ptr: &mut OptState, - symbol_costs: &EntropyCTables, - src: &[u8], - opt_level: i32, -) -> Result<()> { - #[rustfmt::skip] - const BASE_LL_FREQS: [u32; MAX_LL as usize + 1] = [ - 4, 2, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, - ]; - #[rustfmt::skip] - const BASE_OFC_FREQS: [u32; MAX_OFF as usize + 1] = [ - 6, 2, 1, 1, 2, 3, 4, 4, - 4, 3, 2, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - ]; - - let compressed_literals = compressed_literals(opt_ptr); - opt_ptr.price_type = OptPrice::Dynamic; - - if opt_ptr.lit_length_sum == 0 { - if src.len() <= ZSTD_PREDEF_THRESHOLD { - opt_ptr.price_type = OptPrice::Predef; - } - - if symbol_costs.huf.repeat_mode == HufRepeat::Valid { - // A Huffman table that has all the values is presumed to come from a dictionary. - opt_ptr.price_type = OptPrice::Dynamic; - - if compressed_literals { - let huf = &symbol_costs.huf.c_table; - opt_ptr.lit_sum = - freqs_from_bit_costs(&mut opt_ptr.lit_freq, MAX_LIT, 11, |lit| { - huf.get_nb_bits(lit) - })?; - } - let fse = &symbol_costs.fse; - opt_ptr.lit_length_sum = - freqs_from_bit_costs(&mut opt_ptr.lit_length_freq, MAX_LL, 10, |ll| { - fse.litlength_c_table.get_max_nb_bits(ll) - })?; - opt_ptr.match_length_sum = - freqs_from_bit_costs(&mut opt_ptr.match_length_freq, MAX_ML, 10, |ml| { - fse.matchlength_c_table.get_max_nb_bits(ml) - })?; - opt_ptr.off_code_sum = - freqs_from_bit_costs(&mut opt_ptr.off_code_freq, MAX_OFF, 10, |of| { - fse.offcode_c_table.get_max_nb_bits(of) - })?; - } else { - if compressed_literals { - let mut lit = MAX_LIT; - hist::count_simple(&mut opt_ptr.lit_freq, &mut lit, src)?; - opt_ptr.lit_sum = downscale_stats( - &mut opt_ptr.lit_freq, - MAX_LIT, - 8, - BaseDirective::Base0Possible, - )?; - } - - opt_ptr.lit_length_freq.write_at(0, &BASE_LL_FREQS)?; - opt_ptr.lit_length_sum = sum_u32(&BASE_LL_FREQS); - - opt_ptr - .match_length_freq - .sub_mut(..=MAX_ML as usize)? - .fill(1); - opt_ptr.match_length_sum = MAX_ML + 1; - - opt_ptr.off_code_freq.write_at(0, &BASE_OFC_FREQS)?; - opt_ptr.off_code_sum = sum_u32(&BASE_OFC_FREQS); - } - } else { - if compressed_literals { - opt_ptr.lit_sum = scale_stats(&mut opt_ptr.lit_freq, MAX_LIT, 12)?; - } - opt_ptr.lit_length_sum = scale_stats(&mut opt_ptr.lit_length_freq, MAX_LL, 11)?; - opt_ptr.match_length_sum = scale_stats(&mut opt_ptr.match_length_freq, MAX_ML, 11)?; - opt_ptr.off_code_sum = scale_stats(&mut opt_ptr.off_code_freq, MAX_OFF, 11)?; - } - - set_base_prices(opt_ptr, opt_level); - Ok(()) -} - -/// `ZSTD_rawLiteralsCost`: the price of `literals`, without that of their length. -#[inline(always)] -fn raw_literals_cost(literals: &[u8], opt_ptr: &OptState, opt_level: i32) -> Result { - let lit_length = literals.len() as u32; - if lit_length == 0 { - return Ok(0); - } - - if !compressed_literals(opt_ptr) { - // 8 bits per literal. - return Ok((lit_length << 3).wrapping_mul(BITCOST_MULTIPLIER)); - } - - if opt_ptr.price_type == OptPrice::Predef { - // 6 bits per literal. - return Ok(lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER)); - } - - let mut price = opt_ptr.lit_sum_base_price.wrapping_mul(lit_length); - let lit_price_max = opt_ptr.lit_sum_base_price.wrapping_sub(BITCOST_MULTIPLIER); - for &literal in literals { - let lit_price = weight(opt_ptr.lit_freq.at(usize::from(literal))?, opt_level); - price = price.wrapping_sub(lit_price.min(lit_price_max)); - } - Ok(price) -} - -/// `ZSTD_litLengthPrice`: the price of the symbol for a length of literals. -#[inline(always)] -fn lit_length_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - if opt_ptr.price_type == OptPrice::Predef { - return Ok(weight(lit_length, opt_level)); - } - - // The format has no code for the length of a whole block of the largest size, which is then all literals. It is - // made to cost 1 bit more than the length before it. - let (lit_length, extra) = if lit_length == ZSTD_BLOCKSIZE_MAX as u32 { - (ZSTD_BLOCKSIZE_MAX as u32 - 1, BITCOST_MULTIPLIER) - } else { - (lit_length, 0) - }; - - let ll_code = ll_code(lit_length) as usize; - Ok(extra - .wrapping_add(u32::from(LL_BITS.at(ll_code)?) * BITCOST_MULTIPLIER) - .wrapping_add(opt_ptr.lit_length_sum_base_price) - .wrapping_sub(weight(opt_ptr.lit_length_freq.at(ll_code)?, opt_level))) -} - -/// `ZSTD_getMatchPrice`: the price of the offset and of the match length of a sequence. `off_base` is as in -/// [`SeqStore::store_seq`]. Below level 2, it favours small offsets, which are faster to decompress. -#[inline(always)] -fn get_match_price( - off_base: u32, - match_length: u32, - opt_ptr: &OptState, - opt_level: i32, -) -> Result { - let off_code = highbit32(off_base); - let ml_base = match_length.wrapping_sub(MINMATCH); - - if opt_ptr.price_type == OptPrice::Predef { - return Ok(weight(ml_base, opt_level).wrapping_add((16 + off_code) * BITCOST_MULTIPLIER)); - } - - let off_code_weight = weight(opt_ptr.off_code_freq.at(off_code as usize)?, opt_level); - let mut price = (off_code * BITCOST_MULTIPLIER).wrapping_add( - opt_ptr - .off_code_sum_base_price - .wrapping_sub(off_code_weight), - ); - if opt_level < 2 && off_code >= 20 { - price = price.wrapping_add((off_code - 19) * 2 * BITCOST_MULTIPLIER); - } - - let ml_code = ml_code(ml_base) as usize; - let ml_code_weight = weight(opt_ptr.match_length_freq.at(ml_code)?, opt_level); - price = price - .wrapping_add(u32::from(ML_BITS.at(ml_code)?) * BITCOST_MULTIPLIER) - .wrapping_add( - opt_ptr - .match_length_sum_base_price - .wrapping_sub(ml_code_weight), - ); - - // Matches cost a bit more, to have fewer sequences, which is faster to decompress. - Ok(price.wrapping_add(BITCOST_MULTIPLIER / 5)) -} - -/// `ZSTD_updateStats` -#[inline(always)] -fn update_stats( - opt_ptr: &mut OptState, - literals: &[u8], - off_base: u32, - match_length: u32, -) -> Result<()> { - #[inline(always)] - fn increment(freqs: &mut [u32], code: u32, sum: &mut u32) -> Result<()> { - let freq = freqs.at_mut(code as usize)?; - *freq = freq.wrapping_add(1); - *sum = sum.wrapping_add(1); - Ok(()) - } - - let lit_length = literals.len() as u32; - if compressed_literals(opt_ptr) { - for &literal in literals { - let freq = opt_ptr.lit_freq.at_mut(usize::from(literal))?; - *freq = freq.wrapping_add(ZSTD_LITFREQ_ADD); - } - opt_ptr.lit_sum = opt_ptr - .lit_sum - .wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD)); - } - - increment( - &mut opt_ptr.lit_length_freq, - ll_code(lit_length), - &mut opt_ptr.lit_length_sum, - )?; - increment( - &mut opt_ptr.off_code_freq, - highbit32(off_base), - &mut opt_ptr.off_code_sum, - )?; - increment( - &mut opt_ptr.match_length_freq, - ml_code(match_length.wrapping_sub(MINMATCH)), - &mut opt_ptr.match_length_sum, - ) -} - -/// `ZSTD_readMINMATCH`: `length` bytes, 3 or 4, to compare with others. Reads 4. -#[inline(always)] -fn read_minmatch(buf: &[u8], pos: usize, length: u32) -> Result { - let value = read_le32(buf, pos)?; - Ok(if length == 3 { value << 8 } else { value }) -} - -/// `ZSTD_insertAndFindFirstIndexHash3`: indexes the 3-byte hashes up to `ip`, which is not included, and looks it up. -/// All of it is in the current segment. -fn insert_and_find_first_index_hash3( - ms: &mut MatchState, - next_to_update3: &mut u32, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - let hash_table3 = ms.hash_table3.as_mut_slice(); - let hash_log3 = ms.hash_log3; - let target = prefix.index_of(ip); - let hash3 = hash3_ptr(prefix.buf, ip, hash_log3)?; - - for idx in *next_to_update3..target { - hash_table3.put(hash3_ptr(prefix.buf, prefix.pos_of(idx), hash_log3)?, idx)?; - } - - *next_to_update3 = target; - hash_table3.at(hash3) -} - -// ── binary tree search ──────────────────────────────────────────────────── - -/// `*ptr = value`, where `ptr` is an element of `bt`, or `None` for a place that nothing reads. -#[inline(always)] -fn store(bt: &mut [u32], ptr: Option, value: u32) -> Result<()> { - match ptr { - Some(ptr) => bt.put(ptr, value), - None => Ok(()), - } -} - -/// `ZSTD_insertBt1`: adds the position `ip`, which is at most `iend - 8`, to the tree. `target` is up to where the -/// caller is filling it, and `dict` the older segment, which is looked at if `ext_dict`. Returns how many positions -/// to move forward. -#[inline(always)] -fn insert_bt1( - ms: &mut MatchState, - prefix: Prefix<'_>, - dict: Prefix<'_>, - ip: usize, - iend: usize, - target: u32, - mls: u32, - ext_dict: bool, -) -> Result { - let c_params = ms.c_params; - let h = hash_ptr(prefix.buf, ip, c_params.hash_log, mls)?; - let bt_log = c_params.chain_log.wrapping_sub(1); - let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = dict.pos_of(dict_limit); - let prefix_start = prefix.pos_of(dict_limit); - let curr = prefix.index_of(ip); - let bt_low = curr.saturating_sub(bt_mask); - let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); - let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); - // Only the positions that are still in the window once the tree is filled are of use. - let window_low = ms.get_lowest_match_index(target, c_params.window_log); - let mut match_end_idx = curr.wrapping_add(8 + 1); - let mut best_length = 8usize; - let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); - - let bt = ms.chain_table.as_mut_slice(); - let hash_entry = ms.hash_table.at_mut(h)?; - let mut match_index = *hash_entry; - *hash_entry = curr; - - while nb_compares != 0 && match_index >= window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - // What is sure to be common. - let mut match_length = common_length_smaller.min(common_length_larger); - - // A position of the older segment, as one of the current segment, only makes sense with a length that - // brings it there. - let mut match_buf = prefix.buf; - let mut match_pos = prefix.pos_of(match_index); - if !ext_dict || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - prefix.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - match_length += count_2segments( - prefix.buf, - ip + match_length, - dict.buf, - dict.pos_of(match_index) + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - match_buf = dict.buf; - match_pos = dict.pos_of(match_index); - } - } - - if match_length > best_length { - best_length = match_length; - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); - } - } - - if ip + match_length == iend { - // They are equal: there is no telling which is smaller. Dropping it keeps the tree consistent. - break; - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < prefix.buf.at(ip + match_length)? { - // All the smaller ones will have at least this much in common. - store(bt, smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. - smaller_ptr = None; - break; - } - smaller_ptr = Some(next_ptr + 1); - match_index = bt.at(next_ptr + 1)?; - } else { - store(bt, larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = None; - break; - } - larger_ptr = Some(next_ptr); - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - store(bt, smaller_ptr, 0)?; - store(bt, larger_ptr, 0)?; - - let positions = if best_length > 384 { - 192.min((best_length - 384) as u32) - } else { - 0 - }; - Ok(positions.max(match_end_idx.wrapping_sub(curr.wrapping_add(8)))) -} - -/// `ZSTD_updateTree_internal` -#[inline(always)] -fn update_tree_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - dict: Prefix<'_>, - ip: usize, - iend: usize, - mls: u32, - dict_mode: DictMode, -) -> Result<()> { - let target = prefix.index_of(ip); - let mut idx = ms.next_to_update; - - while idx < target { - let forward = insert_bt1( - ms, - prefix, - dict, - prefix.pos_of(idx), - iend, - target, - mls, - dict_mode == DictMode::ExtDict, - )?; - idx = idx.wrapping_add(forward); - } - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. -/// Matches are not compared past the position `iend`. -pub(crate) fn update_tree( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - iend: usize, -) -> Result<()> { - let no_dict = Prefix { buf: &[], base: 0 }; - let mls = ms.c_params.min_match; - update_tree_internal(ms, prefix, no_dict, ip, iend, mls, DictMode::NoDict) -} - -/// `ZSTD_insertBtAndGetAllMatches`: adds the position `ip` to the tree, and stores in `matches` the matches found on -/// the way, each longer than the one before and all at least `length_to_beat` long. `ll0` is whether no literals -/// come before `ip`. Returns how many there are. -#[inline(always)] -fn insert_bt_and_get_all_matches( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - dict_mode: DictMode, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, - mls: u32, -) -> Result { - let buf = src.buf; - let c_params = ms.c_params; - let sufficient_len = c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); - let curr = src.index_of(ip); - let min_match = if mls == 3 { 3 } else { 4 }; - let h = hash_ptr(buf, ip, c_params.hash_log, mls)?; - let bt_log = c_params.chain_log.wrapping_sub(1); - let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt_low = curr.saturating_sub(bt_mask); - let window_low = ms.get_lowest_match_index(curr, c_params.window_log); - let match_low = window_low.max(1); - let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); - let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); - // The farthest position any match refers to, which detects repetitive patterns. - let mut match_end_idx = curr.wrapping_add(8 + 1); - let mut mnum = 0u32; - let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); - - let dms = if dict_mode == DictMode::DictMatchState { - Some(src.dms.ok_or(BUG)?) - } else { - None - }; - let dms_high_limit = dms.map_or(0, |dms| dms.end.wrapping_sub(dms.base) as u32); - let dms_low_limit = dms.map_or(0, |dms| dms.ms.window.low_limit); - let dms_index_delta = dms.map_or(0, |_| window_low.wrapping_sub(dms_high_limit)); - - let mut best_length = length_to_beat.wrapping_sub(1) as usize; - - // The repeat offsets. - let ll0 = u32::from(ll0); - let [rep0, _, _] = *rep; - for rep_code in ll0..ZSTD_REP_NUM as u32 + ll0 { - let rep_offset = match rep.get(rep_code as usize) { - Some(&offset) => offset, - None => rep0.wrapping_sub(1), - }; - let rep_index = curr.wrapping_sub(rep_offset); - let mut rep_len = 0u32; - // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. - if rep_offset.wrapping_sub(1) < curr.wrapping_sub(dict_limit) { - // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. - let rep_match = ip.wrapping_sub(rep_offset as usize); - if rep_index >= window_low - && read_minmatch(buf, ip, min_match)? == read_minmatch(buf, rep_match, min_match)? - { - rep_len = count( - buf, - ip + min_match as usize, - buf, - rep_match + min_match as usize, - i_limit, - )? as u32 - + min_match; - } - } else if dict_mode == DictMode::ExtDict { - // `curr > rep_index >= window_low` - if rep_offset.wrapping_sub(1) < curr.wrapping_sub(window_low) - && index_overlap_check(dict_limit, rep_index) - { - let rep_match = src.dict_pos_of(rep_index); - if read_minmatch(buf, ip, min_match)? - == read_minmatch(src.dict_buf, rep_match, min_match)? - { - rep_len = count_2segments( - buf, - ip + min_match as usize, - src.dict_buf, - rep_match + min_match as usize, - i_limit, - dict_end, - prefix_start, - )? as u32 - + min_match; - } - } - } else if let Some(dms) = dms { - // `curr > rep_index >= dms_low_limit` - if rep_offset.wrapping_sub(1) - < curr.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta)) - && index_overlap_check(dict_limit, rep_index) - { - let rep_match = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); - if read_minmatch(buf, ip, min_match)? - == read_minmatch(dms.buf, rep_match, min_match)? - { - rep_len = count_2segments( - buf, - ip + min_match as usize, - dms.buf, - rep_match + min_match as usize, - i_limit, - dms.end, - prefix_start, - )? as u32 - + min_match; - } - } - } - if rep_len as usize > best_length { - best_length = rep_len as usize; - matches.put( - mnum as usize, - Match { - off: rep_code - ll0 + 1, - len: rep_len, - }, - )?; - mnum += 1; - if rep_len > sufficient_len || ip + rep_len as usize == i_limit { - // The best there can be. - return Ok(mnum); - } - } - } - - // The table of 3-byte hashes. That of a dictionary is not filled. - if mls == 3 && best_length < mls as usize { - let match_index3 = - insert_and_find_first_index_hash3(ms, next_to_update3, src.prefix(), ip)?; - // Farther than that is likely to cost too much. - if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) { - let mlen = if dict_mode != DictMode::ExtDict || match_index3 >= dict_limit { - count(buf, ip, buf, src.pos_of(match_index3), i_limit)? - } else { - count_2segments( - buf, - ip, - src.dict_buf, - src.dict_pos_of(match_index3), - i_limit, - dict_end, - prefix_start, - )? - }; - - if mlen >= mls as usize { - best_length = mlen; - matches.put( - 0, - Match { - off: offset_to_offbase(curr.wrapping_sub(match_index3)), - len: mlen as u32, - }, - )?; - mnum = 1; - if mlen > sufficient_len as usize || ip + mlen == i_limit { - // The best there can be: `ip` is not inserted. - ms.next_to_update = curr.wrapping_add(1); - return Ok(1); - } - } - } - } - - let bt = ms.chain_table.as_mut_slice(); - let hash_entry = ms.hash_table.at_mut(h)?; - let mut match_index = *hash_entry; - *hash_entry = curr; - - while nb_compares != 0 && match_index >= match_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - // What is sure to be common. - let mut match_length = common_length_smaller.min(common_length_larger); - - // A position of the older segment, as one of the current segment, only makes sense with a length that - // brings it there. - let mut match_buf = buf; - let mut match_pos = src.pos_of(match_index); - if dict_mode != DictMode::ExtDict - || match_index as usize + match_length >= dict_limit as usize - { - match_length += count( - buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - i_limit, - )?; - } else { - match_length += count_2segments( - buf, - ip + match_length, - src.dict_buf, - src.dict_pos_of(match_index) + match_length, - i_limit, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - match_buf = src.dict_buf; - match_pos = src.dict_pos_of(match_index); - } - } - - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); - } - best_length = match_length; - matches.put( - mnum as usize, - Match { - off: offset_to_offbase(curr.wrapping_sub(match_index)), - len: match_length as u32, - }, - )?; - mnum += 1; - if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { - // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. - if dict_mode == DictMode::DictMatchState { - // The dictionary is not to be searched either. - nb_compares = 0; - } - break; - } - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < buf.at(ip + match_length)? { - // All the smaller ones will have at least this much in common. - store(bt, smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. - smaller_ptr = None; - break; - } - smaller_ptr = Some(next_ptr + 1); - match_index = bt.at(next_ptr + 1)?; - } else { - store(bt, larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = None; - break; - } - larger_ptr = Some(next_ptr); - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - store(bt, smaller_ptr, 0)?; - store(bt, larger_ptr, 0)?; - - if let Some(dms) = dms - && nb_compares != 0 - { - let dms_c_params = &dms.ms.c_params; - let dms_bt_log = dms_c_params.chain_log.wrapping_sub(1); - let dms_bt_mask = 1u32.wrapping_shl(dms_bt_log).wrapping_sub(1); - let dms_bt_low = if dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit) { - dms_high_limit.wrapping_sub(dms_bt_mask) - } else { - dms_low_limit - }; - let dms_h = hash_ptr(buf, ip, dms_c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(dms_h)?; - let dms_bt = dms.ms.chain_table.as_slice(); - common_length_smaller = 0; - common_length_larger = 0; - while nb_compares != 0 && dict_match_index > dms_low_limit { - let next_ptr = 2 * (dict_match_index & dms_bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - match_length += count_2segments( - buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - i_limit, - dms.end, - prefix_start, - )?; - - if match_length > best_length { - match_index = dict_match_index.wrapping_add(dms_index_delta); - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); - } - best_length = match_length; - matches.put( - mnum as usize, - Match { - off: offset_to_offbase(curr.wrapping_sub(match_index)), - len: match_length as u32, - }, - )?; - mnum += 1; - if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { - break; - } - } - - if dict_match_index <= dms_bt_low { - // Beyond the tree. - break; - } - // A match that reaches the end of the dictionary goes on at the start of the current segment. - let match_byte = if dict_match_index as usize + match_length >= dms_high_limit as usize - { - let match_pos = src.pos_of(dict_match_index.wrapping_add(dms_index_delta)); - buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? - }; - if match_byte < buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dms_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dms_bt.at(next_ptr)?; - } - nb_compares -= 1; - } - } - - // Repetitive patterns are skipped. - ms.next_to_update = match_end_idx.wrapping_sub(8); - Ok(mnum) -} - -/// `ZSTD_getAllMatchesFn`: the arguments are `matches`, `ms`, `next_to_update3`, `src`, `ip`, `i_high_limit`, `rep`, -/// `ll0` and `length_to_beat`. -type GetAllMatchesFn = fn( - &mut [Match], - &mut MatchState, - &mut u32, - &BlockSrc<'_>, - usize, - usize, - &[u32; ZSTD_REP_NUM], - bool, - u32, -) -> Result; - -/// `ZSTD_btGetAllMatches_internal` -#[inline(always)] -fn bt_get_all_matches_internal( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, - dict_mode: DictMode, - mls: u32, -) -> Result { - if src.index_of(ip) < ms.next_to_update { - // An area that is skipped. - return Ok(0); - } - let dict = Prefix { - buf: src.dict_buf, - base: src.dict_base, - }; - update_tree_internal(ms, src.prefix(), dict, ip, i_high_limit, mls, dict_mode)?; - insert_bt_and_get_all_matches( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - dict_mode, - rep, - ll0, - length_to_beat, - mls, - ) -} - -/// `ZSTD_btGetAllMatches_noDict_3` to `ZSTD_btGetAllMatches_noDict_6` -fn bt_get_all_matches_no_dict( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::NoDict, - MLS, - ) -} - -/// `ZSTD_btGetAllMatches_extDict_3` to `ZSTD_btGetAllMatches_extDict_6` -fn bt_get_all_matches_ext_dict( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::ExtDict, - MLS, - ) -} - -/// `ZSTD_btGetAllMatches_dictMatchState_3` to `ZSTD_btGetAllMatches_dictMatchState_6` -fn bt_get_all_matches_dict_match_state( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::DictMatchState, - MLS, - ) -} - -/// `ZSTD_selectBtGetAllMatches` -fn select_bt_get_all_matches(ms: &MatchState, dict_mode: DictMode) -> GetAllMatchesFn { - match (dict_mode, ms.c_params.min_match) { - (DictMode::NoDict, ..=3) => bt_get_all_matches_no_dict::<3>, - (DictMode::NoDict, 4) => bt_get_all_matches_no_dict::<4>, - (DictMode::NoDict, 5) => bt_get_all_matches_no_dict::<5>, - (DictMode::NoDict, 6..) => bt_get_all_matches_no_dict::<6>, - (DictMode::ExtDict, ..=3) => bt_get_all_matches_ext_dict::<3>, - (DictMode::ExtDict, 4) => bt_get_all_matches_ext_dict::<4>, - (DictMode::ExtDict, 5) => bt_get_all_matches_ext_dict::<5>, - (DictMode::ExtDict, 6..) => bt_get_all_matches_ext_dict::<6>, - (DictMode::DictMatchState, ..=3) => bt_get_all_matches_dict_match_state::<3>, - (DictMode::DictMatchState, 4) => bt_get_all_matches_dict_match_state::<4>, - (DictMode::DictMatchState, 5) => bt_get_all_matches_dict_match_state::<5>, - (DictMode::DictMatchState, 6..) => bt_get_all_matches_dict_match_state::<6>, - } -} - -// ── candidates from long distance matching ──────────────────────────────── - -/// `ZSTD_optLdm_t`: what it takes to decide whether to use a long distance match. -struct OptLdm<'a> { - /// `seqStore.seq`: the `seqStore.size` candidates for this block and the next ones. - seq: &'a [RawSeq], - /// `seqStore.pos` - pos: usize, - /// `seqStore.posInSequence` - pos_in_sequence: usize, - /// Where the current candidate starts. - start_pos_in_block: u32, - /// Where it ends. - end_pos_in_block: u32, - offset: u32, -} - -impl<'a> OptLdm<'a> { - fn new(seq_store: Option<&'a RawSeqStore>) -> Self { - Self { - seq: seq_store - .and_then(|store| store.seq.get(..store.size)) - .unwrap_or_default(), - pos: seq_store.map_or(0, |store| store.pos), - pos_in_sequence: seq_store.map_or(0, |store| store.pos_in_sequence), - start_pos_in_block: 0, - end_pos_in_block: 0, - offset: 0, - } - } - - /// `ZSTD_optLdm_skipRawSeqStoreBytes`: moves forward by `nb_bytes`. - fn skip_raw_seq_store_bytes(&mut self, nb_bytes: usize) { - let mut curr_pos = self.pos_in_sequence.wrapping_add(nb_bytes) as u32; - while curr_pos != 0 { - let Some(curr_seq) = self.seq.get(self.pos) else { - break; - }; - let seq_length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); - if curr_pos >= seq_length { - curr_pos -= seq_length; - self.pos += 1; - } else { - self.pos_in_sequence = curr_pos as usize; - break; - } - } - if curr_pos == 0 || self.pos == self.seq.len() { - self.pos_in_sequence = 0; - } - } - - /// `ZSTD_opt_getNextMatchAndUpdateSeqStore`: finds where the next candidate starts and ends in the block, and - /// moves past it. - fn get_next_match_and_update_seq_store( - &mut self, - curr_pos_in_block: u32, - block_bytes_remaining: u32, - ) { - let Some(&curr_seq) = self.seq.get(self.pos) else { - // No position gets there, so that there is no candidate in this block. - self.start_pos_in_block = u32::MAX; - self.end_pos_in_block = u32::MAX; - return; - }; - // What is left of the literals and of the match. - let pos_in_sequence = self.pos_in_sequence as u32; - let curr_block_end_pos = curr_pos_in_block.wrapping_add(block_bytes_remaining); - let literals_bytes_remaining = curr_seq.lit_length.saturating_sub(pos_in_sequence); - let match_bytes_remaining = if literals_bytes_remaining == 0 { - curr_seq - .match_length - .wrapping_sub(pos_in_sequence.wrapping_sub(curr_seq.lit_length)) - } else { - curr_seq.match_length - }; - - if literals_bytes_remaining >= block_bytes_remaining { - // The match starts after the block. - self.start_pos_in_block = u32::MAX; - self.end_pos_in_block = u32::MAX; - self.skip_raw_seq_store_bytes(block_bytes_remaining as usize); - return; - } - - // What is left of a match can be too short to use, which is checked later. - self.start_pos_in_block = curr_pos_in_block.wrapping_add(literals_bytes_remaining); - self.end_pos_in_block = self.start_pos_in_block.wrapping_add(match_bytes_remaining); - self.offset = curr_seq.offset; - - if self.end_pos_in_block > curr_block_end_pos { - // The match ends after the block, which can only use its start. - self.end_pos_in_block = curr_block_end_pos; - self.skip_raw_seq_store_bytes( - curr_block_end_pos.wrapping_sub(curr_pos_in_block) as usize - ); - } else { - self.skip_raw_seq_store_bytes( - literals_bytes_remaining.wrapping_add(match_bytes_remaining) as usize, - ); - } - } - - /// `ZSTD_optLdm_maybeAddMatch`: adds what is left of the candidate at `curr_pos_in_block` to `matches`, if it is - /// long enough and longer than all of them, since they are in order of length. - fn maybe_add_match( - &self, - matches: &mut [Match], - nb_matches: &mut u32, - curr_pos_in_block: u32, - min_match: u32, - ) -> Result<()> { - let pos_diff = curr_pos_in_block.wrapping_sub(self.start_pos_in_block); - let candidate_match_length = self - .end_pos_in_block - .wrapping_sub(self.start_pos_in_block) - .wrapping_sub(pos_diff); - - if curr_pos_in_block < self.start_pos_in_block - || curr_pos_in_block >= self.end_pos_in_block - || candidate_match_length < min_match - { - return Ok(()); - } - - if *nb_matches == 0 - || (candidate_match_length > matches.at(*nb_matches as usize - 1)?.len - && (*nb_matches as usize) < ZSTD_OPT_NUM) - { - matches.put( - *nb_matches as usize, - Match { - off: offset_to_offbase(self.offset), - len: candidate_match_length, - }, - )?; - *nb_matches += 1; - } - Ok(()) - } - - /// `ZSTD_optLdm_processMatchCandidate` - #[inline(always)] - fn process_match_candidate( - &mut self, - matches: &mut [Match], - nb_matches: &mut u32, - curr_pos_in_block: u32, - remaining_bytes: u32, - min_match: u32, - ) -> Result<()> { - if self.pos >= self.seq.len() { - return Ok(()); - } - - if curr_pos_in_block >= self.end_pos_in_block { - if curr_pos_in_block > self.end_pos_in_block { - // The parser does not stop at the end of each candidate: it is often some bytes past it. - let pos_overshoot = curr_pos_in_block - self.end_pos_in_block; - self.skip_raw_seq_store_bytes(pos_overshoot as usize); - } - self.get_next_match_and_update_seq_store(curr_pos_in_block, remaining_bytes); - } - self.maybe_add_match(matches, nb_matches, curr_pos_in_block, min_match) - } -} - -// ── the optimal parser ──────────────────────────────────────────────────── - -/// `LIT_PRICE`: the price of the literal at `pos`. -#[inline(always)] -fn lit_price(buf: &[u8], pos: usize, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok(raw_literals_cost(buf.span(pos, 1)?, opt_ptr, opt_level)? as i32) -} - -/// `LL_PRICE` -#[inline(always)] -fn ll_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok(lit_length_price(lit_length, opt_ptr, opt_level)? as i32) -} - -/// `LL_INCPRICE`: what one more literal, which makes `lit_length` of them, adds to the price of their length. -#[inline(always)] -fn ll_inc_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok( - ll_price(lit_length, opt_ptr, opt_level)?.wrapping_sub(ll_price( - lit_length.wrapping_sub(1), - opt_ptr, - opt_level, - )?), - ) -} - -/// `ZSTD_compressBlock_opt_generic` -#[inline(always)] -fn compress_block_opt_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - opt_level: i32, - dict_mode: DictMode, -) -> Result { - // The match finders borrow the whole of `ms`. - let mut price_table = mem::take(&mut ms.opt.price_table); - let mut match_table = mem::take(&mut ms.opt.match_table); - let result = parse_block( - ms, - seq_store, - rep, - src, - &mut price_table, - &mut match_table, - opt_level, - dict_mode, - ); - ms.opt.price_table = price_table; - ms.opt.match_table = match_table; - result -} - -/// The body of `ZSTD_compressBlock_opt_generic`. `opt` is `ms.opt.price_table` and `matches` is `ms.opt.match_table`. -#[inline(always)] -fn parse_block( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - opt: &mut [Optimal], - matches: &mut [Match], - opt_level: i32, - dict_mode: DictMode, -) -> Result { - let buf = src.buf; - let istart = src.start; - let mut ip = istart; - let mut anchor = istart; - let iend = src.end; - let ilimit = iend.saturating_sub(8); - let prefix_start = src.pos_of(ms.window.dict_limit); - - let get_all_matches = select_bt_get_all_matches(ms, dict_mode); - - let sufficient_len = ms.c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); - let min_match = if ms.c_params.min_match == 3 { 3 } else { 4 }; - let mut next_to_update3 = ms.next_to_update; - - let mut last_stretch = Optimal::default(); - let mut opt_ldm = OptLdm::new(src.ldm_seq_store); - opt_ldm.get_next_match_and_update_seq_store(0, (iend - ip) as u32); - - rescale_freqs( - &mut ms.opt, - src.symbol_costs, - buf.sub(istart..iend)?, - opt_level, - )?; - ip += usize::from(ip == prefix_start); - - 'series: while ip < ilimit { - let mut last_pos: u32; - - // The forward pass. `opt` holds stretches, each a match followed by literals, unlike a sequence, which is - // literals followed by a match: that way each position of a run of literals can have another match before it. - // It ends with the last stretch in `last_stretch`, which starts at `cur` and ends at `last_pos`. - let mut cur = 'forward: { - // Find a first match. - { - let litlen = (ip - anchor) as u32; - let mut nb_matches = get_all_matches( - matches, - ms, - &mut next_to_update3, - src, - ip, - iend, - rep, - litlen == 0, - min_match, - )?; - opt_ldm.process_match_candidate( - matches, - &mut nb_matches, - (ip - istart) as u32, - (iend - ip) as u32, - min_match, - )?; - if nb_matches == 0 { - ip += 1; - continue 'series; - } - let matches = matches.sub(..nb_matches as usize)?; - - // The price of the literals before the first match is the same whatever comes next, so it is left - // out. But how the price of their length changes depends on their length. - let first_price = ll_price(litlen, &ms.opt, opt_level)?; - { - let first = opt.at_mut(0)?; - first.mlen = 0; - first.litlen = litlen; - first.price = first_price; - first.rep = *rep; - } - - // A large match is encoded at once. - let longest = matches.at(matches.len() - 1)?; - if longest.len > sufficient_len { - last_stretch.litlen = 0; - last_stretch.mlen = longest.len; - last_stretch.off = longest.off; - last_pos = longest.len; - break 'forward 0; - } - - // Set the prices of the first matches. - let ll0_price = ll_price(0, &ms.opt, opt_level)?; - let mut pos = 1u32; - while pos < min_match { - let stretch = opt.at_mut(pos as usize)?; - stretch.price = ZSTD_MAX_PRICE; - stretch.mlen = 0; - stretch.litlen = litlen.wrapping_add(pos); - pos += 1; - } - for candidate in matches { - while pos <= candidate.len { - let match_price = - get_match_price(candidate.off, pos, &ms.opt, opt_level)? as i32; - let sequence_price = first_price.wrapping_add(match_price); - let stretch = opt.at_mut(pos as usize)?; - stretch.mlen = pos; - stretch.off = candidate.off; - stretch.litlen = 0; - stretch.price = sequence_price.wrapping_add(ll0_price); - pos += 1; - } - } - last_pos = pos - 1; - opt.at_mut(pos as usize)?.price = ZSTD_MAX_PRICE; - } - - // Check the positions that follow. - let mut cur = 0u32; - while cur < last_pos { - cur += 1; - let inr = ip + cur as usize; - - // Get to this position by a literal, if it is cheaper. - { - let before = opt.at(cur as usize - 1)?; - let litlen = before.litlen.wrapping_add(1); - let price = before - .price - .wrapping_add(lit_price(buf, inr - 1, &ms.opt, opt_level)?) - .wrapping_add(ll_inc_price(litlen, &ms.opt, opt_level)?); - let prev_match = opt.at(cur as usize)?; - if price <= prev_match.price { - opt.put( - cur as usize, - Optimal { - litlen, - price, - ..before - }, - )?; - // If this replaces a match, and 1 literal is cheaper than none, the match followed by a - // literal may be a cheaper way to the next position. - if opt_level >= 1 - && prev_match.litlen == 0 - && ll_inc_price(1, &ms.opt, opt_level)? < 0 - && inr < iend - { - let next_lit_price = lit_price(buf, inr, &ms.opt, opt_level)?; - let with_1_literal = prev_match - .price - .wrapping_add(next_lit_price) - .wrapping_add(ll_inc_price(1, &ms.opt, opt_level)?); - let with_more_literals = price - .wrapping_add(next_lit_price) - .wrapping_add(ll_inc_price( - litlen.wrapping_add(1), - &ms.opt, - opt_level, - )?); - if with_1_literal < with_more_literals - && with_1_literal < opt.at(cur as usize + 1)?.price - { - let prev = opt.at(cur.wrapping_sub(prev_match.mlen) as usize)?; - opt.put( - cur as usize + 1, - Optimal { - rep: new_rep(&prev.rep, prev_match.off, prev.litlen == 0), - litlen: 1, - price: with_1_literal, - ..prev_match - }, - )?; - last_pos = last_pos.max(cur + 1); - } - } - } - } - - // The repeat offsets are not updated while matches are compared, but here, now that one is selected. - let mut here = opt.at(cur as usize)?; - if here.litlen == 0 { - let prev = opt.at(cur.wrapping_sub(here.mlen) as usize)?; - here.rep = new_rep(&prev.rep, here.off, prev.litlen == 0); - opt.at_mut(cur as usize)?.rep = here.rep; - } - - // The last match starts at least 8 bytes before the end. - if inr > ilimit { - continue; - } - - if cur == last_pos { - break; - } - - if opt_level == 0 - && opt.at(cur as usize + 1)?.price - <= here.price.wrapping_add(BITCOST_MULTIPLIER as i32 / 2) - { - // The next position is cheaper to get to: this one is not promising. - continue; - } - - { - let base_price = here.price.wrapping_add(ll_price(0, &ms.opt, opt_level)?); - let mut nb_matches = get_all_matches( - matches, - ms, - &mut next_to_update3, - src, - inr, - iend, - &here.rep, - here.litlen == 0, - min_match, - )?; - opt_ldm.process_match_candidate( - matches, - &mut nb_matches, - (inr - istart) as u32, - (iend - inr) as u32, - min_match, - )?; - if nb_matches == 0 { - continue; - } - let matches = matches.sub(..nb_matches as usize)?; - - let longest = matches.at(matches.len() - 1)?; - if longest.len > sufficient_len - || cur.wrapping_add(longest.len) as usize >= ZSTD_OPT_NUM - || inr + longest.len as usize >= iend - { - last_stretch.mlen = longest.len; - last_stretch.off = longest.off; - last_stretch.litlen = 0; - last_pos = cur.wrapping_add(longest.len); - break 'forward cur; - } - - // Set the prices that the matches found here give. - let mut start_ml = min_match; - for candidate in matches { - let mut mlen = candidate.len; - while mlen >= start_ml { - let pos = cur + mlen; - let price = base_price.wrapping_add(get_match_price( - candidate.off, - mlen, - &ms.opt, - opt_level, - )? - as i32); - - if pos > last_pos || price < opt.at(pos as usize)?.price { - while last_pos < pos { - // Positions that nothing leads to yet. - last_pos += 1; - let stretch = opt.at_mut(last_pos as usize)?; - stretch.price = ZSTD_MAX_PRICE; - // Not 0, which would be the end of a match. - stretch.litlen = 1; - } - let stretch = opt.at_mut(pos as usize)?; - stretch.mlen = mlen; - stretch.off = candidate.off; - stretch.litlen = 0; - stretch.price = price; - } else if opt_level == 0 { - // Shorter lengths are not tried. - break; - } - mlen -= 1; - } - start_ml = candidate.len + 1; - } - } - opt.at_mut(last_pos as usize + 1)?.price = ZSTD_MAX_PRICE; - } - - last_stretch = opt.at(last_pos as usize)?; - last_pos.wrapping_sub(last_stretch.mlen) - }; - - if last_stretch.mlen == 0 { - // All the matches have been turned into literals. - ip += last_pos as usize; - continue; - } - - if last_stretch.litlen == 0 { - // It ends with a match. - let prev = opt.at(cur as usize)?; - *rep = new_rep(&prev.rep, last_stretch.off, prev.litlen == 0); - } else { - *rep = last_stretch.rep; - cur = cur.wrapping_sub(last_stretch.litlen); - } - - // Write the cheapest path in `opt`, backwards from `store_end`, over the stretches that are not on it, and as - // sequences. The literals that end the last stretch are in no sequence: they are parsed again. (The C code - // starts by making them a last sequence without a match, and overwrites it at once.) - let store_end = cur.wrapping_add(2) as usize; - let mut store_start = store_end; - let mut stretch_pos = cur; - opt.put(store_end, last_stretch)?; - loop { - let next_stretch = opt.at(stretch_pos as usize)?; - opt.at_mut(store_start)?.litlen = next_stretch.litlen; - if next_stretch.mlen == 0 { - // The start of the series. - break; - } - store_start = store_start.wrapping_sub(1); - opt.put(store_start, next_stretch)?; - stretch_pos = - stretch_pos.wrapping_sub(next_stretch.litlen.wrapping_add(next_stretch.mlen)); - } - - for sequence in opt.sub(store_start..=store_end)? { - let llen = sequence.litlen as usize; - let mlen = sequence.mlen as usize; - update_stats( - &mut ms.opt, - buf.span(anchor, llen)?, - sequence.off, - sequence.mlen, - )?; - seq_store.store_seq(llen, buf, anchor, sequence.off, mlen)?; - anchor += llen + mlen; - ip = anchor; - } - - set_base_prices(&mut ms.opt, opt_level); - } - - Ok(iend - anchor) -} - -/// `ZSTD_compressBlock_opt0` -fn compress_block_opt0( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode) -} - -/// `ZSTD_compressBlock_opt2` -fn compress_block_opt2( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode) -} - -/// `ZSTD_initStats_ultra`: a first pass over the block, only to have better statistics to start from. It has to be the -/// first block, with no dictionary and no long distance matching. Returns the block as the window then sees it, since -/// its indices change. -fn init_stats_ultra<'a>( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &[u32; ZSTD_REP_NUM], - src: &BlockSrc<'a>, -) -> Result> { - // The repeat offsets it leaves are of no use. - let mut tmp_rep = *rep; - compress_block_opt2(ms, seq_store, &mut tmp_rep, src, DictMode::NoDict)?; - - // Of the first pass, only the statistics are kept: what the tables have is made out of reach. - let src_size = src.end - src.start; - seq_store.reset(); - ms.window.base = ms.window.base.wrapping_sub(src_size); - ms.window.dict_limit = ms.window.dict_limit.wrapping_add(src_size as u32); - ms.window.low_limit = ms.window.dict_limit; - ms.next_to_update = ms.window.dict_limit; - Ok(BlockSrc { - base: src.base.wrapping_sub(src_size), - ..*src - }) -} - -/// `ZSTD_compressBlock_btultra2` -fn compress_block_btultra2( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let curr = src.index_of(src.start); - - // The first block is parsed twice, the first time to collect statistics for the second. Since the first pass is - // then forgotten, nothing must be in the tables before it: no dictionary, no prefix, no long distance matching. - if ms.opt.lit_length_sum == 0 - && seq_store.sequences.is_empty() - && ms.window.dict_limit == ms.window.low_limit - && curr == ms.window.dict_limit - && src.end - src.start > ZSTD_PREDEF_THRESHOLD - { - let src = init_stats_ultra(ms, seq_store, rep, src)?; - return compress_block_opt2(ms, seq_store, rep, &src, DictMode::NoDict); - } - - compress_block_opt2(ms, seq_store, rep, src, DictMode::NoDict) -} - -/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` -/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. -/// -/// `btultra2` can move `ms.window.base`, which `src.base` comes from. -pub(crate) fn compress_block_opt( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - dict_mode: DictMode, -) -> Result { - match (strategy, dict_mode) { - (Strategy::BtUltra2, DictMode::NoDict) => compress_block_btultra2(ms, seq_store, rep, src), - (Strategy::BtUltra | Strategy::BtUltra2, _) => { - compress_block_opt2(ms, seq_store, rep, src, dict_mode) - } - _ => compress_block_opt0(ms, seq_store, rep, src, dict_mode), - } -} diff --git a/src/zstd/wip/work/opt/zstd/compress/presplit.rs b/src/zstd/wip/work/opt/zstd/compress/presplit.rs deleted file mode 100644 index 8b778e2989f9..000000000000 --- a/src/zstd/wip/work/opt/zstd/compress/presplit.rs +++ /dev/null @@ -1 +0,0 @@ -//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. From 43aa6d8f9cc90dec351456359d8fa664fe19836d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 12:41:28 +0000 Subject: [PATCH 006/199] Bun.zstdCompress: a failed allocation is an out of memory error --- src/runtime/api/BunObject.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runtime/api/BunObject.rs b/src/runtime/api/BunObject.rs index 2704fcde9463..2655b6f88dba 100644 --- a/src/runtime/api/BunObject.rs +++ b/src/runtime/api/BunObject.rs @@ -1954,6 +1954,7 @@ pub(crate) mod JSZstd { fn compress_to_box(input: &[u8], level: i32) -> Result, Failure> { match bun_zstd::compress_alloc(input, level) { Ok(output) => Ok(output.into_boxed_slice()), + Err(bun_zstd::ErrorCode::MemoryAllocation) => Err(Failure::OutOfMemory), Err(err) => Err(Failure::Compression(err.message().to_bytes())), } } From 0f99288ba06727cacf44af81cc4cf483575fb4e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 12:42:13 +0000 Subject: [PATCH 007/199] zstd: optimize the crate in debug builds --- Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index d0704ac8915c..ee902c1ce5a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -495,6 +495,11 @@ bun_simdutf_sys = { path = "src/simdutf_sys" } bun_which = { path = "src/which" } bun_transpiler = { path = "src/transpiler" } +# At opt-level 0 a codec is slow enough for tests to time out. The C library that this replaces was optimized in debug +# builds too. +[profile.dev.package.bun_zstd] +opt-level = 3 + [profile.release.package.bun_react_compiler] opt-level = "s" From 519fe26cce6c8fe99e026def24835fdebd8a416e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 12:43:06 +0000 Subject: [PATCH 008/199] zstd: test that the output is that of the C library --- test/js/bun/util/zstd.test.ts | 198 ++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 880b3bea894b..cd00ad10c97d 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1054,3 +1054,201 @@ describe.concurrent("Zstandard HTTP compression", () => { expect(echoed).toBe(testString); }); }); + +// The compressor is a port of zstd 1.5.7, and gives the same bytes. The expected values are from the C library. +describe("the output is that of zstd 1.5.7", () => { + // Words, copies of earlier parts at all distances, noise and runs: something for every part of the compressor. + function sample(size: number, seed: number) { + let state = seed >>> 0 || 1; + const next = () => { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + return (state >>>= 0); + }; + const words = ["function ", "return ", "const ", "zstd", "\n ", "{", "}", "(", ");", "await ", "=> ", "length", "0"]; + const out = new Uint8Array(size); + let at = 0; + while (at < size) { + const kind = next() % 16; + if (kind < 8) { + for (const c of words[next() % words.length]) if (at < size) out[at++] = c.charCodeAt(0); + } else if (kind < 13 && at > 8) { + const distance = 1 + (next() % Math.min(at, 1 << (next() % 20))); + for (let n = 3 + (next() % (1 << (next() % 10))); n > 0 && at < size; n--, at++) out[at] = out[at - distance]; + } else if (kind < 15) { + for (let n = 1 + (next() % 40); n > 0 && at < size; n--) out[at++] = next(); + } else { + const byte = next(); + for (let n = 1 + (next() % 2000); n > 0 && at < size; n--) out[at++] = byte; + } + } + return out; + } + const digest = (bytes: Uint8Array) => `${bytes.length} ${Bun.hash.xxHash64(bytes).toString(16)}`; + const c = zlib.constants; + + it("at every level", () => { + const large = sample(400_000, 1); + const small = sample(3_000, 2); + const results: Record = {}; + for (let level = 1; level <= 22; level++) { + results[level] = `${digest(zstdCompressSync(large, { level }))}, ${digest(zstdCompressSync(small, { level }))}`; + } + for (const level of [-1, -7, -100]) { + const params = { [c.ZSTD_c_compressionLevel]: level }; + results[level] = `${digest(zlib.zstdCompressSync(large, { params }))}, ${digest(zlib.zstdCompressSync(small, { params }))}`; + } + expect(results).toMatchInlineSnapshot(` + { + "-1": "23477 2ffb3d0f3737833b, 382 c2a4c99592694597", + "-100": "79898 7ffbd56b19ee394, 1100 ae3344236a56deb9", + "-7": "29376 651fda629ac959c4, 481 55bfd66062e7c9ad", + "1": "24480 542ab87e1aa5fc9d, 396 107a4b7a04d25d37", + "10": "22592 19e5d2e09fe1ccad, 363 808b964648b85372", + "11": "22525 e0091e2608de2f00, 365 d83d5f8947b95fc1", + "12": "22525 e0091e2608de2f00, 365 dd249e23858fad1", + "13": "22510 f89e05ebc8ca7534, 358 eb0fe5ebe5118657", + "14": "22503 56827c33f86de3ae, 358 2f482857ef5c63f0", + "15": "22494 cc870ca463c122df, 356 7ceb5d984fe44594", + "16": "22635 c4a94ba8858c12d5, 355 25870ac97fe974", + "17": "22416 9a7fd5afd172f89b, 355 eede2d4cbdcf2340", + "18": "22010 8c967ec9b15ed62c, 353 5fffa6db6ed0ebda", + "19": "21820 967a4e946bf30696, 353 5fffa6db6ed0ebda", + "2": "23951 df37d7c5becb22d8, 384 7f01926a9aa9a46b", + "20": "21820 967a4e946bf30696, 353 478b55fe951a5684", + "21": "21735 ca28a8b1ffaeb4d9, 353 478b55fe951a5684", + "22": "21651 e9a29d4a60784b2c, 354 50650b6ef5b2b80a", + "3": "23931 de5a0598c57be84b, 382 c2a4c99592694597", + "4": "23923 85d64ea007fb86d3, 369 312f9dca641ef386", + "5": "23732 be13ecafaf15c817, 369 f3efffcbd8524307", + "6": "22720 47e610d53a3c1546, 364 6175591ab882c5a6", + "7": "22749 8176e71ad03765e7, 361 c028bca7b43936ae", + "8": "22683 96b450c8ed9a4a06, 361 c028bca7b43936ae", + "9": "22683 96b450c8ed9a4a06, 363 808b964648b85372", + } + `); + }); + + it("with parameters", () => { + const data = sample(700_000, 3); + const cases: Record> = { + checksum: { [c.ZSTD_c_checksumFlag]: 1 }, + noContentSize: { [c.ZSTD_c_contentSizeFlag]: 0 }, + smallWindow: { [c.ZSTD_c_windowLog]: 12, [c.ZSTD_c_compressionLevel]: 7 }, + hashChain: { [c.ZSTD_c_strategy]: c.ZSTD_lazy2, [c.ZSTD_c_windowLog]: 14, [c.ZSTD_c_searchLog]: 6 }, + rows: { [c.ZSTD_c_strategy]: c.ZSTD_greedy, [c.ZSTD_c_searchLog]: 6, [c.ZSTD_c_minMatch]: 6 }, + binaryTree: { [c.ZSTD_c_strategy]: c.ZSTD_btlazy2, [c.ZSTD_c_chainLog]: 14, [c.ZSTD_c_hashLog]: 13 }, + optimal: { [c.ZSTD_c_strategy]: c.ZSTD_btopt, [c.ZSTD_c_targetLength]: 20, [c.ZSTD_c_minMatch]: 3 }, + longDistance: { [c.ZSTD_c_enableLongDistanceMatching]: 1, [c.ZSTD_c_windowLog]: 20 }, + longDistanceOptimal: { + [c.ZSTD_c_enableLongDistanceMatching]: 1, + [c.ZSTD_c_compressionLevel]: 16, + [c.ZSTD_c_ldmMinMatch]: 16, + [c.ZSTD_c_ldmHashLog]: 12, + [c.ZSTD_c_ldmBucketSizeLog]: 2, + [c.ZSTD_c_ldmHashRateLog]: 3, + }, + // ZSTD_c_targetCBlockSize + smallBlocks: { 130: 2000, [c.ZSTD_c_compressionLevel]: 5 }, + // ZSTD_c_format: ZSTD_f_zstd1_magicless + magicless: { 10: 1 }, + workers: { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 1, [c.ZSTD_c_checksumFlag]: 1 }, + workersOverlap: { [c.ZSTD_c_nbWorkers]: 3, [c.ZSTD_c_jobSize]: 1, [c.ZSTD_c_overlapLog]: 9, [c.ZSTD_c_windowLog]: 17 }, + workersLongDistance: { + [c.ZSTD_c_nbWorkers]: 2, + [c.ZSTD_c_jobSize]: 1, + [c.ZSTD_c_enableLongDistanceMatching]: 1, + [c.ZSTD_c_windowLog]: 18, + }, + }; + const results: Record = {}; + for (const [name, params] of Object.entries(cases)) { + const compressed = zlib.zstdCompressSync(data, { params }); + results[name] = digest(compressed); + if (name !== "magicless") expect(zlib.zstdDecompressSync(compressed).equals(data)).toBe(true); + } + expect(results).toMatchInlineSnapshot(` + { + "binaryTree": "39948 1232d8597f5c6", + "checksum": "41241 3cd85e71c12cf382", + "hashChain": "43217 7356b4e36e4bf4d8", + "longDistance": "41055 eedc98c3e261bec7", + "longDistanceOptimal": "38970 b85121583248455b", + "magicless": "41233 5ee803ef2ad12e33", + "noContentSize": "41234 b8ffb3bc3aef5fff", + "optimal": "38610 6d0c3f5689a3abe9", + "rows": "40635 26480c2e116c84d5", + "smallBlocks": "41058 4ef9d31b35c0acf8", + "smallWindow": "47463 95833be523748c4a", + "workers": "42269 b08e46380ba74997", + "workersLongDistance": "42141 793c24a764fcda50", + "workersOverlap": "42697 e672bcac52118c4d", + } + `); + }); + + it("with a dictionary", () => { + const dictionary = sample(20_000, 4); + const results: Record = {}; + for (const level of [-3, 1, 3, 5, 8, 13, 16, 19]) { + // Short input searches the tables of the dictionary, long input starts from a copy of them. + for (const size of [5_000, 200_000]) { + const data = new Uint8Array(size); + data.set(sample(size, 5)); + data.set(dictionary.subarray(3_000, 6_000), 1_000); + const options = { dictionary, params: { [c.ZSTD_c_compressionLevel]: level }, pledgedSrcSize: size }; + const compressed = zlib.zstdCompressSync(data, options); + results[`${level}, ${size}`] = digest(compressed); + expect(zlib.zstdDecompressSync(compressed, { dictionary }).equals(data)).toBe(true); + } + } + expect(results).toMatchInlineSnapshot(` + { + "-3, 200000": "11919 4646c7adfd3c7bfc", + "-3, 5000": "99 bcf9a6b6959f244", + "1, 200000": "11393 8ca74189c4dd8590", + "1, 5000": "85 5d94a445fbc1debf", + "13, 200000": "10612 72de0a04d3ab1dcc", + "13, 5000": "81 cf05da8002ffcca1", + "16, 200000": "10283 7e7f3e5510d0205b", + "16, 5000": "76 83586779a3533541", + "19, 200000": "10258 ecf6c3f4322413b5", + "19, 5000": "76 83586779a3533541", + "3, 200000": "11130 17e414f5ae34a715", + "3, 5000": "86 9b1f540d55f03dd", + "5, 200000": "10837 6628de069ef68382", + "5, 5000": "81 e436c64b13b807ee", + "8, 200000": "10482 8c6f6c78776fc48e", + "8, 5000": "78 ba67e3347e65a4a8", + } + `); + }); + + it("in a stream with flushes", async () => { + const data = sample(600_000, 6); + const results: Record = {}; + for (const level of [1, 6, 12, 17]) { + const stream = zlib.createZstdCompress({ params: { [c.ZSTD_c_compressionLevel]: level } }); + const chunks: Buffer[] = []; + stream.on("data", chunk => chunks.push(chunk)); + const { promise, resolve, reject } = Promise.withResolvers(); + stream.on("end", resolve).on("error", reject); + for (let at = 0, n = 1; at < data.length; at += n, n = (n * 7) % 90_001) { + stream.write(data.subarray(at, at + n)); + if (n % 3 === 0) stream.flush(); + } + stream.end(); + await promise; + results[level] = digest(Buffer.concat(chunks)); + } + expect(results).toMatchInlineSnapshot(` + { + "1": "34466 1bd006842da7fff7", + "12": "31194 38b6be8e301dd3fc", + "17": "31048 2bcbaf13b8ae7446", + "6": "31602 54d0ef9110ac7561", + } + `); + }); +}); From 41d3dad5405766c465212c7348d69f806d978052 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 12:50:17 +0000 Subject: [PATCH 009/199] zstd: remove the scratch files, and keep one copy of a dictionary --- src/jsc/bindings/xxhash3.cpp | 2 +- src/runtime/node/zlib/NativeZstd.rs | 5 +- src/zstd/compress/cctx.rs | 4 +- src/zstd/compress/params.rs | 2 +- src/zstd/compress/stream.rs | 4 +- src/zstd/wip/PORTING.md | 75 - src/zstd/wip/build-ref.sh | 15 - src/zstd/wip/clevels.rs | 100 -- src/zstd/wip/default_tables.rs | 51 - src/zstd/wip/harness/Cargo.toml | 20 - src/zstd/wip/harness/build.rs | 6 - src/zstd/wip/harness/rust-toolchain.toml | 35 - src/zstd/wip/harness/src/c.rs | 104 -- src/zstd/wip/harness/src/comp.rs | 296 ---- src/zstd/wip/harness/src/data.rs | 65 - src/zstd/wip/harness/src/dec.rs | 184 --- src/zstd/wip/harness/src/main.rs | 18 - src/zstd/wip/par.sh | 11 - src/zstd/wip/strip.py | 7 - src/zstd/wip/template/mkwork.sh | 61 - src/zstd/wip/template/seqs.rs | 73 - src/zstd/wip/template/testing.rs | 140 -- src/zstd/wip/work/entropy/STATUS.md | 46 - src/zstd/wip/work/entropy/harness/Cargo.toml | 20 - src/zstd/wip/work/entropy/harness/build.rs | 20 - .../wip/work/entropy/harness/entropy_shim.c | 43 - .../work/entropy/harness/rust-toolchain.toml | 35 - src/zstd/wip/work/entropy/harness/src/c.rs | 104 -- src/zstd/wip/work/entropy/harness/src/data.rs | 65 - .../wip/work/entropy/harness/src/entropy.rs | 774 --------- src/zstd/wip/work/entropy/harness/src/main.rs | 22 - src/zstd/wip/work/entropy/harness/src/seqs.rs | 73 - src/zstd/wip/work/entropy/zstd/lib.rs | 385 ----- src/zstd/wip/work/entropy/zstd/testing.rs | 457 ------ src/zstd/wip/work/fast/STATUS.md | 35 - src/zstd/wip/work/fast/harness/Cargo.toml | 20 - src/zstd/wip/work/fast/harness/build.rs | 21 - src/zstd/wip/work/fast/harness/c/shim.c | 169 -- .../wip/work/fast/harness/c/shim_cctx.inc | 30 - .../wip/work/fast/harness/rust-toolchain.toml | 35 - src/zstd/wip/work/fast/harness/src/c.rs | 104 -- src/zstd/wip/work/fast/harness/src/data.rs | 65 - src/zstd/wip/work/fast/harness/src/frames.rs | 178 --- src/zstd/wip/work/fast/harness/src/main.rs | 44 - src/zstd/wip/work/fast/harness/src/seqs.rs | 78 - src/zstd/wip/work/fast/harness/src/stream.rs | 301 ---- src/zstd/wip/work/fast/zstd/lib.rs | 386 ----- src/zstd/wip/work/fast/zstd/testing.rs | 364 ----- src/zstd/wip/work/lazy/STATUS.md | 53 - src/zstd/wip/work/lazy/harness/Cargo.toml | 20 - src/zstd/wip/work/lazy/harness/build.rs | 18 - .../wip/work/lazy/harness/rust-toolchain.toml | 35 - src/zstd/wip/work/lazy/harness/shim.c | 41 - src/zstd/wip/work/lazy/harness/src/c.rs | 104 -- src/zstd/wip/work/lazy/harness/src/data.rs | 65 - src/zstd/wip/work/lazy/harness/src/main.rs | 27 - src/zstd/wip/work/lazy/harness/src/seqs.rs | 193 --- src/zstd/wip/work/lazy/lazy.exp-filter.rs | 1360 ---------------- src/zstd/wip/work/lazy/lazy.exp-prefetch.rs | 1384 ---------------- src/zstd/wip/work/lazy/lazy.v1.rs | 1353 ---------------- src/zstd/wip/work/lazy/lazy.v2.rs | 1419 ----------------- src/zstd/wip/work/lazy/lib.orig.rs | 385 ----- src/zstd/wip/work/lazy/zstd/lib.rs | 386 ----- src/zstd/wip/work/lazy/zstd/testing.rs | 372 ----- src/zstd/wip/work/ldm/harness/Cargo.toml | 20 - src/zstd/wip/work/ldm/harness/build.rs | 19 - .../wip/work/ldm/harness/rust-toolchain.toml | 35 - src/zstd/wip/work/ldm/harness/shim.c | 180 --- src/zstd/wip/work/ldm/harness/src/c.rs | 104 -- src/zstd/wip/work/ldm/harness/src/data.rs | 65 - src/zstd/wip/work/ldm/harness/src/ldm.rs | 452 ------ src/zstd/wip/work/ldm/harness/src/main.rs | 30 - src/zstd/wip/work/ldm/harness/src/seqs.rs | 73 - src/zstd/wip/work/ldm/zstd/lib.rs | 385 ----- src/zstd/wip/work/ldm/zstd/testing.rs | 656 -------- src/zstd/wip/work/opt/STATUS.md | 38 - src/zstd/wip/work/opt/harness/Cargo.toml | 20 - src/zstd/wip/work/opt/harness/build.rs | 20 - .../wip/work/opt/harness/rust-toolchain.toml | 35 - src/zstd/wip/work/opt/harness/shim.c | 98 -- src/zstd/wip/work/opt/harness/src/c.rs | 104 -- src/zstd/wip/work/opt/harness/src/data.rs | 65 - src/zstd/wip/work/opt/harness/src/main.rs | 25 - src/zstd/wip/work/opt/harness/src/seqs.rs | 235 --- src/zstd/wip/work/opt/zstd/lib.rs | 385 ----- src/zstd/wip/work/opt/zstd/testing.rs | 288 ---- 86 files changed, 9 insertions(+), 15690 deletions(-) delete mode 100644 src/zstd/wip/PORTING.md delete mode 100755 src/zstd/wip/build-ref.sh delete mode 100644 src/zstd/wip/clevels.rs delete mode 100644 src/zstd/wip/default_tables.rs delete mode 100644 src/zstd/wip/harness/Cargo.toml delete mode 100644 src/zstd/wip/harness/build.rs delete mode 100644 src/zstd/wip/harness/rust-toolchain.toml delete mode 100644 src/zstd/wip/harness/src/c.rs delete mode 100644 src/zstd/wip/harness/src/comp.rs delete mode 100644 src/zstd/wip/harness/src/data.rs delete mode 100644 src/zstd/wip/harness/src/dec.rs delete mode 100644 src/zstd/wip/harness/src/main.rs delete mode 100755 src/zstd/wip/par.sh delete mode 100644 src/zstd/wip/strip.py delete mode 100755 src/zstd/wip/template/mkwork.sh delete mode 100644 src/zstd/wip/template/seqs.rs delete mode 100644 src/zstd/wip/template/testing.rs delete mode 100644 src/zstd/wip/work/entropy/STATUS.md delete mode 100644 src/zstd/wip/work/entropy/harness/Cargo.toml delete mode 100644 src/zstd/wip/work/entropy/harness/build.rs delete mode 100644 src/zstd/wip/work/entropy/harness/entropy_shim.c delete mode 100644 src/zstd/wip/work/entropy/harness/rust-toolchain.toml delete mode 100644 src/zstd/wip/work/entropy/harness/src/c.rs delete mode 100644 src/zstd/wip/work/entropy/harness/src/data.rs delete mode 100644 src/zstd/wip/work/entropy/harness/src/entropy.rs delete mode 100644 src/zstd/wip/work/entropy/harness/src/main.rs delete mode 100644 src/zstd/wip/work/entropy/harness/src/seqs.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/lib.rs delete mode 100644 src/zstd/wip/work/entropy/zstd/testing.rs delete mode 100644 src/zstd/wip/work/fast/STATUS.md delete mode 100644 src/zstd/wip/work/fast/harness/Cargo.toml delete mode 100644 src/zstd/wip/work/fast/harness/build.rs delete mode 100644 src/zstd/wip/work/fast/harness/c/shim.c delete mode 100644 src/zstd/wip/work/fast/harness/c/shim_cctx.inc delete mode 100644 src/zstd/wip/work/fast/harness/rust-toolchain.toml delete mode 100644 src/zstd/wip/work/fast/harness/src/c.rs delete mode 100644 src/zstd/wip/work/fast/harness/src/data.rs delete mode 100644 src/zstd/wip/work/fast/harness/src/frames.rs delete mode 100644 src/zstd/wip/work/fast/harness/src/main.rs delete mode 100644 src/zstd/wip/work/fast/harness/src/seqs.rs delete mode 100644 src/zstd/wip/work/fast/harness/src/stream.rs delete mode 100644 src/zstd/wip/work/fast/zstd/lib.rs delete mode 100644 src/zstd/wip/work/fast/zstd/testing.rs delete mode 100644 src/zstd/wip/work/lazy/STATUS.md delete mode 100644 src/zstd/wip/work/lazy/harness/Cargo.toml delete mode 100644 src/zstd/wip/work/lazy/harness/build.rs delete mode 100644 src/zstd/wip/work/lazy/harness/rust-toolchain.toml delete mode 100644 src/zstd/wip/work/lazy/harness/shim.c delete mode 100644 src/zstd/wip/work/lazy/harness/src/c.rs delete mode 100644 src/zstd/wip/work/lazy/harness/src/data.rs delete mode 100644 src/zstd/wip/work/lazy/harness/src/main.rs delete mode 100644 src/zstd/wip/work/lazy/harness/src/seqs.rs delete mode 100644 src/zstd/wip/work/lazy/lazy.exp-filter.rs delete mode 100644 src/zstd/wip/work/lazy/lazy.exp-prefetch.rs delete mode 100644 src/zstd/wip/work/lazy/lazy.v1.rs delete mode 100644 src/zstd/wip/work/lazy/lazy.v2.rs delete mode 100644 src/zstd/wip/work/lazy/lib.orig.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/lib.rs delete mode 100644 src/zstd/wip/work/lazy/zstd/testing.rs delete mode 100644 src/zstd/wip/work/ldm/harness/Cargo.toml delete mode 100644 src/zstd/wip/work/ldm/harness/build.rs delete mode 100644 src/zstd/wip/work/ldm/harness/rust-toolchain.toml delete mode 100644 src/zstd/wip/work/ldm/harness/shim.c delete mode 100644 src/zstd/wip/work/ldm/harness/src/c.rs delete mode 100644 src/zstd/wip/work/ldm/harness/src/data.rs delete mode 100644 src/zstd/wip/work/ldm/harness/src/ldm.rs delete mode 100644 src/zstd/wip/work/ldm/harness/src/main.rs delete mode 100644 src/zstd/wip/work/ldm/harness/src/seqs.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/lib.rs delete mode 100644 src/zstd/wip/work/ldm/zstd/testing.rs delete mode 100644 src/zstd/wip/work/opt/STATUS.md delete mode 100644 src/zstd/wip/work/opt/harness/Cargo.toml delete mode 100644 src/zstd/wip/work/opt/harness/build.rs delete mode 100644 src/zstd/wip/work/opt/harness/rust-toolchain.toml delete mode 100644 src/zstd/wip/work/opt/harness/shim.c delete mode 100644 src/zstd/wip/work/opt/harness/src/c.rs delete mode 100644 src/zstd/wip/work/opt/harness/src/data.rs delete mode 100644 src/zstd/wip/work/opt/harness/src/main.rs delete mode 100644 src/zstd/wip/work/opt/harness/src/seqs.rs delete mode 100644 src/zstd/wip/work/opt/zstd/lib.rs delete mode 100644 src/zstd/wip/work/opt/zstd/testing.rs diff --git a/src/jsc/bindings/xxhash3.cpp b/src/jsc/bindings/xxhash3.cpp index f088916b24a8..490186bbc470 100644 --- a/src/jsc/bindings/xxhash3.cpp +++ b/src/jsc/bindings/xxhash3.cpp @@ -15,7 +15,7 @@ // verbatim; they do not depend on vector width. Verified against the xxHash // reference test vectors and SMHasher constants in test/js/bun/util/hash.test.js. // -// References (byte-identical constants): vendor/zstd/lib/common/xxhash.h +// References (byte-identical constants): xxhash.h of https://github.com/Cyan4973/xxHash // (XXH3_kSecret, PRIME*), and the twox-hash crate this replaces. // No "root.h" — like the sibling Highway TU image_resize.cpp, this file uses no diff --git a/src/runtime/node/zlib/NativeZstd.rs b/src/runtime/node/zlib/NativeZstd.rs index 924578bd52a7..24cb287685d0 100644 --- a/src/runtime/node/zlib/NativeZstd.rs +++ b/src/runtime/node/zlib/NativeZstd.rs @@ -92,8 +92,9 @@ mod _impl { /// Per-mode external-allocation footprint, fixed at construction. fn external_size_for(mode: NodeMode) -> usize { match mode { - NodeMode::ZSTD_COMPRESS => 5272, // estimate of ZSTD_sizeof_CCtx - NodeMode::ZSTD_DECOMPRESS => 95968, // estimate of ZSTD_sizeof_DCtx + // About what a new context takes, with the tables it allocates at once. + NodeMode::ZSTD_COMPRESS => 14 * 1024, + NodeMode::ZSTD_DECOMPRESS => 28 * 1024, _ => 0, } } diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index df7ed1a8085f..ec2b6eacdbbd 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -516,7 +516,7 @@ fn compress_insert_dictionary( /// `ZSTD_CDict`: a dictionary that is indexed, ready to start frames with. pub(crate) struct CDict { - dict_content: Arc<[u8]>, + dict_content: Arc>, match_state: MatchState, c_block_state: Box, dict_id: u32, @@ -525,7 +525,7 @@ pub(crate) struct CDict { impl CDict { /// `ZSTD_createCDict_advanced2` - pub(crate) fn new(dict: Arc<[u8]>, original_cctx_params: &CCtxParams) -> Result { + pub(crate) fn new(dict: Arc>, original_cctx_params: &CCtxParams) -> Result { let mut params = *original_cctx_params; params.c_params = get_c_params_from_cctx_params( ¶ms, diff --git a/src/zstd/compress/params.rs b/src/zstd/compress/params.rs index bc7a74097961..fd1d2b8d4af4 100644 --- a/src/zstd/compress/params.rs +++ b/src/zstd/compress/params.rs @@ -4,6 +4,7 @@ use super::internal::{ CompressionParameters, LdmParams, ParamSwitch, Strategy, ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, }; +use super::lazy::ZSTD_ROW_HASH_TAG_BITS; use super::{ldm, row_match_finder_supported}; use crate::common::{ LITBITS, MAX_LL, MAX_ML, MAX_OFF, WILDCOPY_OVERLENGTH, ZSTD_BLOCKSIZE_MAX, @@ -42,7 +43,6 @@ const ZSTD_TARGETCBLOCKSIZE_MIN: i32 = 1340; const ZSTD_TARGETCBLOCKSIZE_MAX: i32 = ZSTD_BLOCKSIZE_MAX as i32; const ZSTD_LDM_DEFAULT_WINDOW_LOG: u32 = ZSTD_WINDOWLOG_LIMIT_DEFAULT; pub(crate) const ZSTD_HASHLOG3_MAX: u32 = 17; -pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; pub(crate) const ZSTDMT_NBWORKERS_MAX: i32 = 256; pub(crate) const ZSTDMT_JOBSIZE_MIN: i32 = 512 << 10; diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs index e5c1cfe24b9c..f5bfecb8904d 100644 --- a/src/zstd/compress/stream.rs +++ b/src/zstd/compress/stream.rs @@ -48,7 +48,7 @@ pub struct CCtx { stream_stage: StreamStage, frame_ended: bool, /// `localDict.dict` - local_dict: Option>, + local_dict: Option>>, /// `localDict.cdict`, which is made when the first frame starts. cdict: Option>, mtctx: Option>, @@ -140,7 +140,7 @@ impl CCtx { copy.try_reserve_exact(dict.len()) .map_err(|_| ErrorCode::MemoryAllocation)?; copy.extend_from_slice(dict); - self.local_dict = Some(Arc::from(copy)); + self.local_dict = Some(Arc::new(copy)); Ok(()) } diff --git a/src/zstd/wip/PORTING.md b/src/zstd/wip/PORTING.md deleted file mode 100644 index 007ba3a706e6..000000000000 --- a/src/zstd/wip/PORTING.md +++ /dev/null @@ -1,75 +0,0 @@ -# Porting zstd 1.5.7 to safe Rust for Bun (`src/zstd`, crate `bun_zstd`) - -Goal: replace the C library with a port that gives **byte-for-byte the same compressed output, the same decompressed -output and the same error codes** as the C code, for every input. It is checked by differential tests against the C -library, so "equivalent but different" algorithms are wrong whenever they can change a single output byte. - -## Where things are - -- C source (pin f8745da6 = v1.5.7, with Bun's two patches): `/root/zstd-ref/zstd/lib/` -- The same without comments/asserts-noise, quicker to read: `/root/zstd-ref/stripped/` (read the original when a - comment would help to understand intent) -- The Rust crate: `/root/repos/oven-sh/bun/src/zstd/` (`common/`, `decompress/` are done and are the style reference; - `compress/internal.rs` is the port of `zstd_compress_internal.h` and is what the parts of the compressor share) -- Reference C static libs: `/root/zstd-ref/build/ref/libzstd.a` (as Bun built it), `/root/zstd-ref/build/noasm/` -- Differential harness (not part of the repo): `/root/zstd-ref/harness/` (`src/c.rs` has the FFI to the C lib). Build: - `. ~/.bun-dev-env && cd /root/zstd-ref/harness && RUSTFLAGS="--cap-lints=warn" cargo build --release` - It is built with overflow checks and debug assertions ON, on purpose. -- Type-check the crate: `. ~/.bun-dev-env && cd /root/repos/oven-sh/bun && cargo check -p bun_zstd` (dead-code errors - for items that nothing calls yet are expected while the port is incomplete; every other error is yours to fix). - Lints: `cargo clippy -p bun_zstd` must have no finding in your files other than dead code. - -## Hard rules - -1. `#![forbid(unsafe_code)]`. No `unsafe`, no `core::arch`, no `std::simd`, no new dependencies. -2. **No panics**: the crate denies `clippy::indexing_slicing`, `unwrap_used`, `expect_used`, `panic`, `unreachable`. - No `a[i]`, no `&a[i..j]`, no `.unwrap()`. Use `crate::util::SliceExt` (`at`, `at_mut`, `put`, `sub`, `sub_mut`, - `span`, `span_mut`, `array_at`, `array_at_mut`, `move_within`, `write_at`) and `read_le16/24/32/64`, - `write_le16/24/32/64`, all of which return `crate::error::Result` (`Err(ErrorCode::Generic)` = "broken - invariant, no input should cause it") and are used with `?`. Also avoid the std methods that panic - (`split_at`, `copy_from_slice` on lengths that are not provably equal, `copy_within`, `chunks(0)`, `/ 0`, ...). - Destructuring fixed arrays (`let [a, b, c] = arr;`), iterators, `zip`, `as_chunks`, `first_chunk`, `get` are good. - An index masked to a power-of-two array length (`arr.get(i & (N - 1))`) lets LLVM drop the check. -3. **No arithmetic overflow**: debug and fuzz builds have overflow checks, and a failed check aborts the process. - Wherever the C code relies on unsigned wrap-around (index arithmetic such as `curr - matchIndex`, `(U32)(a - b)`, - hash multiplications, `offset_1 > maxRep` tricks, `x - 1` on a possibly-0 value) use `wrapping_*`. Shifts by a - value that is not provably less than the width need a mask or `checked_sh*`. Casts: `as` truncates like C. - Keep the C integer widths (`U32` -> `u32`, `size_t` -> `usize`, `int` -> `i32`, `U64` -> `u64`, `BYTE` -> `u8`, - `S16`/`short` -> `i16`). All of Bun's targets are 64-bit little-endian, so drop `MEM_32bits()` branches and - big-endian branches. -4. Functions that return `size_t` with error codes return `Result`; `ERROR(x)` is `Err(ErrorCode::X)` - (`src/zstd/error.rs`). Keep the **same error code in the same order of checks**. Void C functions that need checked - accesses return `Result<()>`. -5. Pointers become positions (`usize`) into slices. Never keep a reference where C keeps a pointer into a buffer - that is mutated or that someone else owns: pass the slice down as a parameter. -6. Workspaces (`void* workSpace, size_t wkspSize`) become local arrays/structs or typed scratch structs. If a - `wkspSize` check can fail for some *input* (not just for a programming error) keep it, computed from the C sizes. -7. C "templates" (FORCE_INLINE_TEMPLATE functions specialised by constant arguments, macro-generated variants) become - `#[inline(always)]` generic functions with `const` generic parameters (or plain arguments that are constants at - the call sites), plus small non-inline wrappers/dispatch `match`es, so that each variant is compiled separately as - in C. -8. Drop what cannot be reached (listed per task), BMI2/ASM/NEON/SSE variants (keep the portable scalar code path that - gives the same result), tracing, DEBUGLOG, and `assert`s (a few `debug_assert!`s on real invariants are welcome). - PREFETCH is dropped. -9. Performance matters (this replaces optimised C): keep the structure of the hot loops, avoid allocation in them, - avoid bounds checks where a cheap restructuring (iterators, chunking, masks, hoisted sub-slices) removes them, mark - hot small helpers `#[inline(always)]`. But correctness and exact equivalence come first. - -## Style - -- Rust names in snake_case derived from the C names without the `ZSTD_`/`HUF_`/`FSE_`/`HIST_` prefix where the module - already says it (e.g. `HUF_buildCTable_wksp` -> `huf::build_ctable`). Every ported function gets a one-line doc - comment that starts with the C name in backticks: ``/// `HUF_buildCTable_wksp` `` plus a short sentence only if it - says something a reader needs (a precondition, what is returned). -- Items are `pub(crate)` at most (`unreachable_pub` is denied), private when only the module uses them. -- Comments: few, short, plain English, about the code as it is. Never mention the porting process, this - document, a conversation or "the original"; say "the C code" only when explaining a deliberate difference. - No commented-out code. No `TODO`. -- `rustfmt` (run `cargo fmt -p bun_zstd`), lines up to 120 columns in comments. `#[rustfmt::skip]` on big tables. -- Workspace lints are strict (see `[workspace.lints]` in `/root/repos/oven-sh/bun/Cargo.toml`), e.g. - `needless_pass_by_value`, `trivially_copy_pass_by_ref`, `large_types_passed_by_value`, `manual_memcpy`, - `redundant_clone`, `precedence`, `implicit_saturating_sub`, `manual_swap`; `too_many_arguments`, - `needless_range_loop` and `collapsible_if` are allowed. -- Do not edit files that are not yours (listed per task). If `compress/internal.rs`, `util.rs` or `common/` lacks - something you need, add a helper to your own file and say so in your report. -- Do not run `git commit`, `git add`, `git stash` or anything else that changes the repository's history or index. diff --git a/src/zstd/wip/build-ref.sh b/src/zstd/wip/build-ref.sh deleted file mode 100755 index 209061b63378..000000000000 --- a/src/zstd/wip/build-ref.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Builds the reference C zstd exactly as Bun's scripts/build/deps/zstd.ts did (pin f8745da6 + Bun's two patches). -set -e -cd /root/zstd-ref -SRC="common/debug common/entropy_common common/error_private common/fse_decompress common/pool common/threading common/xxhash common/zstd_common compress/fse_compress compress/hist compress/huf_compress compress/zstd_compress compress/zstd_compress_literals compress/zstd_compress_sequences compress/zstd_compress_superblock compress/zstd_double_fast compress/zstd_fast compress/zstd_lazy compress/zstd_ldm compress/zstd_opt compress/zstd_preSplit compress/zstdmt_compress decompress/huf_decompress decompress/zstd_ddict decompress/zstd_decompress decompress/zstd_decompress_block" -build() { # name, extra flags, asm? - local name=$1; shift; local asm=$1; shift - mkdir -p build/$name; local objs="" - for s in $SRC; do o=build/$name/$(basename $s).o; clang-23 -c -O2 -g -fPIC -march=nehalem -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=0 -DXXH_NAMESPACE=ZSTD_ -Izstd/lib -Izstd/lib/common "$@" zstd/lib/$s.c -o $o & objs="$objs $o"; done - if [ $asm = 1 ]; then clang-23 -c -fPIC zstd/lib/decompress/huf_decompress_amd64.S -o build/$name/huf_asm.o; objs="$objs build/$name/huf_asm.o"; fi - wait - rm -f build/$name/libzstd.a; llvm-ar-23 rcs build/$name/libzstd.a $objs -} -build ref 1 -build noasm 0 -DZSTD_DISABLE_ASM=1 -DDYNAMIC_BMI2=0 diff --git a/src/zstd/wip/clevels.rs b/src/zstd/wip/clevels.rs deleted file mode 100644 index f3c9b55dedaa..000000000000 --- a/src/zstd/wip/clevels.rs +++ /dev/null @@ -1,100 +0,0 @@ - [ - cp(19, 12, 13, 1, 6, 1, Strategy::Fast), - cp(19, 13, 14, 1, 7, 0, Strategy::Fast), - cp(20, 15, 16, 1, 6, 0, Strategy::Fast), - cp(21, 16, 17, 1, 5, 0, Strategy::DFast), - cp(21, 18, 18, 1, 5, 0, Strategy::DFast), - cp(21, 18, 19, 3, 5, 2, Strategy::Greedy), - cp(21, 18, 19, 3, 5, 4, Strategy::Lazy), - cp(21, 19, 20, 4, 5, 8, Strategy::Lazy), - cp(21, 19, 20, 4, 5, 16, Strategy::Lazy2), - cp(22, 20, 21, 4, 5, 16, Strategy::Lazy2), - cp(22, 21, 22, 5, 5, 16, Strategy::Lazy2), - cp(22, 21, 22, 6, 5, 16, Strategy::Lazy2), - cp(22, 22, 23, 6, 5, 32, Strategy::Lazy2), - cp(22, 22, 22, 4, 5, 32, Strategy::BtLazy2), - cp(22, 22, 23, 5, 5, 32, Strategy::BtLazy2), - cp(22, 23, 23, 6, 5, 32, Strategy::BtLazy2), - cp(22, 22, 22, 5, 5, 48, Strategy::BtOpt), - cp(23, 23, 22, 5, 4, 64, Strategy::BtOpt), - cp(23, 23, 22, 6, 3, 64, Strategy::BtUltra), - cp(23, 24, 22, 7, 3, 256, Strategy::BtUltra2), - cp(25, 25, 23, 7, 3, 256, Strategy::BtUltra2), - cp(26, 26, 24, 7, 3, 512, Strategy::BtUltra2), - cp(27, 27, 25, 9, 3, 999, Strategy::BtUltra2), - ], - [ - cp(18, 12, 13, 1, 5, 1, Strategy::Fast), - cp(18, 13, 14, 1, 6, 0, Strategy::Fast), - cp(18, 14, 14, 1, 5, 0, Strategy::DFast), - cp(18, 16, 16, 1, 4, 0, Strategy::DFast), - cp(18, 16, 17, 3, 5, 2, Strategy::Greedy), - cp(18, 17, 18, 5, 5, 2, Strategy::Greedy), - cp(18, 18, 19, 3, 5, 4, Strategy::Lazy), - cp(18, 18, 19, 4, 4, 4, Strategy::Lazy), - cp(18, 18, 19, 4, 4, 8, Strategy::Lazy2), - cp(18, 18, 19, 5, 4, 8, Strategy::Lazy2), - cp(18, 18, 19, 6, 4, 8, Strategy::Lazy2), - cp(18, 18, 19, 5, 4, 12, Strategy::BtLazy2), - cp(18, 19, 19, 7, 4, 12, Strategy::BtLazy2), - cp(18, 18, 19, 4, 4, 16, Strategy::BtOpt), - cp(18, 18, 19, 4, 3, 32, Strategy::BtOpt), - cp(18, 18, 19, 6, 3, 128, Strategy::BtOpt), - cp(18, 19, 19, 6, 3, 128, Strategy::BtUltra), - cp(18, 19, 19, 8, 3, 256, Strategy::BtUltra), - cp(18, 19, 19, 6, 3, 128, Strategy::BtUltra2), - cp(18, 19, 19, 8, 3, 256, Strategy::BtUltra2), - cp(18, 19, 19, 10, 3, 512, Strategy::BtUltra2), - cp(18, 19, 19, 12, 3, 512, Strategy::BtUltra2), - cp(18, 19, 19, 13, 3, 999, Strategy::BtUltra2), - ], - [ - cp(17, 12, 12, 1, 5, 1, Strategy::Fast), - cp(17, 12, 13, 1, 6, 0, Strategy::Fast), - cp(17, 13, 15, 1, 5, 0, Strategy::Fast), - cp(17, 15, 16, 2, 5, 0, Strategy::DFast), - cp(17, 17, 17, 2, 4, 0, Strategy::DFast), - cp(17, 16, 17, 3, 4, 2, Strategy::Greedy), - cp(17, 16, 17, 3, 4, 4, Strategy::Lazy), - cp(17, 16, 17, 3, 4, 8, Strategy::Lazy2), - cp(17, 16, 17, 4, 4, 8, Strategy::Lazy2), - cp(17, 16, 17, 5, 4, 8, Strategy::Lazy2), - cp(17, 16, 17, 6, 4, 8, Strategy::Lazy2), - cp(17, 17, 17, 5, 4, 8, Strategy::BtLazy2), - cp(17, 18, 17, 7, 4, 12, Strategy::BtLazy2), - cp(17, 18, 17, 3, 4, 12, Strategy::BtOpt), - cp(17, 18, 17, 4, 3, 32, Strategy::BtOpt), - cp(17, 18, 17, 6, 3, 256, Strategy::BtOpt), - cp(17, 18, 17, 6, 3, 128, Strategy::BtUltra), - cp(17, 18, 17, 8, 3, 256, Strategy::BtUltra), - cp(17, 18, 17, 10, 3, 512, Strategy::BtUltra), - cp(17, 18, 17, 5, 3, 256, Strategy::BtUltra2), - cp(17, 18, 17, 7, 3, 512, Strategy::BtUltra2), - cp(17, 18, 17, 9, 3, 512, Strategy::BtUltra2), - cp(17, 18, 17, 11, 3, 999, Strategy::BtUltra2), - ], - [ - cp(14, 12, 13, 1, 5, 1, Strategy::Fast), - cp(14, 14, 15, 1, 5, 0, Strategy::Fast), - cp(14, 14, 15, 1, 4, 0, Strategy::Fast), - cp(14, 14, 15, 2, 4, 0, Strategy::DFast), - cp(14, 14, 14, 4, 4, 2, Strategy::Greedy), - cp(14, 14, 14, 3, 4, 4, Strategy::Lazy), - cp(14, 14, 14, 4, 4, 8, Strategy::Lazy2), - cp(14, 14, 14, 6, 4, 8, Strategy::Lazy2), - cp(14, 14, 14, 8, 4, 8, Strategy::Lazy2), - cp(14, 15, 14, 5, 4, 8, Strategy::BtLazy2), - cp(14, 15, 14, 9, 4, 8, Strategy::BtLazy2), - cp(14, 15, 14, 3, 4, 12, Strategy::BtOpt), - cp(14, 15, 14, 4, 3, 24, Strategy::BtOpt), - cp(14, 15, 14, 5, 3, 32, Strategy::BtUltra), - cp(14, 15, 15, 6, 3, 64, Strategy::BtUltra), - cp(14, 15, 15, 7, 3, 256, Strategy::BtUltra), - cp(14, 15, 15, 5, 3, 48, Strategy::BtUltra2), - cp(14, 15, 15, 6, 3, 128, Strategy::BtUltra2), - cp(14, 15, 15, 7, 3, 256, Strategy::BtUltra2), - cp(14, 15, 15, 8, 3, 256, Strategy::BtUltra2), - cp(14, 15, 15, 8, 3, 512, Strategy::BtUltra2), - cp(14, 15, 15, 9, 3, 512, Strategy::BtUltra2), - cp(14, 15, 15, 10, 3, 999, Strategy::BtUltra2), - ], diff --git a/src/zstd/wip/default_tables.rs b/src/zstd/wip/default_tables.rs deleted file mode 100644 index 5237a892d9ba..000000000000 --- a/src/zstd/wip/default_tables.rs +++ /dev/null @@ -1,51 +0,0 @@ -#[rustfmt::skip] -const LL_DEFAULT: [SeqSymbol; 64] = [ - s( 0, 0, 4, 0), s(16, 0, 4, 0), s(32, 0, 5, 1), s( 0, 0, 5, 3), - s( 0, 0, 5, 4), s( 0, 0, 5, 6), s( 0, 0, 5, 7), s( 0, 0, 5, 9), - s( 0, 0, 5, 10), s( 0, 0, 5, 12), s( 0, 0, 6, 14), s( 0, 1, 5, 16), - s( 0, 1, 5, 20), s( 0, 1, 5, 22), s( 0, 2, 5, 28), s( 0, 3, 5, 32), - s( 0, 4, 5, 48), s(32, 6, 5, 64), s( 0, 7, 5, 128), s( 0, 8, 6, 256), - s( 0, 10, 6, 1024), s( 0, 12, 6, 4096), s(32, 0, 4, 0), s( 0, 0, 4, 1), - s( 0, 0, 5, 2), s(32, 0, 5, 4), s( 0, 0, 5, 5), s(32, 0, 5, 7), - s( 0, 0, 5, 8), s(32, 0, 5, 10), s( 0, 0, 5, 11), s( 0, 0, 6, 13), - s(32, 1, 5, 16), s( 0, 1, 5, 18), s(32, 1, 5, 22), s( 0, 2, 5, 24), - s(32, 3, 5, 32), s( 0, 3, 5, 40), s( 0, 6, 4, 64), s(16, 6, 4, 64), - s(32, 7, 5, 128), s( 0, 9, 6, 512), s( 0, 11, 6, 2048), s(48, 0, 4, 0), - s(16, 0, 4, 1), s(32, 0, 5, 2), s(32, 0, 5, 3), s(32, 0, 5, 5), - s(32, 0, 5, 6), s(32, 0, 5, 8), s(32, 0, 5, 9), s(32, 0, 5, 11), - s(32, 0, 5, 12), s( 0, 0, 6, 15), s(32, 1, 5, 18), s(32, 1, 5, 20), - s(32, 2, 5, 24), s(32, 2, 5, 28), s(32, 3, 5, 40), s(32, 4, 5, 48), - s( 0, 16, 6, 65536), s( 0, 15, 6, 32768), s( 0, 14, 6, 16384), s( 0, 13, 6, 8192), -]; - -#[rustfmt::skip] -const OF_DEFAULT: [SeqSymbol; 32] = [ - s( 0, 0, 5, 0), s( 0, 6, 4, 61), s( 0, 9, 5, 509), s( 0, 15, 5, 32765), - s( 0, 21, 5, 2097149), s( 0, 3, 5, 5), s( 0, 7, 4, 125), s( 0, 12, 5, 4093), - s( 0, 18, 5, 262141), s( 0, 23, 5, 8388605), s( 0, 5, 5, 29), s( 0, 8, 4, 253), - s( 0, 14, 5, 16381), s( 0, 20, 5, 1048573), s( 0, 2, 5, 1), s(16, 7, 4, 125), - s( 0, 11, 5, 2045), s( 0, 17, 5, 131069), s( 0, 22, 5, 4194301), s( 0, 4, 5, 13), - s(16, 8, 4, 253), s( 0, 13, 5, 8189), s( 0, 19, 5, 524285), s( 0, 1, 5, 1), - s(16, 6, 4, 61), s( 0, 10, 5, 1021), s( 0, 16, 5, 65533), s( 0, 28, 5, 268435453), - s( 0, 27, 5, 134217725), s( 0, 26, 5, 67108861), s( 0, 25, 5, 33554429), s( 0, 24, 5, 16777213), -]; - -#[rustfmt::skip] -const ML_DEFAULT: [SeqSymbol; 64] = [ - s( 0, 0, 6, 3), s( 0, 0, 4, 4), s(32, 0, 5, 5), s( 0, 0, 5, 6), - s( 0, 0, 5, 8), s( 0, 0, 5, 9), s( 0, 0, 5, 11), s( 0, 0, 6, 13), - s( 0, 0, 6, 16), s( 0, 0, 6, 19), s( 0, 0, 6, 22), s( 0, 0, 6, 25), - s( 0, 0, 6, 28), s( 0, 0, 6, 31), s( 0, 0, 6, 34), s( 0, 1, 6, 37), - s( 0, 1, 6, 41), s( 0, 2, 6, 47), s( 0, 3, 6, 59), s( 0, 4, 6, 83), - s( 0, 7, 6, 131), s( 0, 9, 6, 515), s(16, 0, 4, 4), s( 0, 0, 4, 5), - s(32, 0, 5, 6), s( 0, 0, 5, 7), s(32, 0, 5, 9), s( 0, 0, 5, 10), - s( 0, 0, 6, 12), s( 0, 0, 6, 15), s( 0, 0, 6, 18), s( 0, 0, 6, 21), - s( 0, 0, 6, 24), s( 0, 0, 6, 27), s( 0, 0, 6, 30), s( 0, 0, 6, 33), - s( 0, 1, 6, 35), s( 0, 1, 6, 39), s( 0, 2, 6, 43), s( 0, 3, 6, 51), - s( 0, 4, 6, 67), s( 0, 5, 6, 99), s( 0, 8, 6, 259), s(32, 0, 4, 4), - s(48, 0, 4, 4), s(16, 0, 4, 5), s(32, 0, 5, 7), s(32, 0, 5, 8), - s(32, 0, 5, 10), s(32, 0, 5, 11), s( 0, 0, 6, 14), s( 0, 0, 6, 17), - s( 0, 0, 6, 20), s( 0, 0, 6, 23), s( 0, 0, 6, 26), s( 0, 0, 6, 29), - s( 0, 0, 6, 32), s( 0, 16, 6, 65539), s( 0, 15, 6, 32771), s( 0, 14, 6, 16387), - s( 0, 13, 6, 8195), s( 0, 12, 6, 4099), s( 0, 11, 6, 2051), s( 0, 10, 6, 1027), -]; diff --git a/src/zstd/wip/harness/Cargo.toml b/src/zstd/wip/harness/Cargo.toml deleted file mode 100644 index e9cca9310c3e..000000000000 --- a/src/zstd/wip/harness/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "zstd_harness" -version = "0.0.0" -edition = "2024" - -[dependencies] -bun_zstd = { path = "/root/repos/oven-sh/bun/src/zstd" } - -[profile.release] -debug = "line-tables-only" -overflow-checks = true -debug-assertions = true -panic = "abort" - -[profile.bench] -inherits = "release" -overflow-checks = false -debug-assertions = false - -[workspace] diff --git a/src/zstd/wip/harness/build.rs b/src/zstd/wip/harness/build.rs deleted file mode 100644 index e0b8fe9e7a87..000000000000 --- a/src/zstd/wip/harness/build.rs +++ /dev/null @@ -1,6 +0,0 @@ -fn main() { - let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); - println!("cargo:rerun-if-env-changed=ZSTD_REF"); - println!("cargo:rustc-link-search=native=/root/zstd-ref/build/{variant}"); - println!("cargo:rustc-link-lib=static=zstd"); -} diff --git a/src/zstd/wip/harness/rust-toolchain.toml b/src/zstd/wip/harness/rust-toolchain.toml deleted file mode 100644 index c71778e8ccd3..000000000000 --- a/src/zstd/wip/harness/rust-toolchain.toml +++ /dev/null @@ -1,35 +0,0 @@ -[toolchain] -channel = "nightly-2026-09-15" -# rust-src is needed for -Zbuild-std (Tier 3 targets like -# aarch64-unknown-freebsd have no prebuilt std). miri is for -# `bun run rust:miri`. targets ensures the -# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain -# (CI's images install this same channel with these components and -# targets, from scripts/build/ci-images/spec.ts; a source lint keeps -# the two files in step). Note: rustup ignores this whole file when -# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree -# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` -# self-heals that path. Keep this list as a fallback for direct -# `cargo build`/`cargo check`/rust-analyzer use. -# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd -# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). -# darwin/windows triples build natively on their own agents, so the host std -# comes with the toolchain install regardless; they're listed here so a dev -# running `bun run rust:check-all` or `cargo check --target` from any host -# still gets prebuilt std for every Tier 1/2 triple. -# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at -# what it emits. The build does not use them. -components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] -targets = [ - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "x86_64-unknown-linux-musl", - "aarch64-linux-android", - "x86_64-linux-android", - "x86_64-unknown-freebsd", - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-pc-windows-msvc", - "x86_64-pc-windows-msvc", -] diff --git a/src/zstd/wip/harness/src/c.rs b/src/zstd/wip/harness/src/c.rs deleted file mode 100644 index d5a82e1f6b78..000000000000 --- a/src/zstd/wip/harness/src/c.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! The reference C library. -#![allow(non_camel_case_types, dead_code)] -use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; - -#[repr(C)] -pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } -#[repr(C)] -pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } -pub enum ZSTD_CCtx {} -pub enum ZSTD_DCtx {} -pub enum ZSTD_DDict {} - -unsafe extern "C" { - pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; - pub fn ZSTD_compressBound(size: usize) -> usize; - pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_isError(code: usize) -> c_uint; - pub fn ZSTD_getErrorCode(code: usize) -> c_uint; - pub fn ZSTD_getErrorName(code: usize) -> *const c_char; - pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; - pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; - pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; - pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; - pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; - pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; - pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; - pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; - pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; - pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; - pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; - pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; - pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; - pub fn ZSTD_defaultCLevel() -> c_int; - pub fn ZSTD_minCLevel() -> c_int; - pub fn ZSTD_maxCLevel() -> c_int; -} - -/// `Ok(value)` or `Err(error code)`. -pub fn res(code: usize) -> Result { - unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } -} - -pub struct CCtx(pub *mut ZSTD_CCtx); -impl CCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } - pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - /// Returns (result, out pos, in pos). - pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); - (r, o.pos, i.pos) - } -} -impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } - -pub struct DCtx(pub *mut ZSTD_DCtx); -impl DCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } - pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } - pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); - (r, o.pos, i.pos) - } -} -impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } - -pub fn compress(src: &[u8], level: i32) -> Vec { - let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; - let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); - dst.truncate(n); - dst -} -pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) -} diff --git a/src/zstd/wip/harness/src/comp.rs b/src/zstd/wip/harness/src/comp.rs deleted file mode 100644 index 2f86d9854aff..000000000000 --- a/src/zstd/wip/harness/src/comp.rs +++ /dev/null @@ -1,296 +0,0 @@ -//! The compressor against the C one: same calls, same results, same bytes, same positions. -use crate::c; -use crate::data::{Rng, data}; -use bun_zstd as z; - -fn r2(r: Result) -> Result { r.map_err(|e| e as u32) } -fn r0(r: Result<(), z::ErrorCode>) -> Result { r.map(|()| 0).map_err(|e| e as u32) } - -/// A dictionary: raw content, or in zstd's format (made from a real one by the C trainer is not available, so entropy -/// tables are taken from a frame... here: raw, or magic + garbage to exercise the error path). -fn dictionary(rng: &mut Rng, base: &[u8]) -> Vec { - let len = rng.size(1 << 17); - let mut d = if rng.chance(1, 2) && !base.is_empty() { - let start = rng.below(base.len()); - let end = (start + len).min(base.len()); - base[start..end].to_vec() - } else { - data(rng, len) - }; - if rng.chance(1, 12) && d.len() >= 8 { - d[..4].copy_from_slice(&0xEC30A437u32.to_le_bytes()); - } - d -} - -fn level(rng: &mut Rng) -> i32 { - match rng.below(10) { - 0 => -(rng.range(1, 50) as i32), - 1 => rng.range(0, 30) as i32 - 4, - 2 => -(rng.size(200000) as i32), - _ => rng.range(1, 22) as i32, - } -} - -pub fn one_shot(rng: &mut Rng, max: usize) { - let len = rng.size(max); - let src = data(rng, len); - let lvl = level(rng); - let bound = unsafe { c::ZSTD_compressBound(len) }; - assert_eq!(bound, z::compress_bound(len)); - let cap = match rng.below(8) { 0 => rng.below(bound + 1), 1 => bound.saturating_sub(rng.below(20)), _ => bound + rng.below(50) }; - let mut a = vec![0u8; cap]; - let mut b = vec![0u8; cap]; - let ra = c::res(unsafe { c::ZSTD_compress(a.as_mut_ptr().cast(), cap, src.as_ptr().cast(), len, lvl) }); - let rb = r2(z::compress(&mut b, &src, lvl)); - assert_eq!(ra, rb, "one shot: len {len} level {lvl} cap {cap}"); - if let Ok(n) = ra { - if a[..n] != b[..n] { - let at = a[..n].iter().zip(&b[..n]).position(|(x, y)| x != y); - panic!("one shot bytes differ at {at:?}/{n}: len {len} level {lvl}"); - } - } -} - -const PARAMS: &[(i32, i32, i32)] = &[ - (10, 0, 1), (100, -50, 22), (101, 10, 27), (102, 6, 26), (103, 6, 26), (104, 1, 12), (105, 3, 7), (106, 0, 4096), - (107, 1, 9), (130, 0, 20000), (160, 0, 2), (161, 6, 24), (162, 4, 4096), (163, 1, 8), (164, 0, 12), - (200, 0, 1), (201, 0, 1), (202, 0, 1), -]; - -fn set_params(rng: &mut Rng, a: &mut c::CCtx, b: &mut z::CCtx, log: &mut Vec, mt: bool) { - let n = match rng.below(4) { 0 => 0, 1 => 1, 2 => rng.below(4), _ => rng.below(9) }; - for _ in 0..n { - let (p, v) = if rng.chance(1, 25) { - (rng.below(403) as i32, rng.range(0, 40) as i32 - 5) - } else { - let &(p, lo, hi) = rng.pick(PARAMS); - let v = if rng.chance(1, 20) { rng.range(0, 3000000) as i32 - 200000 } else { lo + rng.below((hi - lo + 1) as usize) as i32 }; - (p, v) - }; - if !mt && (400..=402).contains(&p) { continue; } - let ra = a.set(p as u32, v); - let rb = r2(b.set_parameter(p, v)); - log.push(format!("set {p} {v} -> {ra:?}")); - assert_eq!(ra, rb, "{log:?}"); - } -} - -/// One context, several frames, random calls. -pub fn stream(rng: &mut Rng, max: usize) { - let mut a = c::CCtx::new(); - let mut b = z::CCtx::new(); - let mut log: Vec = Vec::new(); - let frames = if rng.chance(1, 3) { rng.range(2, 5) } else { 1 }; - for _ in 0..frames { - if rng.chance(1, 6) { - let r = *rng.pick(&[1u32, 2, 3]); - let ra = a.reset(r); - let rb = r0(b.reset(match r { 1 => z::ResetDirective::SessionOnly, 2 => z::ResetDirective::Parameters, _ => z::ResetDirective::SessionAndParameters })); - log.push(format!("reset {r} -> {ra:?}")); - assert_eq!(ra, rb, "{log:?}"); - } - set_params(rng, &mut a, &mut b, &mut log, false); - let len = rng.size(max); - let src = data(rng, len); - if rng.chance(1, 4) && std::env::var_os("NODICT").is_none() { - let d = if rng.chance(1, 10) { Vec::new() } else { dictionary(rng, &src) }; - let ra = a.load_dict(&d); - let rb = r0(b.load_dictionary(&d)); - log.push(format!("dict {} -> {ra:?}", d.len())); - assert_eq!(ra, rb, "{log:?}"); - } - if rng.chance(1, 4) { - let n = if rng.chance(1, 8) { rng.size(max) as u64 } else { len as u64 }; - let ra = a.pledge(n); - let rb = r0(b.set_pledged_src_size(n)); - log.push(format!("pledge {n} (len {len}) -> {ra:?}")); - assert_eq!(ra, rb, "{log:?}"); - } - log.push(format!("frame len {len}")); - // how the input and the output are cut - let in_mode = rng.below(4); - let out_mode = rng.below(4); - let small = len < 5000; - let mut spos_total = 0usize; - let mut calls = 0usize; - let mut failed = false; - loop { - calls += 1; - let left = len - spos_total; - let chunk = match in_mode { - 0 => left, - 1 => rng.range(0, if small { 64 } else { 100_000 }).min(left), - 2 => rng.size(1 << 19).min(left), - _ => if rng.chance(1, 2) { left } else { rng.below(left + 1) }, - }; - let cap = match out_mode { - 0 => z::compress_bound(chunk) + rng.below(100), - 1 => rng.range(0, if small { 40 } else { 50_000 }), - 2 => rng.size(1 << 18), - _ => 1 << 17, - }; - let last = spos_total + chunk == len; - let end = if last && rng.chance(3, 4) { 2 } else { *rng.pick(&[0u32, 0, 0, 1, 1, 2]) }; - let end = if calls > 4000 { 2 } else { end }; - let input = &src[spos_total..spos_total + chunk]; - let mut oa = vec![0u8; cap]; - let mut ob = vec![0u8; cap]; - let (ra, opa, ipa) = a.stream2(&mut oa, 0, input, 0, end); - let (rb, opb, ipb) = { - let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; - let mut i = z::InBuffer { src: input, pos: 0 }; - let r = r2(b.compress_stream2(&mut o, &mut i, match end { 0 => z::EndDirective::Continue, 1 => z::EndDirective::Flush, _ => z::EndDirective::End })); - (r, o.pos, i.pos) - }; - if (ra, opa, ipa) != (rb, opb, ipb) || oa[..opa] != ob[..opb] { - let at = oa.iter().zip(&ob).position(|(x, y)| x != y); - panic!("stream: call {calls} chunk {chunk} cap {cap} end {end}: C {:?} Rust {:?} first diff {at:?}\n{log:?}", (ra, opa, ipa), (rb, opb, ipb)); - } - spos_total += ipa; - if ra.is_err() { failed = true; break; } - if end == 2 && ra == Ok(0) { - if spos_total == len { break; } - log.push(format!("frame ended at {spos_total}")); - } - if calls > 6000 { panic!("no end {log:?}"); } - } - if failed { - log.push("failed".into()); - let ra = a.reset(1); - let rb = r0(b.reset(z::ResetDirective::SessionOnly)); - assert_eq!(ra, rb); - } - } -} - -pub fn run(seed: u64, iters: u64, max: usize) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if std::env::var_os("TRACE").is_some() { eprintln!("iter {i} seed {s}"); } - if rng.chance(1, 3) { one_shot(&mut rng, max) } else { stream(&mut rng, max) } - } - println!("comp ok: seed {seed}, {iters} iterations"); -} - -/// Multithreaded: when output comes out depends on timing, so only the whole output is compared. -pub fn mt(rng: &mut Rng, max: usize) { - let mut a = c::CCtx::new(); - let mut b = z::CCtx::new(); - let mut log: Vec = Vec::new(); - let frames = if rng.chance(1, 3) { rng.range(2, 3) } else { 1 }; - for _ in 0..frames { - let mut set = |p: i32, v: i32, log: &mut Vec| { - let ra = a.set(p as u32, v); - let rb = r2(b.set_parameter(p, v)); - log.push(format!("set {p} {v} -> {ra:?}")); - assert_eq!(ra, rb, "{log:?}"); - }; - set(400, rng.range(1, 4) as i32, &mut log); - if rng.chance(1, 2) { set(401, rng.range(0, 3 << 20) as i32, &mut log); } - if rng.chance(1, 2) { set(402, rng.range(0, 9) as i32, &mut log); } - set(100, if rng.chance(1, 6) { rng.range(13, 19) as i32 } else { rng.range(1, 12) as i32 - 3 }, &mut log); - if rng.chance(1, 2) { set(101, rng.range(10, 22) as i32, &mut log); } - if rng.chance(1, 3) { set(160, 1, &mut log); if rng.chance(1, 2) { set(162, rng.range(4, 200) as i32, &mut log); } if rng.chance(1,2) { set(161, rng.range(6, 20) as i32, &mut log); } } - if rng.chance(1, 2) { set(201, 1, &mut log); } - if rng.chance(1, 8) { set(130, rng.range(1340, 60000) as i32, &mut log); } - let len = if rng.chance(1, 4) { rng.size(max) } else { rng.range(max / 4, max) }; - let src = data(rng, len); - if rng.chance(1, 5) { - let d = dictionary(rng, &src); - let ra = a.load_dict(&d); - let rb = r0(b.load_dictionary(&d)); - log.push(format!("dict {} -> {ra:?}", d.len())); - assert_eq!(ra, rb, "{log:?}"); - } - if rng.chance(1, 4) { - let ra = a.pledge(len as u64); - let rb = r0(b.set_pledged_src_size(len as u64)); - log.push(format!("pledge {len}")); - assert_eq!(ra, rb, "{log:?}"); - } - log.push(format!("frame len {len}")); - let mut out_a: Vec = Vec::new(); - let mut out_b: Vec = Vec::new(); - let mut pos = 0usize; - let in_mode = rng.below(3); - loop { - let left = len - pos; - let chunk = match in_mode { 0 => left, 1 => rng.size(1 << 21).min(left), _ => rng.range(0, 300_000).min(left) }; - let last = pos + chunk == len; - let end = if last { 2u32 } else { *rng.pick(&[0u32, 0, 0, 0, 1]) }; - let input = &src[pos..pos + chunk]; - let cap = z::compress_bound(len.max(1 << 20)) + 1024; - let mut buf = vec![0u8; cap]; - let mut err_a = None; - let mut err_b = None; - let mut ip = 0usize; - loop { - let (r, op, np) = a.stream2(&mut buf, 0, input, ip, end); - out_a.extend_from_slice(&buf[..op]); - ip = np; - match r { Err(e) => { err_a = Some(e); break; } Ok(left) => if (end == 0 && ip == chunk) || (end != 0 && left == 0) { break; } } - } - let mut ip = 0usize; - loop { - let mut o = z::OutBuffer { dst: &mut buf, pos: 0 }; - let mut i = z::InBuffer { src: input, pos: ip }; - let r = r2(b.compress_stream2(&mut o, &mut i, match end { 0 => z::EndDirective::Continue, 1 => z::EndDirective::Flush, _ => z::EndDirective::End })); - let op = o.pos; - ip = i.pos; - out_b.extend_from_slice(&buf[..op]); - match r { Err(e) => { err_b = Some(e); break; } Ok(left) => if (end == 0 && ip == chunk) || (end != 0 && left == 0) { break; } } - } - assert_eq!(err_a, err_b, "{log:?}"); - log.push(format!("chunk {chunk} end {end}")); - if err_a.is_some() { let _ = a.reset(1); let _ = b.reset(z::ResetDirective::SessionOnly); break; } - pos += chunk; - if last { break; } - } - if out_a != out_b { - let at = out_a.iter().zip(&out_b).position(|(x, y)| x != y); - panic!("mt: {} vs {} bytes, first diff {at:?}\n{log:?}", out_a.len(), out_b.len()); - } - } -} - -pub fn run_mt(seed: u64, iters: u64, max: usize) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if std::env::var_os("TRACE").is_some() { eprintln!("iter {i} seed {s}"); } - mt(&mut rng, max); - } - println!("mt ok: seed {seed}, {iters} iterations"); -} - -/// Dictionaries, by strategy. -pub fn run_dict(seed: u64, iters: u64, max: usize) { - let mut fails = std::collections::BTreeMap::new(); - for i in 0..iters { - let mut rng = Rng(seed.wrapping_mul(1_000_003).wrapping_add(i)); - let strat = rng.range(1, 9) as i32; - let len = rng.size(max); - let src = data(&mut rng, len); - let d = dictionary(&mut rng, &src); - let pledge = rng.chance(1, 2); - let mut a = c::CCtx::new(); - let mut b = z::CCtx::new(); - a.set(107, strat).unwrap(); b.set_parameter(107, strat).unwrap(); - a.load_dict(&d).unwrap(); b.load_dictionary(&d).unwrap(); - let cap = z::compress_bound(len) + 64; - let (mut oa, mut ob) = (vec![0u8; cap], vec![0u8; cap]); - let end = if pledge { 2 } else { 1 }; - let (ra, pa, _) = a.stream2(&mut oa, 0, &src, 0, end); - let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; - let mut inp = z::InBuffer { src: &src, pos: 0 }; - let rb = r2(b.compress_stream2(&mut o, &mut inp, if pledge { z::EndDirective::End } else { z::EndDirective::Flush })); - let pb = o.pos; - let ok = ra == rb && pa == pb && oa[..pa] == ob[..pb]; - let e = fails.entry((strat, pledge)).or_insert((0u32, 0u32)); - e.0 += 1; - if !ok { e.1 += 1; if e.1 == 1 { eprintln!("first fail strat {strat} pledge {pledge} len {len} dict {} iter {i}: {ra:?} {rb:?} {pa} {pb}", d.len()); } } - } - for (k, v) in fails { println!("{k:?}: {} of {} differ", v.1, v.0); } -} diff --git a/src/zstd/wip/harness/src/data.rs b/src/zstd/wip/harness/src/data.rs deleted file mode 100644 index c48a645e1b5d..000000000000 --- a/src/zstd/wip/harness/src/data.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Deterministic inputs. -pub struct Rng(pub u64); -impl Rng { - pub fn next(&mut self) -> u64 { - // splitmix64 - self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); - z ^ (z >> 31) - } - pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } - pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } - pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } - pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } - /// Skewed towards small values. - pub fn size(&mut self, max: usize) -> usize { - let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); - (self.next() as usize & ((1usize << bits) - 1)).min(max) - } -} - -static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; - -/// One buffer of about `len` bytes, of a kind picked by `rng`. -pub fn data(rng: &mut Rng, len: usize) -> Vec { - let mut v = Vec::with_capacity(len + 64); - match rng.below(9) { - 0 => v.resize(len, rng.next() as u8), - 1 => while v.len() < len { v.push(rng.next() as u8) }, - 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, - 3 => { // few symbols, skewed - let n = rng.range(2, 40); - while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } - } - 4 => { // LZ-like: copies of earlier parts, at all distances - while v.len() < len { - if v.len() > 8 && rng.chance(3, 4) { - let dist = 1 + rng.size(v.len() - 1); - let n = 3 + rng.size(300); - for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } - } else { - for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } - } - } - } - 5 => { // periodic with noise - let period = rng.range(1, 300); - let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); - while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } - } - 6 => { // sections of different kinds - while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } - } - 7 => { // numbers as text - let mut x = rng.next() % 1000; - while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } - } - _ => { // long runs - while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } - } - } - v.truncate(len); - v -} diff --git a/src/zstd/wip/harness/src/dec.rs b/src/zstd/wip/harness/src/dec.rs deleted file mode 100644 index d117a936bbf6..000000000000 --- a/src/zstd/wip/harness/src/dec.rs +++ /dev/null @@ -1,184 +0,0 @@ -//! The decoder against the C decoder: valid frames, damaged frames, one call and streaming. -use crate::c; -use crate::data::{Rng, data}; -use bun_zstd as z; - -fn rs(r: Result) -> Result { r.map_err(|e| e as u32) } - -/// A frame (or several) made by the C compressor with parameters picked by `rng`. Returns (frames, content, dictionary). -pub fn make_frames(rng: &mut Rng) -> (Vec, Vec, Vec) { - let mut out = Vec::new(); - let mut content = Vec::new(); - let dict = if rng.chance(1, 4) { let n = rng.size(200_000); data(rng, n) } else { Vec::new() }; - let nframes = if rng.chance(1, 5) { rng.range(2, 4) } else { 1 }; - for _ in 0..nframes { - if rng.chance(1, 12) { - // skippable frame - let n = rng.size(300); - out.extend_from_slice(&(0x184D2A50u32 + rng.below(16) as u32).to_le_bytes()); - out.extend_from_slice(&(n as u32).to_le_bytes()); - out.extend_from_slice(&data(rng, n)); - continue; - } - let max = *rng.pick(&[0usize, 10, 300, 5_000, 70_000, 300_000, 1_500_000]); - let len = rng.size(max); - let mut src = data(rng, len); - if !dict.is_empty() && rng.chance(1, 2) && !src.is_empty() { - // make the dictionary useful - let n = rng.size(dict.len().min(src.len())); - let at = rng.below(dict.len() - n + 1); - let to = rng.below(src.len() - n + 1); - src[to..to + n].copy_from_slice(&dict[at..at + n]); - } - let mut cctx = c::CCtx::new(); - let level = *rng.pick(&[-50, -5, -1, 1, 1, 2, 3, 3, 3, 4, 5, 6, 7, 9, 12, 13, 16, 19, 22]); - cctx.set(100, if src.len() > 400_000 { level.min(12) } else { level }).unwrap(); - if rng.chance(1, 3) { cctx.set(201, 1).unwrap(); } // checksum - if rng.chance(1, 4) { cctx.set(200, 0).unwrap(); } // no content size - if rng.chance(1, 4) { cctx.set(101, rng.range(10, 22) as i32).unwrap(); } // windowLog - if rng.chance(1, 10) { cctx.set(130, rng.range(1340, 20_000) as i32).unwrap(); } // targetCBlockSize - if rng.chance(1, 10) { cctx.set(160, 1).unwrap(); } // LDM - if rng.chance(1, 8) { cctx.set(105, rng.range(3, 7) as i32).unwrap(); } // minMatch - if !dict.is_empty() { cctx.load_dict(&dict).unwrap(); } - let mut dst = vec![0u8; unsafe { c::ZSTD_compressBound(src.len()) } + 1024]; - let n = if rng.chance(1, 2) { - cctx.compress2(&mut dst, &src).unwrap() - } else { - // streaming, with flushes: several blocks, no content size - let (mut dpos, mut spos) = (0, 0); - loop { - let chunk = (spos + 1 + rng.size(200_000)).min(src.len()); - let end = if chunk == src.len() { 2 } else if rng.chance(1, 3) { 1 } else { 0 }; - let (r, d, s) = cctx.stream2(&mut dst, dpos, &src[..chunk], spos, end); - dpos = d; spos = s; - if r.unwrap() == 0 && end == 2 { break dpos; } - } - }; - out.extend_from_slice(&dst[..n]); - content.extend_from_slice(&src); - } - (out, content, dict) -} - -fn mutate(rng: &mut Rng, frames: &mut Vec) { - if frames.is_empty() { return; } - for _ in 0..1 + rng.size(4) { - if frames.is_empty() { return; } - // damage is more interesting near the start of a frame - let at = if rng.chance(1, 2) { rng.size(frames.len() - 1) } else { rng.below(frames.len()) }; - match rng.below(8) { - 0 => frames[at] ^= 1 << rng.below(8), - 1 => frames[at] = rng.next() as u8, - 2 => frames.truncate(at), - 3 => { frames.remove(at); } - 4 => frames.insert(at, rng.next() as u8), - 5 => { let n = rng.size(16).min(frames.len() - at); for b in &mut frames[at..at + n] { *b = rng.next() as u8 } } - 6 => { let n = rng.size(64); let extra = data(rng, n); frames.extend_from_slice(&extra) } - _ => { let from = rng.below(frames.len()); let n = rng.size(32).min(frames.len() - from.max(at)); frames.copy_within(from..from + n, at) } - } - } -} - -fn check_sizes(frames: &[u8]) { - let p = frames.as_ptr().cast(); - unsafe { - assert_eq!(c::ZSTD_findDecompressedSize(p, frames.len()), z::find_decompressed_size(frames), "findDecompressedSize"); - assert_eq!(c::ZSTD_getFrameContentSize(p, frames.len()), z::get_frame_content_size(frames), "getFrameContentSize"); - assert_eq!(c::res(c::ZSTD_findFrameCompressedSize(p, frames.len())), rs(z::find_frame_compressed_size(frames)), "findFrameCompressedSize"); - } -} - -fn one_shot(frames: &[u8], dict: &[u8], cap: usize, prefix: bool) { - let mut cd = vec![0xAAu8; cap]; - let mut rd = vec![0xAAu8; cap]; - let mut cctx = c::DCtx::new(); - let mut rctx = z::DCtx::new(); - let (cr, rr); - if prefix { - cctx.ref_prefix(dict).unwrap(); - cr = cctx.decompress(&mut cd, frames); - // an empty prefix is no dictionary at all - rr = if dict.is_empty() { rs(rctx.decompress(&mut rd, frames)) } else { rs(z::DDict::prefix(dict).and_then(|d| rctx.decompress_using_ddict(&mut rd, frames, &d))) }; - } else { - assert_eq!(cctx.load_dict(dict).map(|_| ()), rs(rctx.load_dictionary(dict)), "loadDictionary"); - cr = cctx.decompress(&mut cd, frames); - rr = rs(rctx.decompress(&mut rd, frames)); - } - assert_eq!(cr, rr, "one-shot result (cap {cap})"); - if let Ok(n) = cr { assert!(cd[..n] == rd[..n], "one-shot output"); } -} - -fn streaming(rng: &mut Rng, frames: &[u8], dict: &[u8], expect: Option<&[u8]>) { - let mut cctx = c::DCtx::new(); - let mut rctx = z::DCtx::new(); - if rng.chance(1, 6) { - let (p, v) = (*rng.pick(&[100u32, 100, 100, 0, 99, 1]), *rng.pick(&[0i32, 9, 10, 15, 20, 27, 31, 32, -1])); - assert_eq!(cctx.set(p, v).map(|_| ()), rs(rctx.set_parameter(p as i32, v)), "setParameter {p} {v}"); - } - assert_eq!(cctx.load_dict(dict).map(|_| ()), rs(rctx.load_dictionary(dict)), "loadDictionary"); - // tiny steps only where they do not take forever - let small = frames.len() < 3_000; - let in_max = *rng.pick(if small { &[1usize, 2, 3, 7, 64, 1000, 70_000, 10_000_000] } else { &[300usize, 1000, 5000, 20_000, 70_000, 200_000, 10_000_000, 10_000_000] }); - let out_max = *rng.pick(if small && expect.is_none_or(|e| e.len() < 20_000) { &[1usize, 2, 5, 100, 4096, 16_384, 140_000, 10_000_000] } else { &[1000usize, 4096, 4096, 16_384, 16_384, 140_000, 1_000_000, 10_000_000] }); - let mut total = Vec::new(); - let (mut cbuf, mut rbuf) = (Vec::new(), Vec::new()); - let mut ipos = 0usize; - let mut stalls = 0; - loop { - let iend = (ipos + rng.range(if stalls > 0 { 1 } else { 0 }, in_max)).min(frames.len()); - let cap = rng.range(if stalls > 1 { 1 } else { 0 }, out_max); - let chunk = &frames[ipos..iend]; - if cbuf.len() < cap { cbuf.resize(cap, 0x55); rbuf.resize(cap, 0x55); } - let cd = &mut cbuf[..cap]; - let rd = &mut rbuf[..cap]; - let (cr, co, ci) = cctx.stream(cd, 0, chunk, 0); - let mut ob = z::OutBuffer { dst: rd, pos: 0 }; - let mut ib = z::InBuffer { src: chunk, pos: 0 }; - let rr = rs(rctx.decompress_stream(&mut ob, &mut ib)); - let (ro, ri) = (ob.pos, ib.pos); - assert_eq!((cr, co, ci), (rr, ro, ri), "stream step at {ipos} (in {} out {cap})", chunk.len()); - assert!(cbuf[..co] == rbuf[..ro], "stream output at {ipos}"); - total.extend_from_slice(&rbuf[..ro]); - ipos += ri; - if cr.is_err() { return; } - if ri == 0 && ro == 0 { stalls += 1 } else { stalls = 0 } - if ipos == frames.len() && (cr == Ok(0) || stalls > 3) { break; } - if cr == Ok(0) && rng.chance(1, 8) { - match rng.below(3) { - 0 => { cctx.init_stream(); rctx.init_stream(); if !dict.is_empty() { return; } } - 1 => { assert_eq!(cctx.reset(1).map(|_| ()), rs(rctx.reset(z::ResetDirective::SessionOnly))); } - _ => {} - } - } - } - if let Some(expect) = expect { assert!(total == expect, "stream total"); } -} - -pub fn run(seed: u64, iters: u64) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if i % 200 == 0 { eprintln!("dec iter {i} (seed {s})"); } - let (mut frames, content, dict) = make_frames(&mut rng); - // valid - check_sizes(&frames); - one_shot(&frames, &dict, content.len(), false); - one_shot(&frames, &dict, content.len() + rng.size(300_000), false); - if !content.is_empty() { let cap = rng.below(content.len()); one_shot(&frames, &dict, cap, false); } - streaming(&mut rng, &frames, &dict, Some(&content)); - streaming(&mut rng, &frames, &dict, Some(&content)); - // wrong or missing dictionary - if rng.chance(1, 4) { let n = rng.size(5000); let other = data(&mut rng, n); one_shot(&frames, &other, content.len(), rng.chance(1, 2)); streaming(&mut rng, &frames, &other, None); } - // damaged - for _ in 0..6 { - let mut bad = frames.clone(); - mutate(&mut rng, &mut bad); - check_sizes(&bad); - let cap = content.len() + if rng.chance(1, 2) { rng.size(200_000) } else { 0 }; - one_shot(&bad, &dict, cap, false); - streaming(&mut rng, &bad, &dict, None); - } - mutate(&mut rng, &mut frames); - } - println!("dec ok: {iters} iterations from seed {seed}"); -} diff --git a/src/zstd/wip/harness/src/main.rs b/src/zstd/wip/harness/src/main.rs deleted file mode 100644 index bf2b969a3a4b..000000000000 --- a/src/zstd/wip/harness/src/main.rs +++ /dev/null @@ -1,18 +0,0 @@ -mod c; -mod comp; -mod data; -mod dec; - -fn main() { - let args: Vec = std::env::args().collect(); - let cmd = args.get(1).map(String::as_str).unwrap_or(""); - let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); - let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); - match cmd { - "dec" => dec::run(seed, iters), - "dict" => comp::run_dict(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(1 << 18)), - "mt" => comp::run_mt(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(8 << 20)), - "comp" => comp::run(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(1 << 20)), - _ => eprintln!("usage: zstd_harness dec "), - } -} diff --git a/src/zstd/wip/par.sh b/src/zstd/wip/par.sh deleted file mode 100755 index eb2cc7acaa98..000000000000 --- a/src/zstd/wip/par.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# par.sh : runs the harness in parallel, prints failures. -cmd=$1; first=$2; procs=$3; iters=$4 -mkdir -p /root/zstd-ref/logs -for i in $(seq 0 $((procs-1))); do - seed=$((first+i)) - ( /root/zstd-ref/harness/target/release/zstd_harness $cmd $seed $iters > /root/zstd-ref/logs/$cmd-$seed.log 2>&1 || echo "FAIL $cmd seed $seed" ) & -done -wait -grep -l "panicked\|Sanitizer\|Aborted" /root/zstd-ref/logs/$cmd-*.log 2>/dev/null | head -echo "par done: $cmd $first+$procs x $iters" diff --git a/src/zstd/wip/strip.py b/src/zstd/wip/strip.py deleted file mode 100644 index 0cc7c6a7f0c8..000000000000 --- a/src/zstd/wip/strip.py +++ /dev/null @@ -1,7 +0,0 @@ -import re,sys -src=open(sys.argv[1]).read() -# remove comments, respecting strings -out=re.sub(r'//[^\n]*|/\*.*?\*/|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'', lambda m: m.group(0) if m.group(0)[0] in '"\'' else ' ', src, flags=re.S) -lines=[l.rstrip() for l in out.split('\n')] -lines=[l for l in lines if l.strip() and not re.match(r'\s*(DEBUGLOG|RAWLOG)\(.*\);\s*$', l)] -open(sys.argv[2],'w').write('\n'.join(lines)+'\n') diff --git a/src/zstd/wip/template/mkwork.sh b/src/zstd/wip/template/mkwork.sh deleted file mode 100755 index e3150a313902..000000000000 --- a/src/zstd/wip/template/mkwork.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# mkwork.sh : a scratch copy of the crate and of the harness, at /root/zstd-ref/work//{zstd,harness} -set -e -name=$1; w=/root/zstd-ref/work/$name -mkdir -p $w && rm -rf $w/zstd $w/harness -cp -r /root/repos/oven-sh/bun/src/zstd $w/zstd -cp /root/zstd-ref/template/testing.rs $w/zstd/testing.rs -echo 'pub mod testing;' >> $w/zstd/lib.rs -python3 -I - "$w" <<'PY' -import re,sys -w=sys.argv[1] -root=open('/root/repos/oven-sh/bun/Cargo.toml').read() -def table(name): - m=re.search(r'^\[workspace\.lints\.'+name+r'\]\n(.*?)(?=^\[)',root,re.S|re.M) - return m.group(1) -open(w+'/zstd/Cargo.toml','w').write(f'''[package] -name = "bun_zstd" -version = "0.0.0" -edition = "2024" - -[lib] -path = "lib.rs" - -[dependencies] -strum = {{ version = "0.26", features = ["derive"] }} -bun_pod = {{ path = "/root/repos/oven-sh/bun/src/pod", features = ["alloc"] }} - -[workspace] - -[lints.rust] -{table("rust")} -[lints.clippy] -{table("clippy")} -''') -PY -cp /root/repos/oven-sh/bun/rust-toolchain.toml /root/repos/oven-sh/bun/clippy.toml /root/repos/oven-sh/bun/rustfmt.toml $w/zstd/ 2>/dev/null || true -mkdir -p $w/harness/src -cp /root/zstd-ref/harness/{Cargo.toml,build.rs,rust-toolchain.toml} $w/harness/ -cp /root/zstd-ref/harness/src/{c.rs,data.rs} /root/zstd-ref/template/seqs.rs $w/harness/src/ -sed -i "s#/root/repos/oven-sh/bun/src/zstd#$w/zstd#" $w/harness/Cargo.toml -cat > $w/harness/src/main.rs <<'RS' -mod c; -mod data; -mod seqs; - -fn main() { - let args: Vec = std::env::args().collect(); - let cmd = args.get(1).map(String::as_str).unwrap_or(""); - let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); - let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); - match cmd { - // seqs - "seqs" => { - let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); - seqs::run(seed, iters, &strategies, args.get(5).is_some_and(|s| s == "1")); - } - _ => eprintln!("unknown command"), - } -} -RS -echo "$w ready" diff --git a/src/zstd/wip/template/seqs.rs b/src/zstd/wip/template/seqs.rs deleted file mode 100644 index 7c03d3b0f952..000000000000 --- a/src/zstd/wip/template/seqs.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! One block through a Rust block compressor and through the C library (`ZSTD_generateSequences`). -use crate::c; -use crate::data::{Rng, data}; -use std::ffi::{c_uint, c_void}; - -#[repr(C)] -#[derive(Clone, Copy, Default, Debug)] -struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } -#[repr(C)] -#[derive(Clone, Copy, Debug)] -struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } -unsafe extern "C" { - fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; - fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; - fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; -} - -fn c_sequences(cp: [u32; 7], use_row: bool, dict: &[u8], src: &[u8]) -> Vec<(u32, u32, u32)> { - let mut cctx = c::CCtx::new(); - for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } - cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder - cctx.set(1010, 2).unwrap(); // no block splitter - if !dict.is_empty() { cctx.ref_prefix(dict).unwrap(); } - let mut out = vec![Sequence::default(); src.len() / 3 + 8]; - let n = c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }).unwrap(); - out[..n].iter().map(|s| (s.lit_length, s.match_length, s.offset)).collect() -} - -/// `strategies`: which of 1..=9 to test. `with_dict`: also test with a prefix, which is the extDict variants. -pub fn run(seed: u64, iters: u64, strategies: &[u32], with_dict: bool) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if i % 500 == 0 { eprintln!("seqs iter {i} (seed {s})"); } - let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071]); - let len = rng.range(7, max); - let mut src = data(&mut rng, len); - let dict = if with_dict && rng.chance(1, 2) { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; - if !dict.is_empty() && rng.chance(2, 3) { - for _ in 0..rng.range(1, 6) { - let n = rng.size(dict.len().min(src.len())); - let at = rng.below(dict.len() - n + 1); - let to = rng.below(src.len() - n + 1); - src[to..to + n].copy_from_slice(&dict[at..at + n]); - } - } - let strategy = *rng.pick(strategies); - let p = if rng.chance(1, 3) { - // the parameters of a level, with the strategy forced - let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; - p.strategy = strategy; - p - } else { - CParams { - window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, - search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, - target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, - } - }; - let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; - let cp = [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy]; - let use_row = rng.chance(1, 2); - let expected = c_sequences(cp, use_row, &dict, &src); - let got = bun_zstd::testing::first_block(cp, use_row, &dict, &src); - let got = got.unwrap_or_else(|e| panic!("seed {s}: error {e:?} (cp {cp:?} row {use_row} dict {} src {})", dict.len(), src.len())); - if got != expected { - let at = got.iter().zip(&expected).position(|(a, b)| a != b).unwrap_or(got.len().min(expected.len())); - panic!("seed {s}: cp {cp:?} row {use_row} dict {} src {}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", - dict.len(), src.len(), got.get(at), expected.get(at), got.len(), expected.len()); - } - } - println!("seqs ok: {iters} iterations from seed {seed}, strategies {strategies:?}, dict {with_dict}"); -} diff --git a/src/zstd/wip/template/testing.rs b/src/zstd/wip/template/testing.rs deleted file mode 100644 index ccf602c2a591..000000000000 --- a/src/zstd/wip/template/testing.rs +++ /dev/null @@ -1,140 +0,0 @@ -//! Hooks for the differential tests. Only in scratch copies of the crate. -#![allow(clippy::all, dead_code, unreachable_pub)] - -use crate::ErrorCode; -use crate::compress::internal::*; -use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; - -fn bitmix(mut val: u64, len: u64) -> u64 { - val ^= val.rotate_right(49) ^ val.rotate_right(24); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^= (val >> 35).wrapping_add(len); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^ (val >> 28) -} - -/// The sequences of the first block of a frame, as (literal length, match length, offset), and then the literals -/// that are left as (length, 0, 0). -/// -/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, already adjusted to the sizes. -/// `dict` is a prefix (raw content, as `ZSTD_CCtx_refPrefix`), or empty. `src` is from 7 to 131071 bytes. -pub fn first_block(cp: [u32; 7], use_row: bool, dict: &[u8], src: &[u8]) -> Result, ErrorCode> { - let strategy = Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?; - let c_params = CompressionParameters { - window_log: cp[0], chain_log: cp[1], hash_log: cp[2], search_log: cp[3], min_match: cp[4], target_length: cp[5], - strategy: Some(strategy), - }; - let row_used = row_match_finder_supported(strategy) && use_row; - - // ZSTD_reset_matchState - let mut ms = MatchState::new(); - ms.hash_log3 = if c_params.min_match == 3 { c_params.window_log.min(17) } else { 0 }; - ms.window.clear(); - ms.next_to_update = ms.window.dict_limit; - ms.hash_table = vec![0; 1 << c_params.hash_log]; - ms.chain_table = vec![0; if strategy != Strategy::Fast && !row_used { 1 << c_params.chain_log } else { 0 }]; - ms.hash_table3 = vec![0; if ms.hash_log3 != 0 { 1 << ms.hash_log3 } else { 0 }]; - if row_used { - ms.tag_table = vec![0; 1 << c_params.hash_log]; - ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4); - ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); - } - if strategy >= Strategy::BtOpt { - ms.opt.lit_freq = vec![0; 256]; - ms.opt.lit_length_freq = vec![0; 36]; - ms.opt.match_length_freq = vec![0; 53]; - ms.opt.off_code_freq = vec![0; 32]; - ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; - ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; - } - ms.c_params = c_params; - - let block_size = (1usize << c_params.window_log).min(128 << 10); - let mut seq_store = SeqStore::new(); - seq_store.reserve(block_size / if c_params.min_match == 3 { 3 } else { 4 }, block_size)?; - - let bufs = Buffers { in_buff: &[], user: src, dict }; - - // ZSTD_loadDictionaryContent(dtlm_fast, tfp_forCCtx) - if !dict.is_empty() { - let iend = dict.len(); - ms.window.update(BufId::Dict.start(), BufId::Dict, dict.len(), false); - let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); - let ip = iend - dict.len().min(max_dict_size); - let prefix = Prefix { buf: dict, base: ms.window.base.wrapping_sub(BufId::Dict.start()) }; - ms.next_to_update = prefix.index_of(ip); - ms.loaded_dict_end = prefix.index_of(iend); - if iend - ip > HASH_READ_SIZE { - match strategy { - Strategy::Fast => fast::fill_hash_table(&mut ms, prefix, iend, DictTableLoadMethod::Fast, TableFillPurpose::ForCCtx)?, - Strategy::DFast => double_fast::fill_double_hash_table(&mut ms, prefix, iend, DictTableLoadMethod::Fast, TableFillPurpose::ForCCtx)?, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { - if use_row { - ms.tag_table.fill(0); - lazy::row_update(&mut ms, prefix, iend - HASH_READ_SIZE)?; - } else { - lazy::insert_and_find_first_index(&mut ms, prefix, iend - HASH_READ_SIZE)?; - } - } - _ => opt::update_tree(&mut ms, prefix, iend - HASH_READ_SIZE, iend)?, - } - ms.next_to_update = prefix.index_of(iend); - } - } - - // ZSTD_compressContinue_internal - let src_addr = BufId::User.start(); - if !ms.window.update(src_addr, BufId::User, src.len(), ms.force_non_contiguous) { - ms.force_non_contiguous = false; - ms.next_to_update = ms.window.dict_limit; - } - // ZSTD_compress_frameChunk - let max_dist = 1u32 << c_params.window_log; - if ms.window.dict_is_invalid(src_addr + src.len(), max_dist, ms.loaded_dict_end) { - ms.loaded_dict_end = 0; - } - if ms.window.enforce_max_dist(src_addr, max_dist, ms.loaded_dict_end) { - ms.loaded_dict_end = 0; - } - if ms.next_to_update < ms.window.low_limit { - ms.next_to_update = ms.window.low_limit; - } - // ZSTD_buildSeqStore - let prev = CompressedBlockState::new(); - let block = BlockSrc { - buf: bufs.get(ms.window.base_buf), - base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), - dict_buf: bufs.get(ms.window.dict_buf), - dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), - start: 0, - end: src.len(), - dms: None, - symbol_costs: &prev.entropy, - ldm_seq_store: None, - }; - let curr = block.index_of(0); - if curr > ms.next_to_update + 384 { - ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); - } - let dict_mode = ms.dict_mode(None); - let mut rep = prev.rep; - let last_ll = compress::compress_block(&mut ms, &mut seq_store, &mut rep, &block, strategy, use_row, dict_mode)?; - - let mut out = Vec::new(); - let mut reps = prev.rep; - for (i, seq) in seq_store.sequences.iter().enumerate() { - let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); - if seq_store.long_length_pos as usize == i { - match seq_store.long_length_type { - LongLengthType::LiteralLength => ll += 0x10000, - LongLengthType::MatchLength => ml += 0x10000, - LongLengthType::None => {} - } - } - update_rep(&mut reps, seq.off_base, seq.lit_length == 0); - out.push((ll, ml, reps[0])); - } - out.push((last_ll as u32, 0, 0)); - assert_eq!(reps, rep, "the repeat offsets the block compressor leaves are those of its sequences"); - Ok(out) -} diff --git a/src/zstd/wip/work/entropy/STATUS.md b/src/zstd/wip/work/entropy/STATUS.md deleted file mode 100644 index 09d439397c7d..000000000000 --- a/src/zstd/wip/work/entropy/STATUS.md +++ /dev/null @@ -1,46 +0,0 @@ -# entropy encoders: status at 12:27 UTC, 2026-10-27 - -## Deliverables (finished, in their last tested state) -- `zstd/compress/hist.rs` (132 lines), `zstd/compress/fse.rs` (678), `zstd/compress/huf.rs` (1162) -- Test: `harness/src/entropy.rs`, `harness/entropy_shim.c` (the inline functions of fse.h), `harness/build.rs` (compiles the - shim with clang-23), `harness/src/main.rs` (commands `entropy`, `entropy-bench`), wrappers at the end of `zstd/testing.rs` - (`pub mod entropy`, `pub mod entropy_bench`; testing.rs also got `clippy::indexing_slicing` allowed and was rustfmt'ed). -- No other file of the crate differs from the real one (`common/bitstream.rs` was changed for an experiment and restored; - `diff` with the real one is empty). - -## Verified -- `cargo fmt --check`: ok. `cargo clippy`: the only finding in the three files is dead code - (`FseRepeat::Check`/`Valid` never constructed). In the real crate, without testing.rs, most items will be dead code. -- Differential run on the final code, overflow checks and debug assertions on: seeds 100..105 x 90000 iterations, 6 processes, - no failure. 172.9 M comparisons in all, of which 135 M are the cheap `fse_logs` ones. Per seed: - hist 540000, fse_logs 22500000, fse 674465, fse_tables 2644542, fse_bad_norm 90000, huf_table 2171995, huf_read 90000, - huf_repeat 110228. Logs: `harness/logs/entropy-10*.log`. -- Each comparison checks the return value (which error too), the whole destination buffer up to its capacity (not only the - returned size), the tables (Huffman: all 257 words, from the same random or zero start; FSE: converted to the C layout), - `repeat`, `maxSymbolValue`. -- Line coverage (an earlier build, 20000 iterations): every line reached but those listed under "not reached". - -## How to run -``` -. ~/.bun-dev-env && cd /root/zstd-ref/work/entropy/harness -RUSTFLAGS="--cap-lints=warn" cargo build --release -./target/release/zstd_harness entropy [all|hist|fse_logs|fse|fse_tables|fse_bad_norm|huf_table|huf_read|huf_repeat] -ENTROPY_TRACE=1 ... # prints the seed of each case -# speed, C against Rust. Its own target dir: the `bench` profile writes to target/release and would replace the checked build. -CARGO_TARGET_DIR=$PWD/target-bench RUSTFLAGS="--cap-lints=warn" cargo build --profile bench -./target-bench/release/zstd_harness entropy-bench -``` - -## Not reached by the tests (all are broken-invariant or unreachable paths) -- `normalize_m2`: `to_distribute == 0` (cannot happen once `min_table_log` has passed), `weight < 1` -> Generic, and my guard - against an endless loop (`no norm > 0` -> Generic). -- `read_ctable`: `table_log > 12` (`huf_read_stats` refuses it first). -- `optimal_table_log`: `build_ctable` failing inside the search. - -## Open -- Speed, not correctness (see the report): Huffman encoding 1.08-1.20x the C time (C without BMI2), histogram 1.08x, - `build_ctable` 1.07-1.28x, `write_ctable` 1.25x, `write_ncount` 1.35x (50 ns), `FseCTable::build` 1.0x, - three-state FSE encoding 1.34x, which falls to 1.20x with a 3-line change to `BitCStream::flush` in - `common/bitstream.rs` (not my file): `let pos = self.pos.min(self.end);` write at `pos`, `self.pos = pos + nb_bytes;`. - LLVM then drops the bounds check on the write. The same trick is in `HufCStream::flush_bits`. -- The last speed numbers for `build_ctable` and the Huffman encoder were measured on a machine under load (load average 17). diff --git a/src/zstd/wip/work/entropy/harness/Cargo.toml b/src/zstd/wip/work/entropy/harness/Cargo.toml deleted file mode 100644 index 814bb28bb616..000000000000 --- a/src/zstd/wip/work/entropy/harness/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "zstd_harness" -version = "0.0.0" -edition = "2024" - -[dependencies] -bun_zstd = { path = "/root/zstd-ref/work/entropy/zstd" } - -[profile.release] -debug = "line-tables-only" -overflow-checks = true -debug-assertions = true -panic = "abort" - -[profile.bench] -inherits = "release" -overflow-checks = false -debug-assertions = false - -[workspace] diff --git a/src/zstd/wip/work/entropy/harness/build.rs b/src/zstd/wip/work/entropy/harness/build.rs deleted file mode 100644 index 582ebed0a7b9..000000000000 --- a/src/zstd/wip/work/entropy/harness/build.rs +++ /dev/null @@ -1,20 +0,0 @@ -use std::process::Command; - -fn main() { - let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); - println!("cargo:rerun-if-env-changed=ZSTD_REF"); - println!("cargo:rustc-link-search=native=/root/zstd-ref/build/{variant}"); - println!("cargo:rustc-link-lib=static=zstd"); - - // The inline functions of fse.h. - println!("cargo:rerun-if-changed=entropy_shim.c"); - let out = std::env::var("OUT_DIR").unwrap(); - let ok = Command::new("clang-23") - .args(["-c", "-O2", "-g", "-fPIC", "-march=nehalem", "-I/root/zstd-ref/zstd/lib", "-I/root/zstd-ref/zstd/lib/common", "entropy_shim.c", "-o"]) - .arg(format!("{out}/entropy_shim.o")).status().unwrap().success(); - assert!(ok); - let ok = Command::new("llvm-ar-23").args(["rcs", &format!("{out}/libentropy_shim.a"), &format!("{out}/entropy_shim.o")]).status().unwrap().success(); - assert!(ok); - println!("cargo:rustc-link-search=native={out}"); - println!("cargo:rustc-link-lib=static=entropy_shim"); -} diff --git a/src/zstd/wip/work/entropy/harness/entropy_shim.c b/src/zstd/wip/work/entropy/harness/entropy_shim.c deleted file mode 100644 index 32426c9fe716..000000000000 --- a/src/zstd/wip/work/entropy/harness/entropy_shim.c +++ /dev/null @@ -1,43 +0,0 @@ -/* The inline functions of fse.h, for the differential test. */ -#define FSE_STATIC_LINKING_ONLY -#include "mem.h" -#include "bitstream.h" -#include "fse.h" - -size_t shim_fse_encode3(void* dst, size_t cap, const FSE_CTable* ta, const FSE_CTable* tb, const FSE_CTable* tc, - const BYTE* sa, const BYTE* sb, const BYTE* sc, size_t n) -{ - BIT_CStream_t bitC; - FSE_CState_t a, b, c, d; - size_t i; - { size_t const e = BIT_initCStream(&bitC, dst, cap); if (ERR_isError(e)) return e; } - FSE_initCState2(&a, ta, sa[n-1]); - FSE_initCState2(&b, tb, sb[n-1]); - FSE_initCState2(&c, tc, sc[n-1]); - for (i = n-2; i < n; i--) { - FSE_encodeSymbol(&bitC, &b, sb[i]); - FSE_encodeSymbol(&bitC, &a, sa[i]); - FSE_encodeSymbol(&bitC, &c, sc[i]); - BIT_flushBits(&bitC); - } - FSE_flushCState(&bitC, &a); - FSE_flushCState(&bitC, &b); - FSE_flushCState(&bitC, &c); - FSE_initCState(&d, ta); - FSE_flushCState(&bitC, &d); - return BIT_closeCStream(&bitC); -} - -U32 shim_fse_getMaxNbBits(const FSE_CTable* ct, U32 symbol) -{ - FSE_CState_t s; - FSE_initCState(&s, ct); - return FSE_getMaxNbBits(s.symbolTT, symbol); -} - -U32 shim_fse_bitCost(const FSE_CTable* ct, U32 symbol, U32 accuracyLog) -{ - FSE_CState_t s; - FSE_initCState(&s, ct); - return FSE_bitCost(s.symbolTT, s.stateLog, symbol, accuracyLog); -} diff --git a/src/zstd/wip/work/entropy/harness/rust-toolchain.toml b/src/zstd/wip/work/entropy/harness/rust-toolchain.toml deleted file mode 100644 index c71778e8ccd3..000000000000 --- a/src/zstd/wip/work/entropy/harness/rust-toolchain.toml +++ /dev/null @@ -1,35 +0,0 @@ -[toolchain] -channel = "nightly-2026-09-15" -# rust-src is needed for -Zbuild-std (Tier 3 targets like -# aarch64-unknown-freebsd have no prebuilt std). miri is for -# `bun run rust:miri`. targets ensures the -# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain -# (CI's images install this same channel with these components and -# targets, from scripts/build/ci-images/spec.ts; a source lint keeps -# the two files in step). Note: rustup ignores this whole file when -# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree -# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` -# self-heals that path. Keep this list as a fallback for direct -# `cargo build`/`cargo check`/rust-analyzer use. -# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd -# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). -# darwin/windows triples build natively on their own agents, so the host std -# comes with the toolchain install regardless; they're listed here so a dev -# running `bun run rust:check-all` or `cargo check --target` from any host -# still gets prebuilt std for every Tier 1/2 triple. -# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at -# what it emits. The build does not use them. -components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] -targets = [ - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "x86_64-unknown-linux-musl", - "aarch64-linux-android", - "x86_64-linux-android", - "x86_64-unknown-freebsd", - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-pc-windows-msvc", - "x86_64-pc-windows-msvc", -] diff --git a/src/zstd/wip/work/entropy/harness/src/c.rs b/src/zstd/wip/work/entropy/harness/src/c.rs deleted file mode 100644 index d5a82e1f6b78..000000000000 --- a/src/zstd/wip/work/entropy/harness/src/c.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! The reference C library. -#![allow(non_camel_case_types, dead_code)] -use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; - -#[repr(C)] -pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } -#[repr(C)] -pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } -pub enum ZSTD_CCtx {} -pub enum ZSTD_DCtx {} -pub enum ZSTD_DDict {} - -unsafe extern "C" { - pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; - pub fn ZSTD_compressBound(size: usize) -> usize; - pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_isError(code: usize) -> c_uint; - pub fn ZSTD_getErrorCode(code: usize) -> c_uint; - pub fn ZSTD_getErrorName(code: usize) -> *const c_char; - pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; - pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; - pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; - pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; - pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; - pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; - pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; - pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; - pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; - pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; - pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; - pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; - pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; - pub fn ZSTD_defaultCLevel() -> c_int; - pub fn ZSTD_minCLevel() -> c_int; - pub fn ZSTD_maxCLevel() -> c_int; -} - -/// `Ok(value)` or `Err(error code)`. -pub fn res(code: usize) -> Result { - unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } -} - -pub struct CCtx(pub *mut ZSTD_CCtx); -impl CCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } - pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - /// Returns (result, out pos, in pos). - pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); - (r, o.pos, i.pos) - } -} -impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } - -pub struct DCtx(pub *mut ZSTD_DCtx); -impl DCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } - pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } - pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); - (r, o.pos, i.pos) - } -} -impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } - -pub fn compress(src: &[u8], level: i32) -> Vec { - let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; - let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); - dst.truncate(n); - dst -} -pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) -} diff --git a/src/zstd/wip/work/entropy/harness/src/data.rs b/src/zstd/wip/work/entropy/harness/src/data.rs deleted file mode 100644 index c48a645e1b5d..000000000000 --- a/src/zstd/wip/work/entropy/harness/src/data.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Deterministic inputs. -pub struct Rng(pub u64); -impl Rng { - pub fn next(&mut self) -> u64 { - // splitmix64 - self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); - z ^ (z >> 31) - } - pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } - pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } - pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } - pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } - /// Skewed towards small values. - pub fn size(&mut self, max: usize) -> usize { - let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); - (self.next() as usize & ((1usize << bits) - 1)).min(max) - } -} - -static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; - -/// One buffer of about `len` bytes, of a kind picked by `rng`. -pub fn data(rng: &mut Rng, len: usize) -> Vec { - let mut v = Vec::with_capacity(len + 64); - match rng.below(9) { - 0 => v.resize(len, rng.next() as u8), - 1 => while v.len() < len { v.push(rng.next() as u8) }, - 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, - 3 => { // few symbols, skewed - let n = rng.range(2, 40); - while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } - } - 4 => { // LZ-like: copies of earlier parts, at all distances - while v.len() < len { - if v.len() > 8 && rng.chance(3, 4) { - let dist = 1 + rng.size(v.len() - 1); - let n = 3 + rng.size(300); - for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } - } else { - for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } - } - } - } - 5 => { // periodic with noise - let period = rng.range(1, 300); - let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); - while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } - } - 6 => { // sections of different kinds - while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } - } - 7 => { // numbers as text - let mut x = rng.next() % 1000; - while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } - } - _ => { // long runs - while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } - } - } - v.truncate(len); - v -} diff --git a/src/zstd/wip/work/entropy/harness/src/entropy.rs b/src/zstd/wip/work/entropy/harness/src/entropy.rs deleted file mode 100644 index 120a61fc48ce..000000000000 --- a/src/zstd/wip/work/entropy/harness/src/entropy.rs +++ /dev/null @@ -1,774 +0,0 @@ -//! The entropy encoders (`hist`, `fse`, `huf`) against the C library, function by function. -use crate::c::res; -use crate::data::{Rng, data}; -use bun_zstd::ErrorCode; -use bun_zstd::testing::entropy as rs; -use std::ffi::{c_int, c_uint, c_void}; - -#[repr(C)] -struct CTableHeader { table_log: u8, max_symbol_value: u8, unused: [u8; 6] } - -unsafe extern "C" { - fn HIST_add(count: *mut c_uint, src: *const c_void, size: usize); - fn HIST_count_simple(count: *mut c_uint, max: *mut c_uint, src: *const c_void, size: usize) -> c_uint; - fn HIST_countFast_wksp(count: *mut c_uint, max: *mut c_uint, src: *const c_void, size: usize, wksp: *mut c_void, wksp_size: usize) -> usize; - fn HIST_count_wksp(count: *mut c_uint, max: *mut c_uint, src: *const c_void, size: usize, wksp: *mut c_void, wksp_size: usize) -> usize; - - fn FSE_buildCTable_wksp(ct: *mut u32, norm: *const i16, max: c_uint, table_log: c_uint, wksp: *mut c_void, wksp_size: usize) -> usize; - fn FSE_buildCTable_rle(ct: *mut u32, symbol: u8) -> usize; - fn FSE_NCountWriteBound(max: c_uint, table_log: c_uint) -> usize; - fn FSE_writeNCount(buf: *mut c_void, size: usize, norm: *const i16, max: c_uint, table_log: c_uint) -> usize; - fn FSE_optimalTableLog(max_log: c_uint, size: usize, max: c_uint) -> c_uint; - fn FSE_optimalTableLog_internal(max_log: c_uint, size: usize, max: c_uint, minus: c_uint) -> c_uint; - fn FSE_normalizeCount(norm: *mut i16, table_log: c_uint, count: *const c_uint, total: usize, max: c_uint, low_prob: c_uint) -> usize; - fn FSE_compress_usingCTable(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, ct: *const u32) -> usize; - - fn HUF_compressBound(size: usize) -> usize; - fn HUF_buildCTable_wksp(t: *mut u64, count: *const c_uint, max: u32, max_nb_bits: u32, wksp: *mut c_void, wksp_size: usize) -> usize; - fn HUF_writeCTable_wksp(dst: *mut c_void, cap: usize, t: *const u64, max: c_uint, huff_log: c_uint, wksp: *mut c_void, wksp_size: usize) -> usize; - fn HUF_readCTable(t: *mut u64, max: *mut c_uint, src: *const c_void, size: usize, has_zero: *mut c_uint) -> usize; - fn HUF_estimateCompressedSize(t: *const u64, count: *const c_uint, max: c_uint) -> usize; - fn HUF_validateCTable(t: *const u64, count: *const c_uint, max: c_uint) -> c_int; - fn HUF_optimalTableLog(max_log: c_uint, size: usize, max: c_uint, wksp: *mut c_void, wksp_size: usize, t: *mut u64, count: *const c_uint, flags: c_int) -> c_uint; - fn HUF_minTableLog(cardinality: c_uint) -> c_uint; - fn HUF_cardinality(count: *const c_uint, max: c_uint) -> c_uint; - fn HUF_getNbBitsFromCTable(t: *const u64, symbol: u32) -> u32; - fn HUF_readCTableHeader(t: *const u64) -> CTableHeader; - fn HUF_compress1X_usingCTable(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, t: *const u64, flags: c_int) -> usize; - fn HUF_compress4X_usingCTable(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, t: *const u64, flags: c_int) -> usize; - fn HUF_compress1X_repeat(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, max: c_uint, log: c_uint, wksp: *mut c_void, wksp_size: usize, t: *mut u64, repeat: *mut c_uint, flags: c_int) -> usize; - fn HUF_compress4X_repeat(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, max: c_uint, log: c_uint, wksp: *mut c_void, wksp_size: usize, t: *mut u64, repeat: *mut c_uint, flags: c_int) -> usize; - - fn shim_fse_encode3(dst: *mut c_void, cap: usize, ta: *const u32, tb: *const u32, tc: *const u32, sa: *const u8, sb: *const u8, sc: *const u8, n: usize) -> usize; - fn shim_fse_getMaxNbBits(ct: *const u32, symbol: u32) -> u32; - fn shim_fse_bitCost(ct: *const u32, symbol: u32, accuracy_log: u32) -> u32; -} - -type R = Result; -fn conv(r: Result) -> R { r.map_err(|e| e as u8 as u32) } - -/// What the C code may write past the end of, in cases that are bugs of the caller. -const SLACK: usize = 2048; -const FILL: u8 = 0xA5; - -/// A workspace for the C code, zeroed. -fn wksp() -> Vec { vec![0u64; 8704 / 8 + 64] } -const WKSP_SIZE: usize = 8704; - -pub struct Stats { pub cases: u64 } -impl Stats { fn hit(&mut self) { self.cases += 1; } } - -/// An output buffer for each side. The C one has slack after `cap`. -struct Out { c: Vec, r: Vec, cap: usize } -impl Out { - fn new(cap: usize) -> Self { Self { c: vec![FILL; cap + SLACK], r: vec![FILL; cap], cap } } - fn c_ptr(&mut self) -> *mut c_void { self.c.as_mut_ptr().cast() } - #[track_caller] - fn check(&self, what: &str, ctx: &dyn std::fmt::Debug) { - assert!(self.c[..self.cap] == self.r[..], "{what}: output differs at {:?} ({ctx:?})", self.c.iter().zip(&self.r).position(|(a, b)| a != b)); - assert!(self.c[self.cap..].iter().all(|&b| b == FILL), "{what}: the C code wrote past the end ({ctx:?})"); - } -} - -// ---------------------------------------------------------------- inputs - -fn sizes(rng: &mut Rng) -> usize { - match rng.below(8) { - 0 => *rng.pick(&[0usize, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 15, 16, 17, 19, 20, 21, 31, 32, 33, 63, 64, 65]), - 1 => *rng.pick(&[255usize, 256, 257, 1023, 1024, 1025, 1499, 1500, 1501, 4095, 4096, 40959, 40960, 40961, 65535, 65536, 131071, 131072]), - 2 => rng.range(0, 64), - 3 | 4 => rng.range(0, 2000), - 5 | 6 => rng.range(0, 20000), - _ => rng.range(0, 131072), - } -} - -/// 256 counts, of which those above `max` are 0. At least one is not 0. They add up to at most about `budget`. -fn counts(rng: &mut Rng, max: usize, budget: usize) -> Vec { - let n = max + 1; - let mut c = vec![0u32; 256]; - match rng.below(12) { - 0 => { let len = rng.range(1, budget); for b in data(rng, len) { c[b as usize % n] += 1; } } - 1 => { // geometric - let mut v = budget as f64 / 2.0; let ratio = 0.3 + 0.69 * (rng.below(1000) as f64 / 1000.0); - for x in c[..n].iter_mut() { *x = v as u32; v *= ratio; } - } - 2 => { // Fibonacci: the deepest trees - let (mut a, mut b) = (1u32, 1u32); let mut total = 0usize; - for x in c[..n].iter_mut() { if total + a as usize > budget { break; } *x = a; total += a as usize; (a, b) = (b, a + b); } - } - 3 => { let v = (*rng.pick(&[1u32, 2, 3, 163, 164, 165, 166, 200, 255, 256, 1000])).min((budget / n).max(1) as u32); c[..n].fill(v); } - 4 => { // many ties, around the counts that share buckets - let lo = rng.range(1, 600).min((budget / n).max(1)); let span = rng.range(1, 300); - for x in c[..n].iter_mut() { *x = (lo + rng.below(span)).min((budget / n).max(1)) as u32; } - } - 5 => { for _ in 0..rng.range(1, 6) { c[rng.below(n)] = rng.range(1, (budget / 8).max(1)) as u32; } } - 6 => { c[..n].fill(1); c[rng.below(n)] = (budget.saturating_sub(n)).max(1) as u32; } - 7 => { let top = (usize::BITS - (budget / n).max(1).leading_zeros()) as usize; for x in c[..n].iter_mut() { *x = 1 << rng.below(top); } } - 8 => { for x in c[..n].iter_mut() { if rng.chance(1, 2) { *x = rng.size((2 * budget / n).max(1)) as u32; } } } - 9 => { for x in c[..n].iter_mut() { *x = rng.range(0, 3) as u32; } } - 10 => { // two plateaus - let hi = (budget / n).max(2) as u32; let cut = rng.below(n); - for (i, x) in c[..n].iter_mut().enumerate() { *x = if i < cut { hi } else { 1 }; } - } - _ => { let m = (budget / n).max(1); for x in c[..n].iter_mut() { *x = rng.range(0, m) as u32; } } - } - if rng.chance(1, 2) { // any order - for i in (1..n).rev() { let j = rng.below(i + 1); c.swap(i, j); } - } - if c.iter().all(|&x| x == 0) { c[rng.below(n)] = rng.range(1, 5) as u32; } - c -} - -/// Symbols that occur as `counts` says, in any order. -fn data_of(rng: &mut Rng, counts: &[u32]) -> Vec { - let mut v = Vec::new(); - for (s, &c) in counts.iter().enumerate() { v.resize(v.len() + c as usize, s as u8); } - match rng.below(4) { - 0 => {} - 1 => v.reverse(), - _ => for i in (1..v.len()).rev() { let j = rng.below(i + 1); v.swap(i, j); }, - } - v -} - -/// `len` symbols among those whose weight is not 0. -fn data_among(rng: &mut Rng, weights: &[i16], len: usize) -> Vec { - let mut pool = Vec::new(); - for (s, &w) in weights.iter().enumerate() { for _ in 0..w.unsigned_abs() { pool.push(s as u8); } } - let uniform: Vec = weights.iter().enumerate().filter(|(_, w)| **w != 0).map(|(s, _)| s as u8).collect(); - let pool = if rng.chance(1, 4) { uniform } else { pool }; - (0..len).map(|_| *rng.pick(&pool)).collect() -} - -/// Normalized counts that add up to `1 << table_log`, made directly. -fn random_norm(rng: &mut Rng, max: usize, table_log: u32) -> Vec { - let size = 1usize << table_log; - let mut norm = vec![0i16; max + 1]; - let mut left = size; - let low = if rng.chance(1, 2) { rng.below((max + 1).min(size - 1) + 1) } else { 0 }; - let mut order: Vec = (0..=max).collect(); - for i in (1..order.len()).rev() { let j = rng.below(i + 1); order.swap(i, j); } - for &s in &order[..low] { norm[s] = -1; left -= 1; } - let others = &order[low..]; - if others.is_empty() || left == 0 { - // All the slots must be taken: give what is left to a low probability symbol. - if left > 0 { let s = order[0]; norm[s] = 1 + left as i16; } - return norm; - } - let used = rng.range(1, others.len().min(left)); - for &s in &others[..used] { norm[s] = 1; left -= 1; } - while left > 0 { - let s = others[rng.below(used)]; - let n = if rng.chance(1, 3) { left } else { 1 + rng.size(left - 1) }; - norm[s] += n as i16; left -= n; - } - norm -} - -// ---------------------------------------------------------------- hist - -fn t_hist(rng: &mut Rng, st: &mut Stats) { - let len = sizes(rng); - let mut src = data(rng, len); - if rng.chance(1, 2) { let m = rng.range(1, 256); for b in src.iter_mut() { *b = (*b as usize % m) as u8; } } - let actual = src.iter().copied().max().unwrap_or(0) as u32; - let mut padded = src.clone(); padded.extend_from_slice(&[0xEE; 32]); - let p: *const c_void = padded.as_ptr().cast(); - let garbage: Vec = (0..256).map(|_| rng.next() as u32 >> 8).collect(); - let mut w = wksp(); - - // Checked. - for _ in 0..3 { - let max = match rng.below(4) { 0 => rng.below(256) as u32, 1 => actual, 2 => actual.saturating_sub(1), _ => *rng.pick(&[255u32, 256, 1000]) }; - let (mut cc, mut cr, mut mc, mut mr) = (garbage.clone(), garbage.clone(), max, max); - let a = res(unsafe { HIST_count_wksp(cc.as_mut_ptr(), &mut mc, p, len, w.as_mut_ptr().cast(), 4096) }); - let b = conv(rs::hist_count_wksp(&mut cr, &mut mr, &src)); - assert_eq!((a, mc, &cc), (b, mr, &cr), "HIST_count_wksp len {len} max {max}"); - st.hit(); - } - // Trusted. - let max = rng.range(actual as usize, 255) as u32; - let (mut cc, mut cr, mut mc, mut mr) = (garbage.clone(), garbage.clone(), max, max); - let a = res(unsafe { HIST_countFast_wksp(cc.as_mut_ptr(), &mut mc, p, len, w.as_mut_ptr().cast(), 4096) }); - let b = conv(rs::hist_count_fast_wksp(&mut cr, &mut mr, &src)); - assert_eq!((a, mc, &cc), (b, mr, &cr), "HIST_countFast_wksp len {len} max {max}"); - st.hit(); - - let (mut cc, mut cr, mut mc, mut mr) = (garbage.clone(), garbage.clone(), max, max); - let a = unsafe { HIST_count_simple(cc.as_mut_ptr(), &mut mc, p, len) }; - let b = rs::hist_count_simple(&mut cr, &mut mr, &src).unwrap(); - assert_eq!((a, mc, &cc), (b, mr, &cr), "HIST_count_simple len {len} max {max}"); - st.hit(); - - let (mut cc, mut cr) = (garbage.clone(), garbage.clone()); - unsafe { HIST_add(cc.as_mut_ptr(), p, len) }; - rs::hist_add(&mut cr, &src).unwrap(); - assert_eq!(cc, cr, "HIST_add"); - st.hit(); -} - -// ---------------------------------------------------------------- fse - -fn t_fse_logs(rng: &mut Rng, st: &mut Stats) { - for _ in 0..50 { - let max_log = rng.below(15) as u32; - let size = match rng.below(3) { 0 => rng.range(2, 40), 1 => rng.range(2, 5000), _ => 2 + rng.size(1 << 20) }; - let max = rng.range(1, 255) as u32; - let minus = rng.range(1, 2) as u32; - assert_eq!(unsafe { FSE_optimalTableLog(max_log, size, max) }, rs::fse_optimal_table_log(max_log, size, max), "FSE_optimalTableLog {max_log} {size} {max}"); - assert_eq!(unsafe { FSE_optimalTableLog_internal(max_log, size, max, minus) }, rs::fse_optimal_table_log_internal(max_log, size, max, minus), "FSE_optimalTableLog_internal {max_log} {size} {max} {minus}"); - let max0 = rng.below(256) as u32; - assert_eq!(unsafe { FSE_NCountWriteBound(max0, max_log) }, rs::fse_ncount_write_bound(max0, max_log)); - assert_eq!(unsafe { HUF_compressBound(size) }, rs::huf_compress_bound(size)); - assert_eq!(unsafe { HUF_minTableLog(max + 1) }, rs::huf_min_table_log(max + 1)); - st.cases += 5; - } -} - -/// `FSE_CTable` of the C code, with garbage that it is not to touch... which is zero, as for the other side. -fn c_fse_table() -> Vec { vec![0u32; 1 + (1 << 11) + 256 * 2 + 16] } - -fn write_ncount_both(rng: &mut Rng, st: &mut Stats, norm: &[i16], max: u32, table_log: u32) -> R { - let bound = unsafe { FSE_NCountWriteBound(max, table_log) }; - let mut full = Err(0); - for k in 0..4 { - let cap = match k { 0 => 512 + rng.below(8), 1 => bound, 2 => bound.saturating_sub(1 + rng.below(4)), _ => rng.below(bound + 2) }; - let mut out = Out::new(cap); - let a = res(unsafe { FSE_writeNCount(out.c_ptr(), cap, norm.as_ptr(), max, table_log) }); - let b = conv(rs::fse_write_ncount(&mut out.r, norm, max, table_log)); - assert_eq!(a, b, "FSE_writeNCount cap {cap} max {max} log {table_log} {norm:?}"); - out.check("FSE_writeNCount", &(cap, max, table_log, norm)); - st.hit(); - if k == 0 { full = a; } - } - full -} - -/// Builds the table on both sides and uses it. `norm` is valid, `table_log <= 9`, `max <= 52`. -fn use_fse_table(rng: &mut Rng, st: &mut Stats, norm: &[i16], max: u32, table_log: u32) -> (Vec, rs::Fse) { - let mut w = wksp(); - let mut ct = c_fse_table(); - let mut rt = rs::Fse::new(); - let a = res(unsafe { FSE_buildCTable_wksp(ct.as_mut_ptr(), norm.as_ptr(), max, table_log, w.as_mut_ptr().cast(), WKSP_SIZE) }); - let b = conv(rt.build(norm, max, table_log)); - assert_eq!(a, b.map(|_| 0), "FSE_buildCTable_wksp"); - let conv_t = rt.to_c(); - assert_eq!(&ct[..conv_t.len()], &conv_t[..], "FSE_buildCTable_wksp table: max {max} log {table_log} {norm:?}"); - st.hit(); - - for s in 0..=max { - assert_eq!(unsafe { shim_fse_getMaxNbBits(ct.as_ptr(), s) }, rt.get_max_nb_bits(s), "FSE_getMaxNbBits"); - if norm[s as usize] != 0 { - assert_eq!(unsafe { shim_fse_bitCost(ct.as_ptr(), s, 8) }, rt.bit_cost(s, 8), "FSE_bitCost"); - } - } - st.hit(); - - for _ in 0..3 { - let len = match rng.below(4) { 0 => rng.below(12), 1 => rng.below(300), 2 => rng.below(300), _ => rng.below(3000) }; - let src = data_among(rng, norm, len); - let bound = len + (len >> 7) + 4 + 8; - let cap = match rng.below(5) { 0 => bound, 1 => bound - 1, 2 => bound + rng.below(50), 3 => rng.below(bound), _ => rng.below(20) }; - let mut out = Out::new(cap); - let a = unsafe { FSE_compress_usingCTable(out.c_ptr(), cap, src.as_ptr().cast(), len, ct.as_ptr()) }; - let b = rt.compress(&mut out.r, &src); - // `FSE_BLOCKBOUND` is no bound for rare symbols with a large table log, and the C code trusts it. - if cap >= bound && out.c[cap..].iter().any(|&b| b != FILL) { assert!(table_log > 7); assert_eq!(b, 0); continue; } - assert_eq!(a, b, "FSE_compress_usingCTable len {len} cap {cap}"); - out.check("FSE_compress_usingCTable", &(len, cap, table_log)); - st.hit(); - } - (ct, rt) -} - -fn t_fse(rng: &mut Rng, st: &mut Stats) { - // The alphabets of sequences and of Huffman weights, or any. - let max = match rng.below(6) { 0 => 12, 1 => 35, 2 => 52, 3 => 31, 4 => rng.range(1, 52), _ => rng.range(1, 255) }; - let budget = *rng.pick(&[8usize, 40, 255, 1000, 20000, 131072, 1 << 22]); - let mut count = counts(rng, max, budget); - // As the callers do, `max` is the last symbol present, most of the time. - let max = if rng.chance(4, 5) { count.iter().rposition(|&c| c != 0).unwrap().max(1) } else { max } as u32; - let mut total: usize = count.iter().map(|&c| c as usize).sum(); - if total < 2 { count[0] += 2; total += 2; } - let low_prob = rng.chance(1, 2); - let table_log = match rng.below(4) { - 0 => rng.below(15) as u32, - _ => { let max_log = *rng.pick(&[5u32, 6, 8, 9, 9, 11, 12, 0]); unsafe { FSE_optimalTableLog(max_log, total, max) } } - }; - let mut nc = vec![0x5A5Ai16; 256]; - let mut nr = nc.clone(); - let a = res(unsafe { FSE_normalizeCount(nc.as_mut_ptr(), table_log, count.as_ptr(), total, max, low_prob as u32) }); - let b = conv(rs::fse_normalize_count(&mut nr, table_log, &count, total, max, low_prob)); - assert_eq!((a, &nc), (b, &nr), "FSE_normalizeCount log {table_log} total {total} max {max} low {low_prob} {:?}", &count[..=max as usize]); - st.hit(); - let Ok(log) = a else { return }; - if log == 0 { return; } - let log = log as u32; - let norm = &nc[..=max as usize]; - assert_eq!(norm.iter().map(|n| n.unsigned_abs() as usize).sum::(), 1 << log); - write_ncount_both(rng, st, norm, max, log).unwrap(); - if log <= 9 && max <= 52 { use_fse_table(rng, st, norm, max, log); } -} - -fn t_fse_tables(rng: &mut Rng, st: &mut Stats) { - let mut made = Vec::new(); - for _ in 0..3 { - let table_log = rng.range(5, 9) as u32; - let max = match rng.below(4) { 0 => 52, 1 => 35, 2 => 31, _ => rng.range(1, 52) }; - let norm = random_norm(rng, max, table_log); - assert_eq!(norm.iter().map(|n| n.unsigned_abs() as usize).sum::(), 1 << table_log, "{norm:?}"); - if norm.iter().filter(|&&n| n != 0).count() > 1 { write_ncount_both(rng, st, &norm, max as u32, table_log).unwrap(); } - let (ct, rt) = use_fse_table(rng, st, &norm, max as u32, table_log); - made.push((ct, rt, norm)); - } - // A run-length table in place of one of them, built over what is there. - if rng.chance(1, 3) { - let i = rng.below(3); - let symbol = rng.below(53) as u8; - let (ct, rt, norm) = &mut made[i]; - assert_eq!(unsafe { FSE_buildCTable_rle(ct.as_mut_ptr(), symbol) }, 0); - rt.build_rle(symbol).unwrap(); - let t = rt.to_c(); - // Only the header, the two states and the transform of the symbol are defined. - assert_eq!(ct[..2], t[..2], "FSE_buildCTable_rle"); - assert_eq!(ct[2 + 2 * symbol as usize..][..2], t[2 + 2 * symbol as usize..][..2], "FSE_buildCTable_rle"); - *norm = vec![0; 53]; norm[symbol as usize] = 1; - st.hit(); - let len = rng.below(40); - let src = vec![symbol; len]; - let cap = rng.below(60); - let mut out = Out::new(cap); - let a = unsafe { FSE_compress_usingCTable(out.c_ptr(), cap, src.as_ptr().cast(), len, ct.as_ptr()) }; - let b = rt.compress(&mut out.r, &src); - assert_eq!(a, b, "FSE_compress_usingCTable rle"); - out.check("FSE_compress_usingCTable rle", &(len, cap)); - st.hit(); - } - let n = 1 + rng.size(2000); - let syms: Vec> = made.iter().map(|(_, _, norm)| data_among(rng, norm, n)).collect(); - let bound = n * 4 + 32; - let cap = match rng.below(4) { 0 => rng.below(20), 1 => rng.below(bound), _ => bound }; - let mut out = Out::new(cap); - let a = res(unsafe { shim_fse_encode3(out.c_ptr(), cap, made[0].0.as_ptr(), made[1].0.as_ptr(), made[2].0.as_ptr(), syms[0].as_ptr(), syms[1].as_ptr(), syms[2].as_ptr(), n) }); - let b = conv(rs::fse_encode3(&mut out.r, [&made[0].1, &made[1].1, &made[2].1], [&syms[0], &syms[1], &syms[2]])); - assert_eq!(a, b, "encode3 n {n} cap {cap}"); - out.check("encode3", &(n, cap)); - st.hit(); - - // Too large for the table. - let norm = random_norm(rng, 52, 10); - let mut ct = c_fse_table(); - let mut w = wksp(); - // The workspace that `ZSTD_buildCTable` has. - let a = res(unsafe { FSE_buildCTable_wksp(ct.as_mut_ptr(), norm.as_ptr(), 52, 10, w.as_mut_ptr().cast(), 1140) }); - assert_eq!(a, conv(rs::Fse::new().build(&norm, 52, 10)).map(|_| 0)); - st.hit(); -} - -/// Counts that are not normalized. -fn t_fse_bad_norm(rng: &mut Rng, st: &mut Stats) { - let table_log = rng.below(15) as u32; - let max = rng.range(1, 255); - let mut norm = if (5..=12).contains(&table_log) { random_norm(rng, max, table_log) } else { vec![1; max + 1] }; - for _ in 0..rng.below(4) { - let i = rng.below(norm.len()); - norm[i] = match rng.below(4) { 0 => 0, 1 => -1, 2 => norm[i].wrapping_add(rng.range(0, 6) as i16 - 3), _ => rng.below(1 << table_log.min(12)) as i16 }; - } - // A large buffer: past the bound, the C code does not check what it writes. - let cap = 1024; - let mut out = Out::new(cap); - let a = res(unsafe { FSE_writeNCount(out.c_ptr(), cap, norm.as_ptr(), max as u32, table_log) }); - let b = conv(rs::fse_write_ncount(&mut out.r, &norm, max as u32, table_log)); - assert_eq!(a, b, "FSE_writeNCount bad: log {table_log} {norm:?}"); - out.check("FSE_writeNCount bad", &(table_log, &norm)); - st.hit(); -} - -// ---------------------------------------------------------------- huf - -fn bmi2(rng: &mut Rng) -> c_int { (std::arch::is_x86_feature_detected!("bmi2") && rng.chance(1, 2)) as c_int } - -fn c_build(count: &[u32], max: u32, bits: u32, init: &rs::Huf) -> (R, rs::Huf) { - let mut t = *init; - let mut w = wksp(); - let r = res(unsafe { HUF_buildCTable_wksp(t.as_mut_ptr(), count.as_ptr(), max, bits, w.as_mut_ptr().cast(), WKSP_SIZE) }); - (r, t) -} - -fn using_ctable_both(rng: &mut Rng, st: &mut Stats, t: &rs::Huf, src: &[u8]) { - let log = t[0] as u8 as usize; - let tight = ((src.len() * log) >> 3) + 8; - for four in [false, true] { - for _ in 0..2 { - let cap = match rng.below(8) { - 0 => tight, 1 => tight.saturating_sub(1), 2 => tight + 1 + rng.below(30), 3 => rng.below(tight + 1), 4 => rng.below(24), - 5 => tight / 4 + rng.below(40), _ => unsafe { HUF_compressBound(src.len()) }, - }; - let mut out = Out::new(cap); - let f = bmi2(rng); - let (a, b) = if four { - (res(unsafe { HUF_compress4X_usingCTable(out.c_ptr(), cap, src.as_ptr().cast(), src.len(), t.as_ptr(), f) }), conv(rs::huf_compress_4x_using_ctable(&mut out.r, src, t))) - } else { - (res(unsafe { HUF_compress1X_usingCTable(out.c_ptr(), cap, src.as_ptr().cast(), src.len(), t.as_ptr(), f) }), conv(rs::huf_compress_1x_using_ctable(&mut out.r, src, t))) - }; - assert_eq!(a, b, "HUF_compress{}X_usingCTable len {} cap {cap} log {log}", if four { 4 } else { 1 }, src.len()); - out.check("HUF_compressNX_usingCTable", &(four, src.len(), cap, log)); - st.hit(); - } - } -} - -fn t_huf_table(rng: &mut Rng, st: &mut Stats) { - let max = match rng.below(4) { 0 => 255, 1 => rng.range(0, 255), 2 => rng.range(0, 40), _ => rng.range(100, 255) }; - let budget = *rng.pick(&[10usize, 100, 1000, 5000, 20000, 131072, 131072]); - let count = counts(rng, max, budget); - let max = if rng.chance(3, 4) { count.iter().rposition(|&c| c != 0).unwrap() } else { max } as u32; - let total: usize = count.iter().map(|&c| c as usize).sum(); - - let card = unsafe { HUF_cardinality(count.as_ptr(), max) }; - assert_eq!(card, rs::huf_cardinality(&count, max), "HUF_cardinality"); - let min_log = unsafe { HUF_minTableLog(card) }; - assert_eq!(min_log, rs::huf_min_table_log(card)); - st.hit(); - - let bits = match rng.below(6) { 0 => 0, 1 => 11, 2 => min_log, _ => rng.range(min_log as usize, 12) as u32 }; - let mut init = [0u64; 257]; - if rng.chance(1, 2) { for x in init.iter_mut() { *x = rng.next(); } } - let (a, ct) = c_build(&count, max, bits, &init); - let mut rt = init; - let b = conv(rs::huf_build_ctable(&mut rt, &count, max, bits)); - assert_eq!(a, b, "HUF_buildCTable_wksp max {max} bits {bits} {:?}", &count[..=max as usize]); - assert!(ct == rt, "HUF_buildCTable_wksp table: max {max} bits {bits} {:?}", &count[..=max as usize]); - st.hit(); - let log = a.unwrap() as u32; - - // Out of range. - if rng.chance(1, 8) { - let (a, t) = c_build(&count, max, 13, &init); - let mut tr = init; - assert_eq!(a, conv(rs::huf_build_ctable(&mut tr, &count, max, 13)), "HUF_buildCTable_wksp 13 bits"); - assert!(t == tr); - let big = 256 + rng.below(3) as u32; - let (a, t) = c_build(&count, big, bits, &init); - let mut tr = init; - assert_eq!(a, conv(rs::huf_build_ctable(&mut tr, &count, big, bits)), "HUF_buildCTable_wksp max {big}"); - assert!(t == tr); - let mut out = Out::new(300); - let mut w = wksp(); - let a = res(unsafe { HUF_writeCTable_wksp(out.c_ptr(), 300, ct.as_ptr(), big, log, w.as_mut_ptr().cast(), WKSP_SIZE) }); - assert_eq!(a, conv(rs::huf_write_ctable(&mut out.r, &rt, big, log)), "HUF_writeCTable_wksp max {big}"); - out.check("HUF_writeCTable_wksp", &big); - st.cases += 3; - } - - let h = unsafe { HUF_readCTableHeader(ct.as_ptr()) }; - assert_eq!((h.table_log, h.max_symbol_value), rs::huf_read_ctable_header(&rt)); - for s in 0..256 { assert_eq!(unsafe { HUF_getNbBitsFromCTable(ct.as_ptr(), s) }, rs::huf_get_nb_bits(&rt, s)); } - assert_eq!(unsafe { HUF_estimateCompressedSize(ct.as_ptr(), count.as_ptr(), max) }, rs::huf_estimate_compressed_size(&rt, &count, max)); - st.cases += 3; - for _ in 0..3 { - let max2 = rng.below(256); - let other = if rng.chance(1, 2) { counts(rng, max2, 1000) } else { let mut o = count.clone(); let i = rng.below(256); o[i] = rng.below(2) as u32; o }; - let max2 = if rng.chance(1, 2) { max2 as u32 } else { max }; - assert_eq!(unsafe { HUF_validateCTable(ct.as_ptr(), other.as_ptr(), max2) } != 0, rs::huf_validate_ctable(&rt, &other, max2), "HUF_validateCTable"); - assert_eq!(unsafe { HUF_estimateCompressedSize(ct.as_ptr(), other.as_ptr(), max2) }, rs::huf_estimate_compressed_size(&rt, &other, max2)); - st.cases += 2; - } - - // The header. - let mut header = Vec::new(); - for k in 0..4 { - let cap = match k { 0 => 300, 1 => rng.below(140), 2 => rng.below(20), _ => 129 + rng.below(3) }; - let mut out = Out::new(cap); - let mut w = wksp(); - let a = res(unsafe { HUF_writeCTable_wksp(out.c_ptr(), cap, ct.as_ptr(), max, log, w.as_mut_ptr().cast(), WKSP_SIZE) }); - let b = conv(rs::huf_write_ctable(&mut out.r, &rt, max, log)); - assert_eq!(a, b, "HUF_writeCTable_wksp cap {cap} max {max} log {log}"); - out.check("HUF_writeCTable_wksp", &(cap, max, log)); - st.hit(); - if let (0, Ok(n)) = (k, a) { header = out.r[..n].to_vec(); } - } - if !header.is_empty() { - header.extend((0..rng.below(8)).map(|_| rng.next() as u8)); - let any = rng.below(256) as u32; - let limit = *rng.pick(&[255, max, max.saturating_sub(1), any]); - read_ctable_both(rng, st, &header, limit); - let n = rng.below(header.len() + 1); - read_ctable_both(rng, st, &header[..n], 255); - let i = rng.below(header.len()); - header[i] = rng.next() as u8; - read_ctable_both(rng, st, &header, 255); - } - - // The best limit on the depth. - if total > 1 && card > 1 { - for flags in [0, 2] { - let max_log = if flags == 0 { rng.below(13) as u32 } else { rng.range(min_log as usize, 12) as u32 }; - let (mut tc, mut tr) = (init, init); - let mut w = wksp(); - // The sizes that the two callers give. - let wsize = *rng.pick(&[4864usize, 7896]); - let a = unsafe { HUF_optimalTableLog(max_log, total, max, w.as_mut_ptr().cast(), wsize, tc.as_mut_ptr(), count.as_ptr(), flags) }; - let b = rs::huf_optimal_table_log(max_log, total, max, &mut tr, &count, flags as u32); - assert_eq!(a, b, "HUF_optimalTableLog flags {flags} max_log {max_log} total {total} max {max} {:?}", &count[..=max as usize]); - assert!(tc == tr, "HUF_optimalTableLog table"); - st.hit(); - } - } - - // Encoding. - let src = data_of(rng, &count); - let n = if rng.chance(1, 2) { src.len() } else { rng.below(src.len() + 1) }; - using_ctable_both(rng, st, &ct, &src[..n]); -} - -fn read_ctable_both(rng: &mut Rng, st: &mut Stats, src: &[u8], max: u32) { - let mut init = [0u64; 257]; - if rng.chance(1, 2) { for x in init.iter_mut() { *x = rng.next(); } } - let (mut tc, mut tr) = (init, init); - let (mut mc, mut zc) = (max, 7u32); - // The C code may look at bytes after the end, without using them. - let mut padded = src.to_vec(); padded.extend_from_slice(&[0; 16]); - let a = res(unsafe { HUF_readCTable(tc.as_mut_ptr(), &mut mc, padded.as_ptr().cast(), src.len(), &mut zc) }); - let b = conv(rs::huf_read_ctable(&mut tr, max, src)); - match (a, b) { - (Ok(n), Ok((size, m, z))) => assert_eq!((n, mc, zc != 0), (size, m, z), "HUF_readCTable"), - (a, b) => assert_eq!(a.err(), b.err(), "HUF_readCTable error: {src:?}"), - } - assert!(tc == tr, "HUF_readCTable table: {src:?}"); - st.hit(); -} - -fn t_huf_read(rng: &mut Rng, st: &mut Stats) { - let len = rng.below(140); - let mut src: Vec = (0..len).map(|_| rng.next() as u8).collect(); - if len > 0 { - match rng.below(3) { - 0 => src[0] = 128 + rng.below(128) as u8, - 1 => { src[0] = 128 + rng.below(128) as u8; for b in src[1..].iter_mut() { *b = (rng.below(4) << 4 | rng.below(4)) as u8; } } - _ => {} - } - } - let limit = *rng.pick(&[255, 255, 100, 10]); - read_ctable_both(rng, st, &src, limit); -} - -fn literals(rng: &mut Rng, len: usize) -> Vec { - match rng.below(6) { - 0 | 1 => data(rng, len), - 2 => { let max = rng.range(1, 255); let c = counts(rng, max, len.max(1)); let mut v = data_of(rng, &c); v.truncate(len); v } - 3 => { // nearly incompressible - let m = rng.range(100, 256); (0..len).map(|_| rng.below(m) as u8).collect() - } - 4 => { // compressible in the middle only - let mut v: Vec = (0..len).map(|_| rng.next() as u8).collect(); - if len > 8192 { for b in v[4096..len - 4096].iter_mut() { *b &= 7; } } - v - } - _ => { let k = rng.range(1, 8); (0..len).map(|_| { let mut b = rng.next() as u8; for _ in 0..k { b &= rng.next() as u8; } b }).collect() } - } -} - -fn t_huf_repeat(rng: &mut Rng, st: &mut Stats) { - let len = sizes(rng) + if rng.chance(1, 200) { 131073 } else { 0 }; - let src = literals(rng, len); - let len = src.len(); - let mut hist = vec![0u32; 256]; - for &b in &src { hist[b as usize] += 1; } - let actual = src.iter().copied().max().unwrap_or(0) as u32; - let card = hist.iter().filter(|&&c| c != 0).count() as u32; - - let flags = (rng.below(8) << 1) as u32; - let four = rng.chance(1, 2); - - // The previous table. - let zero = [0u64; 257]; - let (prev, mut repeat) = match rng.below(7) { - 0 => (zero, 0u32), - 1 if len > 0 => (c_build(&hist, actual, 11, &zero).1, rng.range(0, 2) as u32), // of this very input - 2 => { // has every symbol - let mut c = hist.clone(); for x in c.iter_mut() { *x = *x / rng.range(1, 3) as u32 + 1; } - (c_build(&c, 255, *rng.pick(&[11, 12, 9]), &zero).1, rng.range(0, 2) as u32) - } - 3 => { let c: Vec = (0..256).map(|_| 1 + rng.size(300) as u32).collect(); (c_build(&c, 255, 11, &zero).1, rng.range(0, 2) as u32) } - 4 => { // of other data: may lack symbols, so that it has to be checked - let m = rng.range(1, 255); let c = counts(rng, m, 5000); - let last = c.iter().rposition(|&x| x != 0).unwrap() as u32; - (c_build(&c, last, 11, &zero).1, rng.range(0, 1) as u32) - } - 5 if len > 0 => { // of a part of this input - let mut c = vec![0u32; 256]; for &b in &src[..1 + rng.below(len)] { c[b as usize] += 1; } - let last = c.iter().rposition(|&x| x != 0).unwrap() as u32; - (c_build(&c, last, 11, &zero).1, rng.range(0, 1) as u32) - } - _ => (zero, 0), - }; - let sampled = flags & 8 != 0 && len >= 40960; - let max = match rng.below(6) { - 0 => 0, - 1 => rng.range(actual as usize, 255) as u32, - 2 if !sampled => rng.below(256) as u32, - 3 => *rng.pick(&[256u32, 300]), - _ => 255, - }; - let min_log = if card > 0 { unsafe { HUF_minTableLog(card) } } else { 1 }; - let log = match rng.below(5) { - 0 => 0, - 1 => 11, - 2 => 13, - _ if flags & 2 != 0 => rng.range(min_log as usize, 12) as u32, - _ => rng.range(1, 12) as u32, - }; - let cap = match rng.below(8) { - 0 => rng.below(24), - 1 => rng.below(len + 1), - 2 => len, - 3 => len / 2 + rng.below(200), - _ => unsafe { HUF_compressBound(len) }, - }; - - let mut out = Out::new(cap); - let (mut tc, mut tr) = (prev, prev); - let (mut rc, mut rr) = (repeat, repeat); - let mut w = wksp(); - let cf = flags as c_int | bmi2(rng); - let f = if four { HUF_compress4X_repeat } else { HUF_compress1X_repeat }; - let a = res(unsafe { f(out.c_ptr(), cap, src.as_ptr().cast(), len, max, log, w.as_mut_ptr().cast(), WKSP_SIZE, tc.as_mut_ptr(), &mut rc, cf) }); - let b = conv(rs::huf_compress_repeat(four, &mut out.r, &src, max, log, &mut tr, &mut rr, flags)); - let ctx = (four, len, cap, max, log, flags, repeat); - assert_eq!((a, rc), (b, rr), "HUF_compressNX_repeat {ctx:?}"); - out.check("HUF_compressNX_repeat", &ctx); - assert!(tc == tr, "HUF_compressNX_repeat table {ctx:?}"); - st.hit(); - - // The next block, with the table that this one left, as the literals of a frame. - if let Ok(n) = a && n > 1 { - repeat = if rc == 0 { 1 } else { rc }; - let len2 = sizes(rng); - let mut src2 = if rng.chance(1, 2) { literals(rng, len2) } else { let mut v = src.clone(); v.truncate(len2); v.reverse(); v }; - if rng.chance(1, 4) && !src2.is_empty() { let i = rng.below(src2.len()); src2[i] = rng.next() as u8; } - let cap = unsafe { HUF_compressBound(src2.len()) }; - let mut out = Out::new(cap); - let (mut rc, mut rr) = (repeat, repeat); - let mut w = wksp(); - let a = res(unsafe { f(out.c_ptr(), cap, src2.as_ptr().cast(), src2.len(), 255, 11, w.as_mut_ptr().cast(), WKSP_SIZE, tc.as_mut_ptr(), &mut rc, cf) }); - let b = conv(rs::huf_compress_repeat(four, &mut out.r, &src2, 255, 11, &mut tr, &mut rr, flags)); - assert_eq!((a, rc), (b, rr), "HUF_compressNX_repeat second {ctx:?}"); - out.check("HUF_compressNX_repeat second", &ctx); - assert!(tc == tr, "HUF_compressNX_repeat second table {ctx:?}"); - st.hit(); - } -} - -/// `which`: `all`, or one of the names below. -pub fn run(seed: u64, iters: u64, which: &str) { - assert_eq!(rs::HUF_CONSTANTS, (8704, 131072, 11, 8)); - assert_eq!(rs::FSE_NCOUNTBOUND, 512); - let tests: &[(&str, fn(&mut Rng, &mut Stats))] = &[ - ("hist", t_hist), ("fse_logs", t_fse_logs), ("fse", t_fse), ("fse_tables", t_fse_tables), ("fse_bad_norm", t_fse_bad_norm), - ("huf_table", t_huf_table), ("huf_read", t_huf_read), ("huf_repeat", t_huf_repeat), - ]; - let mut totals = vec![0u64; tests.len()]; - for i in 0..iters { - for (k, (name, test)) in tests.iter().enumerate() { - if which != "all" && which != *name { continue; } - let s = seed.wrapping_mul(1_000_003).wrapping_add(i).wrapping_mul(31).wrapping_add(k as u64); - if std::env::var_os("ENTROPY_TRACE").is_some() { eprintln!("{name} seed {s}"); } - let mut st = Stats { cases: 0 }; - test(&mut Rng(s), &mut st); - totals[k] += st.cases; - } - } - for ((name, _), n) in tests.iter().zip(&totals) { if *n > 0 { println!("{name}: {n} comparisons"); } } - println!("entropy ok: seed {seed}, {iters} iterations, {} comparisons", totals.iter().sum::()); -} - -// ---------------------------------------------------------------- speed - -fn time(name: &str, bytes: usize, mut c: impl FnMut() -> usize, mut r: impl FnMut() -> usize) { - let mut best = [f64::MAX; 2]; - let reps = (200_000_000 / bytes.max(1)).clamp(20, 200_000) / 20; - for _ in 0..7 { - let t = std::time::Instant::now(); - for _ in 0..reps { std::hint::black_box(c()); } - best[0] = best[0].min(t.elapsed().as_secs_f64() / reps as f64); - let t = std::time::Instant::now(); - for _ in 0..reps { std::hint::black_box(r()); } - best[1] = best[1].min(t.elapsed().as_secs_f64() / reps as f64); - } - println!("{name:40} C {:10.0} ns Rust {:10.0} ns ratio {:.2}", best[0] * 1e9, best[1] * 1e9, best[1] / best[0]); -} - -/// Build with `--profile bench`. -pub fn bench() { - let mut rng = Rng(7); - let text: Vec = { let mut v = Vec::new(); while v.len() < 131072 { let n = 1 + rng.size(5000); let mut r2 = Rng(rng.next() | 2); let mut part = Vec::new(); while part.len() < n { part.extend_from_slice(format!("const x{} = await foo({});\n", r2.below(50), r2.below(1000)).as_bytes()); } v.extend_from_slice(&part); } v.truncate(131072); v }; - let skew = |k: usize, rng: &mut Rng| -> Vec { (0..131072).map(|_| { let mut b = rng.next() as u8; for _ in 0..k { b &= rng.next() as u8; } b }).collect() }; - let inputs: Vec<(&str, Vec)> = vec![("text", text), ("skew1", skew(1, &mut rng)), ("skew3", skew(3, &mut rng)), ("random", skew(0, &mut rng))]; - let mut w = wksp(); - let wp: *mut c_void = w.as_mut_ptr().cast(); - for (name, src) in &inputs { - for len in [131072usize, 4000, 300] { - let src = &src[..len]; - let p: *const c_void = src.as_ptr().cast(); - let (mut cc, mut cr) = (vec![0u32; 256], vec![0u32; 256]); - time(&format!("HIST_count_wksp {name} {len}"), len, - || { let mut m = 255; unsafe { HIST_count_wksp(cc.as_mut_ptr(), &mut m, p, len, wp, 4096) } }, - || { let mut m = 255; rs::hist_count_wksp(&mut cr, &mut m, src).unwrap() }); - let max = cc.iter().rposition(|&c| c != 0).unwrap() as u32; - let mut t = [0u64; 257]; - let mut bt = bun_zstd::testing::entropy_bench::Table::new(&t); - time(&format!("HUF_buildCTable_wksp {name} {len}"), 2000, - || unsafe { HUF_buildCTable_wksp(t.as_mut_ptr(), cc.as_ptr(), max, 11, wp, WKSP_SIZE) }, - || bt.build(&cr, max, 11)); - let log = t[0] as u8 as u32; - let cap = unsafe { HUF_compressBound(len) }; - let mut dst = vec![0u8; cap]; - let mut dst2 = vec![0u8; cap]; - time(&format!("HUF_writeCTable_wksp {name} {len}"), 2000, - || unsafe { HUF_writeCTable_wksp(dst.as_mut_ptr().cast(), cap, t.as_ptr(), max, log, wp, WKSP_SIZE) }, - || bt.write(&mut dst2, max, log)); - for bits in [11u32, 10, 9, 8] { - let mut t = [0u64; 257]; - if res(unsafe { HUF_buildCTable_wksp(t.as_mut_ptr(), cc.as_ptr(), max, bits, wp, WKSP_SIZE) }).is_err() { continue; } - if (t[0] as u8 as u32) != bits { continue; } - let bt = bun_zstd::testing::entropy_bench::Table::new(&t); - time(&format!("HUF_compress4X_usingCTable {name} {len} log {bits}"), len, - || unsafe { HUF_compress4X_usingCTable(dst.as_mut_ptr().cast(), cap, p, len, t.as_ptr(), 0) }, - || bt.compress4(&mut dst2, src)); - } - for flags in [0u32, 2] { - time(&format!("HUF_compress4X_repeat {name} {len} flags {flags}"), len, - || { let mut rep = 0; unsafe { HUF_compress4X_repeat(dst.as_mut_ptr().cast(), cap, p, len, 255, 11, wp, WKSP_SIZE, t.as_mut_ptr(), &mut rep, flags as c_int) } }, - || bt.repeat4(&mut dst2, src, flags == 2)); - } - } - } - // Sequences. - for (max, log, n) in [(35u32, 9u32, 5000usize), (52, 9, 5000), (31, 8, 5000), (35, 6, 60)] { - let count = { let mut c = vec![0u32; 256]; for _ in 0..n { let a = rng.below(max as usize + 1); let b = rng.below(max as usize + 1); c[a.min(b)] += 1; } c }; - let max = count.iter().rposition(|&c| c != 0).unwrap() as u32; - let log = unsafe { FSE_optimalTableLog(log, n, max) }; - let (mut nc, mut nr) = (vec![0i16; 256], vec![0i16; 256]); - time(&format!("FSE_normalizeCount {max} {log}"), 500, - || unsafe { FSE_normalizeCount(nc.as_mut_ptr(), log, count.as_ptr(), n, max, 1) }, - || rs::fse_normalize_count(&mut nr, log, &count, n, max, true).unwrap()); - let mut buf = [0u8; 512]; - let mut buf2 = [0u8; 512]; - time(&format!("FSE_writeNCount {max} {log}"), 500, - || unsafe { FSE_writeNCount(buf.as_mut_ptr().cast(), 512, nc.as_ptr(), max, log) }, - || rs::fse_write_ncount(&mut buf2, &nr, max, log).unwrap()); - let mut ct = c_fse_table(); - let mut rt = rs::Fse::new(); - time(&format!("FSE_buildCTable_wksp {max} {log}"), 500, - || unsafe { FSE_buildCTable_wksp(ct.as_mut_ptr(), nc.as_ptr(), max, log, wp, WKSP_SIZE) }, - || { rt.build(&nr, max, log).unwrap(); 0 }); - let syms = data_among(&mut rng, &nc[..=max as usize], n); - let cap = n * 4 + 64; - let (mut d1, mut d2) = (vec![0u8; cap], vec![0u8; cap]); - time(&format!("encode3 {max} {log} n {n}"), n * 3, - || unsafe { shim_fse_encode3(d1.as_mut_ptr().cast(), cap, ct.as_ptr(), ct.as_ptr(), ct.as_ptr(), syms.as_ptr(), syms.as_ptr(), syms.as_ptr(), n) }, - || rs::fse_encode3(&mut d2, [&rt, &rt, &rt], [&syms, &syms, &syms]).unwrap()); - } -} diff --git a/src/zstd/wip/work/entropy/harness/src/main.rs b/src/zstd/wip/work/entropy/harness/src/main.rs deleted file mode 100644 index e54cc30aa80e..000000000000 --- a/src/zstd/wip/work/entropy/harness/src/main.rs +++ /dev/null @@ -1,22 +0,0 @@ -mod c; -mod data; -mod entropy; -mod seqs; - -fn main() { - let args: Vec = std::env::args().collect(); - let cmd = args.get(1).map(String::as_str).unwrap_or(""); - let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); - let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); - match cmd { - // seqs - "seqs" => { - let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); - seqs::run(seed, iters, &strategies, args.get(5).is_some_and(|s| s == "1")); - } - // entropy [all | the name of a test] - "entropy" => entropy::run(seed, iters, args.get(4).map(String::as_str).unwrap_or("all")), - "entropy-bench" => entropy::bench(), - _ => eprintln!("unknown command"), - } -} diff --git a/src/zstd/wip/work/entropy/harness/src/seqs.rs b/src/zstd/wip/work/entropy/harness/src/seqs.rs deleted file mode 100644 index 7c03d3b0f952..000000000000 --- a/src/zstd/wip/work/entropy/harness/src/seqs.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! One block through a Rust block compressor and through the C library (`ZSTD_generateSequences`). -use crate::c; -use crate::data::{Rng, data}; -use std::ffi::{c_uint, c_void}; - -#[repr(C)] -#[derive(Clone, Copy, Default, Debug)] -struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } -#[repr(C)] -#[derive(Clone, Copy, Debug)] -struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } -unsafe extern "C" { - fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; - fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; - fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; -} - -fn c_sequences(cp: [u32; 7], use_row: bool, dict: &[u8], src: &[u8]) -> Vec<(u32, u32, u32)> { - let mut cctx = c::CCtx::new(); - for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } - cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder - cctx.set(1010, 2).unwrap(); // no block splitter - if !dict.is_empty() { cctx.ref_prefix(dict).unwrap(); } - let mut out = vec![Sequence::default(); src.len() / 3 + 8]; - let n = c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }).unwrap(); - out[..n].iter().map(|s| (s.lit_length, s.match_length, s.offset)).collect() -} - -/// `strategies`: which of 1..=9 to test. `with_dict`: also test with a prefix, which is the extDict variants. -pub fn run(seed: u64, iters: u64, strategies: &[u32], with_dict: bool) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if i % 500 == 0 { eprintln!("seqs iter {i} (seed {s})"); } - let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071]); - let len = rng.range(7, max); - let mut src = data(&mut rng, len); - let dict = if with_dict && rng.chance(1, 2) { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; - if !dict.is_empty() && rng.chance(2, 3) { - for _ in 0..rng.range(1, 6) { - let n = rng.size(dict.len().min(src.len())); - let at = rng.below(dict.len() - n + 1); - let to = rng.below(src.len() - n + 1); - src[to..to + n].copy_from_slice(&dict[at..at + n]); - } - } - let strategy = *rng.pick(strategies); - let p = if rng.chance(1, 3) { - // the parameters of a level, with the strategy forced - let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; - p.strategy = strategy; - p - } else { - CParams { - window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, - search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, - target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, - } - }; - let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; - let cp = [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy]; - let use_row = rng.chance(1, 2); - let expected = c_sequences(cp, use_row, &dict, &src); - let got = bun_zstd::testing::first_block(cp, use_row, &dict, &src); - let got = got.unwrap_or_else(|e| panic!("seed {s}: error {e:?} (cp {cp:?} row {use_row} dict {} src {})", dict.len(), src.len())); - if got != expected { - let at = got.iter().zip(&expected).position(|(a, b)| a != b).unwrap_or(got.len().min(expected.len())); - panic!("seed {s}: cp {cp:?} row {use_row} dict {} src {}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", - dict.len(), src.len(), got.get(at), expected.get(at), got.len(), expected.len()); - } - } - println!("seqs ok: {iters} iterations from seed {seed}, strategies {strategies:?}, dict {with_dict}"); -} diff --git a/src/zstd/wip/work/entropy/zstd/lib.rs b/src/zstd/wip/work/entropy/zstd/lib.rs deleted file mode 100644 index 31c98c72e5e4..000000000000 --- a/src/zstd/wip/work/entropy/zstd/lib.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the -//! same bytes in, in both directions, and the same errors. -#![forbid(unsafe_code)] -// Frames come from anywhere and a panic aborts the process. -#![deny( - clippy::indexing_slicing, - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::unreachable -)] -#![warn(unused_must_use)] - -mod common; -mod compress; -mod decompress; -mod error; -mod util; - -pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; -pub use decompress::{ - DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, - find_frame_compressed_size, get_frame_content_size, -}; -pub use error::ErrorCode; - -/// `ZSTD_inBuffer` -pub struct InBuffer<'a> { - pub src: &'a [u8], - /// How much of `src` is read. - pub pos: usize, -} - -/// `ZSTD_outBuffer` -pub struct OutBuffer<'a> { - pub dst: &'a mut [u8], - /// How much of `dst` is written. - pub pos: usize, -} - -/// `ZSTD_ResetDirective` -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum ResetDirective { - SessionOnly = 1, - Parameters = 2, - SessionAndParameters = 3, -} - -// ----------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] -pub enum ZstdError { - /// The output, or the decoder state the frame's window size dictates, could not be allocated. - OutOfMemory, - InvalidZstdData, - DecompressionFailed, - ZstdDecompressionError, - ShortRead, -} - -impl core::fmt::Display for ZstdError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.write_str(<&'static str>::from(self)) - } -} -impl core::error::Error for ZstdError {} - -impl ZstdError { - /// The error for a failed decompression call; `other` is the non-allocation failure. - fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { - if code == ErrorCode::MemoryAllocation { - ZstdError::OutOfMemory - } else { - other - } - } -} - -/// `len` zeros for a decoder to write to. -fn output_buffer(len: usize) -> Result, ZstdError> { - util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) -} - -/// Decompress data, automatically allocating the output buffer. -/// Handles both frames with known and unknown content sizes. -/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. -/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. -pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { - let size = find_decompressed_size(src); - - const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit - - if size == ZSTD_CONTENTSIZE_ERROR { - return Err(ZstdError::InvalidZstdData); - } - - // Use streaming decompression if: - // 1. Content size is unknown, OR - // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) - if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { - let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { - // A frame's output is rarely smaller than its input. - src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) - } else { - // The header size is untrusted: reserve no more than the fast path below would. - MAX_PREALLOCATE_SIZE - }; - let mut list: Vec = Vec::new(); - list.try_reserve_exact(initial_capacity) - .map_err(|_| ZstdError::OutOfMemory)?; - StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; - return Ok(list); - } - - // Fast path: size is known and within reasonable limits - let mut output = output_buffer(size as usize)?; - let written = decompress(&mut output, src) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is -/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. -pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { - const MAX_SIZE: u64 = 16 * 1024 * 1024; - let size = find_decompressed_size(src); - // Also if it is unknown or invalid, which are the two largest values. - if size > MAX_SIZE { - return Err(ZstdError::InvalidZstdData); - } - let mut output = output_buffer(size as usize)?; - let written = DDict::prefix(prefix) - .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum State { - Uninitialized, - Inflating, - End, - Error, -} - -/// Minimum spare output capacity offered to the decoder per call. -const STREAMING_OUTPUT_STEP: usize = 4096; - -// ────────────────────────────────────────────────────────────────────────── -// StreamingDecoder -// ────────────────────────────────────────────────────────────────────────── - -/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) -/// per call, so callers can hold the decoder across multiple body chunks. -pub struct StreamingDecoder { - stream: DCtx, - state: State, - /// The decoder filled its last window and may hold more. `max_output` can end a call there. - output_full: bool, -} - -impl Default for StreamingDecoder { - fn default() -> Self { - Self::new() - } -} - -impl StreamingDecoder { - pub fn new() -> Self { - Self { - stream: DCtx::new(), - state: State::Uninitialized, - output_full: false, - } - } - - #[inline] - pub fn is_inflating(&self) -> bool { - matches!(self.state, State::Inflating) - } - - /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is - /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. - /// Returns `ShortRead` when more input is required and `is_done` is false. - pub fn decompress( - &mut self, - input: &[u8], - out: &mut Vec, - max_output: usize, - is_done: bool, - ) -> Result { - if matches!(self.state, State::End | State::Error) { - return Ok(input.len()); - } - - let mut in_buf = InBuffer { src: input, pos: 0 }; - while matches!(self.state, State::Uninitialized | State::Inflating) { - let next_in_len = input.len() - in_buf.pos; - - // Call the decoder again with no input until it leaves the window short. - if next_in_len == 0 && !self.output_full { - if is_done { - if self.state == State::Inflating { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - self.state = State::End; - } - return Ok(in_buf.pos); - } - - if out.len() >= max_output { - return Ok(in_buf.pos); - } - - if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { - self.state = State::Error; - return Err(ZstdError::OutOfMemory); - } - // The decoder writes to initialized memory only. - let written_before = out.len(); - let window = (out.capacity() - written_before).min(max_output - written_before); - out.resize(written_before + window, 0); - let mut out_buf = OutBuffer { - dst: out.get_mut(written_before..).unwrap_or_default(), - pos: 0, - }; - let read_before = in_buf.pos; - - let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); - let bytes_written = out_buf.pos; - out.truncate(written_before + bytes_written); - let remaining = match result { - Ok(remaining) => remaining, - Err(code) => { - self.state = State::Error; - return Err(ZstdError::for_decompression( - code, - ZstdError::ZstdDecompressionError, - )); - } - }; - - self.output_full = bytes_written == window; - - if remaining == 0 { - // Frame complete, and fully flushed. - self.state = State::Uninitialized; - self.output_full = false; - if in_buf.pos >= input.len() { - if is_done { - self.state = State::End; - } - return Ok(in_buf.pos); - } - // More input available — reinitialize for the next frame. - self.stream.init_stream(); - continue; - } - - self.state = State::Inflating; - - if in_buf.pos - read_before == next_in_len { - if self.output_full { - continue; - } - if is_done { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - return Err(ZstdError::ShortRead); - } - } - Ok(in_buf.pos) - } -} - -// ── compressed embedded assets ──────────────────────────────────────────── - -/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes -/// Bun never executes or parses itself: the shell completion scripts and the -/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, -/// error overlay/page). Anything that runs inside Bun stays uncompressed. -/// -/// The exception is text that only `bun lint`, `bun fmt` and `bun check` -/// read. It may be compressed because it isn't on the path of running a program. -/// -/// Release (`bun_codegen_embed`) builds include the `.zst` twin that -/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to -/// `/compressed/.zst`; other builds evaluate `$fallback`. -#[macro_export] -macro_rules! embed_compressed { - // A codegen output (`/$sub`); debug builds read it at runtime. - (codegen $sub:literal) => { - $crate::embed_compressed!( - ("codegen/", $sub), - ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() - ) - }; - // Same, with a trailing NUL included in the slice (for C-string consumers). - (codegen_nul $sub:literal) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/codegen/", - $sub, - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); - &__COPY.get_or_init(|| { - let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); - let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); - copy.extend_from_slice(text); - copy.push(0); - copy.into_boxed_slice() - })[..] - } - }; - __bytes - }}; - // A file under `src/`; debug builds read it from the source tree at runtime. - (src $sub:literal) => { - $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) - }; - ($name:literal, $fallback:expr) => { - $crate::embed_compressed!(($name), $fallback) - }; - (($($name:literal),+), $fallback:expr) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/", - $($name,)+ - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - $fallback - } - }; - __bytes - }}; -} - -/// Cold, shared body of [`embed_compressed!`]'s release arm. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { - decompress_alloc(compressed) - .expect("embedded asset: invalid zstd frame") - .into_boxed_slice() -} - -/// [`inflate_embedded`] plus a trailing NUL byte. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { - let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); - inflated.reserve_exact(1); - inflated.push(0); - inflated.into_boxed_slice() -} -pub mod testing; diff --git a/src/zstd/wip/work/entropy/zstd/testing.rs b/src/zstd/wip/work/entropy/zstd/testing.rs deleted file mode 100644 index dd4e06c95a2d..000000000000 --- a/src/zstd/wip/work/entropy/zstd/testing.rs +++ /dev/null @@ -1,457 +0,0 @@ -//! Hooks for the differential tests. Only in scratch copies of the crate. -#![allow(clippy::all, clippy::indexing_slicing, dead_code, unreachable_pub)] - -use crate::ErrorCode; -use crate::compress::internal::*; -use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; - -fn bitmix(mut val: u64, len: u64) -> u64 { - val ^= val.rotate_right(49) ^ val.rotate_right(24); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^= (val >> 35).wrapping_add(len); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^ (val >> 28) -} - -/// The sequences of the first block of a frame, as (literal length, match length, offset), and then the literals -/// that are left as (length, 0, 0). -/// -/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, already adjusted to the sizes. -/// `dict` is a prefix (raw content, as `ZSTD_CCtx_refPrefix`), or empty. `src` is from 7 to 131071 bytes. -pub fn first_block( - cp: [u32; 7], - use_row: bool, - dict: &[u8], - src: &[u8], -) -> Result, ErrorCode> { - let strategy = Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?; - let c_params = CompressionParameters { - window_log: cp[0], - chain_log: cp[1], - hash_log: cp[2], - search_log: cp[3], - min_match: cp[4], - target_length: cp[5], - strategy: Some(strategy), - }; - let row_used = row_match_finder_supported(strategy) && use_row; - - // ZSTD_reset_matchState - let mut ms = MatchState::new(); - ms.hash_log3 = if c_params.min_match == 3 { - c_params.window_log.min(17) - } else { - 0 - }; - ms.window.clear(); - ms.next_to_update = ms.window.dict_limit; - ms.hash_table = vec![0; 1 << c_params.hash_log]; - ms.chain_table = vec![ - 0; - if strategy != Strategy::Fast && !row_used { - 1 << c_params.chain_log - } else { - 0 - } - ]; - ms.hash_table3 = vec![ - 0; - if ms.hash_log3 != 0 { - 1 << ms.hash_log3 - } else { - 0 - } - ]; - if row_used { - ms.tag_table = vec![0; 1 << c_params.hash_log]; - ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4); - ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); - } - if strategy >= Strategy::BtOpt { - ms.opt.lit_freq = vec![0; 256]; - ms.opt.lit_length_freq = vec![0; 36]; - ms.opt.match_length_freq = vec![0; 53]; - ms.opt.off_code_freq = vec![0; 32]; - ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; - ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; - } - ms.c_params = c_params; - - let block_size = (1usize << c_params.window_log).min(128 << 10); - let mut seq_store = SeqStore::new(); - seq_store.reserve( - block_size / if c_params.min_match == 3 { 3 } else { 4 }, - block_size, - )?; - - let bufs = Buffers { - in_buff: &[], - user: src, - dict, - }; - - // ZSTD_loadDictionaryContent(dtlm_fast, tfp_forCCtx) - if !dict.is_empty() { - let iend = dict.len(); - ms.window - .update(BufId::Dict.start(), BufId::Dict, dict.len(), false); - let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); - let ip = iend - dict.len().min(max_dict_size); - let prefix = Prefix { - buf: dict, - base: ms.window.base.wrapping_sub(BufId::Dict.start()), - }; - ms.next_to_update = prefix.index_of(ip); - ms.loaded_dict_end = prefix.index_of(iend); - if iend - ip > HASH_READ_SIZE { - match strategy { - Strategy::Fast => fast::fill_hash_table( - &mut ms, - prefix, - iend, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - )?, - Strategy::DFast => double_fast::fill_double_hash_table( - &mut ms, - prefix, - iend, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - )?, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { - if use_row { - ms.tag_table.fill(0); - lazy::row_update(&mut ms, prefix, iend - HASH_READ_SIZE)?; - } else { - lazy::insert_and_find_first_index(&mut ms, prefix, iend - HASH_READ_SIZE)?; - } - } - _ => opt::update_tree(&mut ms, prefix, iend - HASH_READ_SIZE, iend)?, - } - ms.next_to_update = prefix.index_of(iend); - } - } - - // ZSTD_compressContinue_internal - let src_addr = BufId::User.start(); - if !ms - .window - .update(src_addr, BufId::User, src.len(), ms.force_non_contiguous) - { - ms.force_non_contiguous = false; - ms.next_to_update = ms.window.dict_limit; - } - // ZSTD_compress_frameChunk - let max_dist = 1u32 << c_params.window_log; - if ms - .window - .dict_is_invalid(src_addr + src.len(), max_dist, ms.loaded_dict_end) - { - ms.loaded_dict_end = 0; - } - if ms - .window - .enforce_max_dist(src_addr, max_dist, ms.loaded_dict_end) - { - ms.loaded_dict_end = 0; - } - if ms.next_to_update < ms.window.low_limit { - ms.next_to_update = ms.window.low_limit; - } - // ZSTD_buildSeqStore - let prev = CompressedBlockState::new(); - let block = BlockSrc { - buf: bufs.get(ms.window.base_buf), - base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), - dict_buf: bufs.get(ms.window.dict_buf), - dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), - start: 0, - end: src.len(), - dms: None, - symbol_costs: &prev.entropy, - ldm_seq_store: None, - }; - let curr = block.index_of(0); - if curr > ms.next_to_update + 384 { - ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); - } - let dict_mode = ms.dict_mode(None); - let mut rep = prev.rep; - let last_ll = compress::compress_block( - &mut ms, - &mut seq_store, - &mut rep, - &block, - strategy, - use_row, - dict_mode, - )?; - - let mut out = Vec::new(); - let mut reps = prev.rep; - for (i, seq) in seq_store.sequences.iter().enumerate() { - let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); - if seq_store.long_length_pos as usize == i { - match seq_store.long_length_type { - LongLengthType::LiteralLength => ll += 0x10000, - LongLengthType::MatchLength => ml += 0x10000, - LongLengthType::None => {} - } - } - update_rep(&mut reps, seq.off_base, seq.lit_length == 0); - out.push((ll, ml, reps[0])); - } - out.push((last_ll as u32, 0, 0)); - assert_eq!( - reps, rep, - "the repeat offsets the block compressor leaves are those of its sequences" - ); - Ok(out) -} - -/// The entropy encoders. A `HufCTable` is passed as its array, which is what the C code has, and an `FseCTable` is -/// converted to the layout of the C code. -pub mod entropy { - use crate::ErrorCode; - use crate::common::bitstream::BitCStream; - use crate::compress::fse::{self, FseCState, FseCTable}; - use crate::compress::hist; - use crate::compress::huf::{self, HufCTable, HufFlags, HufRepeat}; - - type R = Result; - pub type Huf = [u64; 257]; - - pub fn hist_add(count: &mut [u32], src: &[u8]) -> R<()> { - hist::add(count, src) - } - pub fn hist_count_simple(count: &mut [u32], max: &mut u32, src: &[u8]) -> R { - hist::count_simple(count, max, src) - } - pub fn hist_count_fast_wksp(count: &mut [u32], max: &mut u32, src: &[u8]) -> R { - hist::count_fast_wksp(count, max, src) - } - pub fn hist_count_wksp(count: &mut [u32], max: &mut u32, src: &[u8]) -> R { - hist::count_wksp(count, max, src) - } - - pub struct Fse(FseCTable); - impl Fse { - pub fn new() -> Self { - Self(FseCTable::new()) - } - pub fn build(&mut self, norm: &[i16], max: u32, table_log: u32) -> R<()> { - self.0.build(norm, max, table_log) - } - pub fn build_rle(&mut self, symbol: u8) -> R<()> { - self.0.build_rle(symbol) - } - pub fn get_max_nb_bits(&self, symbol: u32) -> u32 { - self.0.get_max_nb_bits(symbol) - } - pub fn bit_cost(&self, symbol: u32, accuracy_log: u32) -> u32 { - self.0.bit_cost(symbol, accuracy_log) - } - /// The `U32`s of the C table, up to the last symbol's transform. - pub fn to_c(&self) -> Vec { - let t = &self.0; - let mut v = vec![t.table_log | t.max_symbol_value << 16]; - let states = if t.table_log != 0 { - 1usize << t.table_log - } else { - 2 - }; - for pair in t.state_table[..states].chunks(2) { - v.push(pair[0] as u32 | (pair[1] as u32) << 16); - } - for tt in &t.symbol_tt[..=t.max_symbol_value as usize] { - v.push(tt.delta_find_state as u32); - v.push(tt.delta_nb_bits); - } - v - } - pub fn compress(&self, dst: &mut [u8], src: &[u8]) -> usize { - fse::compress_using_ctable(dst, src, &self.0) - } - } - /// Three states on one stream, as the sequence encoder has. - pub fn fse_encode3(dst: &mut [u8], tables: [&Fse; 3], symbols: [&[u8]; 3]) -> R { - let n = symbols[0].len(); - let mut bit_c = BitCStream::new(dst)?; - let mut a = FseCState::new2(&tables[0].0, symbols[0][n - 1] as u32); - let mut b = FseCState::new2(&tables[1].0, symbols[1][n - 1] as u32); - let mut c = FseCState::new2(&tables[2].0, symbols[2][n - 1] as u32); - for i in (0..n - 1).rev() { - b.encode_symbol(&mut bit_c, symbols[1][i] as u32); - a.encode_symbol(&mut bit_c, symbols[0][i] as u32); - c.encode_symbol(&mut bit_c, symbols[2][i] as u32); - bit_c.flush_bits(); - } - a.flush(&mut bit_c); - b.flush(&mut bit_c); - c.flush(&mut bit_c); - let d = FseCState::new(&tables[0].0); - d.flush(&mut bit_c); - Ok(bit_c.close()) - } - pub const FSE_NCOUNTBOUND: usize = fse::FSE_NCOUNTBOUND; - pub fn fse_ncount_write_bound(max: u32, table_log: u32) -> usize { - fse::ncount_write_bound(max, table_log) - } - pub fn fse_write_ncount(dst: &mut [u8], norm: &[i16], max: u32, table_log: u32) -> R { - fse::write_ncount(dst, norm, max, table_log) - } - pub fn fse_optimal_table_log(max_log: u32, size: usize, max: u32) -> u32 { - fse::optimal_table_log(max_log, size, max) - } - pub fn fse_optimal_table_log_internal(max_log: u32, size: usize, max: u32, minus: u32) -> u32 { - fse::optimal_table_log_internal(max_log, size, max, minus) - } - pub fn fse_normalize_count( - norm: &mut [i16], - table_log: u32, - count: &[u32], - total: usize, - max: u32, - low_prob: bool, - ) -> R { - fse::normalize_count(norm, table_log, count, total, max, low_prob) - } - - fn flags(f: u32) -> HufFlags { - HufFlags { - optimal_depth: f & 2 != 0, - prefer_repeat: f & 4 != 0, - suspect_uncompressible: f & 8 != 0, - } - } - fn with(t: &mut Huf, f: impl FnOnce(&mut HufCTable) -> T) -> T { - let mut ct = HufCTable(*t); - let r = f(&mut ct); - *t = ct.0; - r - } - pub const HUF_CONSTANTS: (usize, usize, u32, u32) = ( - huf::HUF_WORKSPACE_SIZE, - huf::HUF_BLOCKSIZE_MAX, - huf::HUF_TABLELOG_DEFAULT, - huf::HUF_OPTIMAL_DEPTH_THRESHOLD as u32, - ); - pub fn huf_compress_bound(size: usize) -> usize { - huf::compress_bound(size) - } - pub fn huf_build_ctable(t: &mut Huf, count: &[u32], max: u32, max_nb_bits: u32) -> R { - with(t, |t| huf::build_ctable(t, count, max, max_nb_bits)) - } - pub fn huf_write_ctable(dst: &mut [u8], t: &Huf, max: u32, huff_log: u32) -> R { - huf::write_ctable(dst, &HufCTable(*t), max, huff_log) - } - /// (size, max symbol value, has zero weights) - pub fn huf_read_ctable(t: &mut Huf, max: u32, src: &[u8]) -> R<(usize, u32, bool)> { - with(t, |t| huf::read_ctable(t, max, src)) - .map(|s| (s.size, s.max_symbol_value, s.has_zero_weights)) - } - pub fn huf_estimate_compressed_size(t: &Huf, count: &[u32], max: u32) -> usize { - huf::estimate_compressed_size(&HufCTable(*t), count, max) - } - pub fn huf_validate_ctable(t: &Huf, count: &[u32], max: u32) -> bool { - huf::validate_ctable(&HufCTable(*t), count, max) - } - pub fn huf_optimal_table_log( - max_log: u32, - size: usize, - max: u32, - t: &mut Huf, - count: &[u32], - f: u32, - ) -> u32 { - with(t, |t| { - huf::optimal_table_log(max_log, size, max, t, count, flags(f)) - }) - } - pub fn huf_min_table_log(cardinality: u32) -> u32 { - huf::min_table_log(cardinality) - } - pub fn huf_cardinality(count: &[u32], max: u32) -> u32 { - huf::cardinality(count, max) - } - pub fn huf_get_nb_bits(t: &Huf, symbol: u32) -> u32 { - HufCTable(*t).get_nb_bits(symbol) - } - pub fn huf_read_ctable_header(t: &Huf) -> (u8, u8) { - let h = huf::read_ctable_header(&HufCTable(*t)); - (h.table_log, h.max_symbol_value) - } - pub fn huf_compress_1x_using_ctable(dst: &mut [u8], src: &[u8], t: &Huf) -> R { - huf::compress_1x_using_ctable(dst, src, &HufCTable(*t)) - } - pub fn huf_compress_4x_using_ctable(dst: &mut [u8], src: &[u8], t: &Huf) -> R { - huf::compress_4x_using_ctable(dst, src, &HufCTable(*t)) - } - /// `repeat` is 0, 1 or 2, as `HUF_repeat`. `f` has the bits of `HUF_flags_e`. - pub fn huf_compress_repeat( - four: bool, - dst: &mut [u8], - src: &[u8], - max: u32, - huff_log: u32, - t: &mut Huf, - repeat: &mut u32, - f: u32, - ) -> R { - let mut rep = match *repeat { - 0 => HufRepeat::None, - 1 => HufRepeat::Check, - _ => HufRepeat::Valid, - }; - let r = with(t, |t| { - if four { - huf::compress_4x_repeat(dst, src, max, huff_log, t, &mut rep, flags(f)) - } else { - huf::compress_1x_repeat(dst, src, max, huff_log, t, &mut rep, flags(f)) - } - }); - *repeat = match rep { - HufRepeat::None => 0, - HufRepeat::Check => 1, - HufRepeat::Valid => 2, - }; - r - } -} - -/// For timing: no copy of the table. -pub mod entropy_bench { - use crate::compress::huf::{self, HufCTable, HufFlags, HufRepeat}; - pub struct Table(HufCTable); - impl Table { - pub fn new(t: &[u64; 257]) -> Self { - Self(HufCTable(*t)) - } - pub fn build(&mut self, count: &[u32], max: u32, bits: u32) -> usize { - huf::build_ctable(&mut self.0, count, max, bits).unwrap_or(0) - } - pub fn write(&self, dst: &mut [u8], max: u32, log: u32) -> usize { - huf::write_ctable(dst, &self.0, max, log).unwrap_or(0) - } - pub fn compress4(&self, dst: &mut [u8], src: &[u8]) -> usize { - huf::compress_4x_using_ctable(dst, src, &self.0).unwrap_or(0) - } - pub fn repeat4(&mut self, dst: &mut [u8], src: &[u8], optimal_depth: bool) -> usize { - let mut rep = HufRepeat::None; - huf::compress_4x_repeat( - dst, - src, - 255, - 11, - &mut self.0, - &mut rep, - HufFlags { - optimal_depth, - ..Default::default() - }, - ) - .unwrap_or(0) - } - } -} diff --git a/src/zstd/wip/work/fast/STATUS.md b/src/zstd/wip/work/fast/STATUS.md deleted file mode 100644 index a02c065a8a2d..000000000000 --- a/src/zstd/wip/work/fast/STATUS.md +++ /dev/null @@ -1,35 +0,0 @@ -# fast / dfast port: status (12:27 UTC) - -## Deliverable -- `/root/zstd-ref/work/fast/zstd/compress/fast.rs` -- `/root/zstd-ref/work/fast/zstd/compress/double_fast.rs` - -Complete, `cargo fmt --check` clean, no clippy finding in either file. `double_fast.rs` uses four `pub(super)` helpers -of `fast.rs` (`FAST_HASH_FILL_STEP`, `Located`, `count_backwards`, `restore_rep`): take both files together. - -## IMPORTANT: the copy of double_fast.rs in the real repo is a deliberately broken mutant -The copy taken into `/root/repos/oven-sh/bun/src/zstd/compress/double_fast.rs` was taken while my mutation test was -rewriting the file in place. It has `match_index_s >= prefix_lowest_index` (line 423) where the right code, as in C, is -`match_index_s > prefix_lowest_index`. That is the dfast + attached CDict mismatch of the end-to-end test. Copy both -files again from the paths above. (`fast.rs` in the repo only differs by `let buf = src.buf;`, see below.) - -## Verified against C (all pass) -Harness: `cd /root/zstd-ref/work/fast/harness && RUSTFLAGS="--cap-lints=warn" cargo build --release` -(NOT `--profile bench`, which writes the same `target/release` without overflow checks). -- `zstd_harness seqs 1,2 <0|1>`: template test, fixed (several blocks, param ids, rep[2], short last - block). 330,000 iterations for each of dict 0 and 1 on the first version, and 320,000 each on the final files. -- `zstd_harness stream `: mine (`c/shim.c`, `src/stream.rs`, `TestMs` in `zstd/testing.rs`). Calls the C - functions directly on hand-made `ZSTD_MatchState_t`: all 6 block compressors and both fill functions with all 4 - (purpose, load method) pairs; attached dictionaries (dictMatchState), loaded dictionaries (contiguous or not), chunks - that are contiguous / elsewhere / overwrite the same buffer (ring buffer), blocks of any size from 1 byte, start - indices up to 3.6e9, repeat offsets at the limits. After every call compares sequences, literals, rep, return value, - long length and both tables. 220,000+ iterations on the final files (about 80,000 dictMatchState blocks per strategy). -- `zstd_harness frames 1,2 <0|1|2>`: copy of the harness of work/lazy (real CDict machinery). - 100,000 iterations of mode 2, 40,000 of mode 1. -- Line coverage of the two files by `stream`: everything but the 4 `end < 8` early returns of the fill functions. -- 45 hand-made mutants (boundary conditions in every function): all caught by `stream`. - -## Open -- Speed (bench profile, 23 MB of Rust source, `zstd_harness bench `): Rust takes 1.4x (level 1), 1.27x - (level 2), 1.2x (levels 3, 4) the time of C; dictMatchState 1.15-1.35x. See the hand-back for where the time goes. -- Nothing open for correctness. diff --git a/src/zstd/wip/work/fast/harness/Cargo.toml b/src/zstd/wip/work/fast/harness/Cargo.toml deleted file mode 100644 index bba38b8e8cf6..000000000000 --- a/src/zstd/wip/work/fast/harness/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "zstd_harness" -version = "0.0.0" -edition = "2024" - -[dependencies] -bun_zstd = { path = "/root/zstd-ref/work/fast/zstd" } - -[profile.release] -debug = "line-tables-only" -overflow-checks = true -debug-assertions = true -panic = "abort" - -[profile.bench] -inherits = "release" -overflow-checks = false -debug-assertions = false - -[workspace] diff --git a/src/zstd/wip/work/fast/harness/build.rs b/src/zstd/wip/work/fast/harness/build.rs deleted file mode 100644 index 86bd64eee215..000000000000 --- a/src/zstd/wip/work/fast/harness/build.rs +++ /dev/null @@ -1,21 +0,0 @@ -use std::process::Command; - -fn main() { - let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); - println!("cargo:rerun-if-env-changed=ZSTD_REF"); - println!("cargo:rerun-if-changed=c/shim.c"); - println!("cargo:rerun-if-changed=c/shim_cctx.inc"); - let out = std::env::var("OUT_DIR").unwrap(); - let ok = Command::new("clang-23") - .args(["-c", "-O2", "-g", "-fPIC", "-march=nehalem", "-DZSTD_MULTITHREAD", "-DZSTD_LEGACY_SUPPORT=0", "-DXXH_NAMESPACE=ZSTD_"]) - .args(["-I/root/zstd-ref/zstd/lib", "-I/root/zstd-ref/zstd/lib/common", "c/shim.c", "-o"]) - .arg(format!("{out}/shim.o")).status().unwrap().success(); - assert!(ok); - let _ = std::fs::remove_file(format!("{out}/libshim.a")); - let ok = Command::new("llvm-ar-23").arg("rcs").arg(format!("{out}/libshim.a")).arg(format!("{out}/shim.o")).status().unwrap().success(); - assert!(ok); - println!("cargo:rustc-link-search=native={out}"); - println!("cargo:rustc-link-lib=static=shim"); - println!("cargo:rustc-link-search=native=/root/zstd-ref/build/{variant}"); - println!("cargo:rustc-link-lib=static=zstd"); -} diff --git a/src/zstd/wip/work/fast/harness/c/shim.c b/src/zstd/wip/work/fast/harness/c/shim.c deleted file mode 100644 index fb4ef9f981d4..000000000000 --- a/src/zstd/wip/work/fast/harness/c/shim.c +++ /dev/null @@ -1,169 +0,0 @@ -/* Drives the block compressors of zstd_fast.c and zstd_double_fast.c on match states made by hand. */ -#define ZSTD_STATIC_LINKING_ONLY -#include "compress/zstd_compress_internal.h" -#include "compress/zstd_fast.h" -#include "compress/zstd_double_fast.h" -#include -#include - -typedef struct { - ZSTD_MatchState_t ms; - SeqStore_t seqStore; -} Shim; - -typedef struct { - U64 base, dictBase, nextSrc; - U32 dictLimit, lowLimit, loadedDictEnd, nextToUpdate, dictMode, pad; -} ShimState; - -Shim* shim_new(const U32* cp, size_t maxBlock) -{ - Shim* const s = (Shim*)calloc(1, sizeof(*s)); - ZSTD_window_init(&s->ms.window); - s->ms.cParams.windowLog = cp[0]; - s->ms.cParams.chainLog = cp[1]; - s->ms.cParams.hashLog = cp[2]; - s->ms.cParams.searchLog = cp[3]; - s->ms.cParams.minMatch = cp[4]; - s->ms.cParams.targetLength = cp[5]; - s->ms.cParams.strategy = (ZSTD_strategy)cp[6]; - s->ms.hashTable = (U32*)calloc((size_t)1 << cp[2], sizeof(U32)); - s->ms.chainTable = (U32*)calloc((size_t)1 << cp[1], sizeof(U32)); - s->ms.nextToUpdate = s->ms.window.dictLimit; - s->seqStore.maxNbSeq = maxBlock / 3 + 1; - s->seqStore.maxNbLit = maxBlock; - s->seqStore.sequencesStart = (SeqDef*)malloc(s->seqStore.maxNbSeq * sizeof(SeqDef)); - s->seqStore.litStart = (BYTE*)malloc(maxBlock + WILDCOPY_OVERLENGTH); - ZSTD_resetSeqStore(&s->seqStore); - return s; -} - -void shim_free(Shim* s) -{ - free(s->ms.hashTable); free(s->ms.chainTable); - free(s->seqStore.sequencesStart); free(s->seqStore.litStart); - free(s); -} - -void shim_state(const Shim* s, ShimState* st) -{ - st->base = (U64)(size_t)s->ms.window.base; - st->dictBase = (U64)(size_t)s->ms.window.dictBase; - st->nextSrc = (U64)(size_t)s->ms.window.nextSrc; - st->dictLimit = s->ms.window.dictLimit; - st->lowLimit = s->ms.window.lowLimit; - st->loadedDictEnd = s->ms.loadedDictEnd; - st->nextToUpdate = s->ms.nextToUpdate; - st->dictMode = (U32)ZSTD_matchState_dictMode(&s->ms); - st->pad = 0; -} - -const U32* shim_hash_table(const Shim* s) { return s->ms.hashTable; } -const U32* shim_chain_table(const Shim* s) { return s->ms.chainTable; } - -/* Indices go on from `index`, as after many frames. */ -void shim_bump(Shim* s, U32 index) -{ - if (s->ms.window.dictLimit < index) { - s->ms.window.nextSrc = s->ms.window.base + index; - ZSTD_window_clear(&s->ms.window); - s->ms.nextToUpdate = s->ms.window.dictLimit; - } -} - -/* ZSTD_loadDictionaryContent. Returns whether the tables were filled; `before` is the state right before that. */ -int shim_load_dict(Shim* s, const void* src, size_t srcSize, int dtlm, int tfp, int forceNonContiguous, ShimState* before) -{ - ZSTD_MatchState_t* const ms = &s->ms; - const BYTE* ip = (const BYTE*)src; - const BYTE* const iend = ip + srcSize; - { U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; - if (tfp == ZSTD_tfp_forCDict) { - U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; - maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize); - } - if (srcSize > maxDictSize) { ip = iend - maxDictSize; src = ip; srcSize = maxDictSize; } - } - ZSTD_window_update(&ms->window, src, srcSize, 0); - { U32 maxDictSize = 1U << MIN(MAX(ms->cParams.hashLog + 3, ms->cParams.chainLog + 1), 31); - if (srcSize > maxDictSize) { ip = iend - maxDictSize; src = ip; srcSize = maxDictSize; } - } - ms->nextToUpdate = (U32)(ip - ms->window.base); - ms->loadedDictEnd = (U32)(iend - ms->window.base); - ms->forceNonContiguous = forceNonContiguous; - shim_state(s, before); - if (srcSize <= HASH_READ_SIZE) return 0; - if (ms->cParams.strategy == ZSTD_fast) - ZSTD_fillHashTable(ms, iend, (ZSTD_dictTableLoadMethod_e)dtlm, (ZSTD_tableFillPurpose_e)tfp); - else - ZSTD_fillDoubleHashTable(ms, iend, (ZSTD_dictTableLoadMethod_e)dtlm, (ZSTD_tableFillPurpose_e)tfp); - ms->nextToUpdate = (U32)(iend - ms->window.base); - return 1; -} - -/* The end of ZSTD_resetCCtx_byAttachingCDict. Returns whether the dictionary is attached. */ -int shim_attach(Shim* s, const Shim* dict) -{ - U32 const cdictEnd = (U32)(dict->ms.window.nextSrc - dict->ms.window.base); - U32 const cdictLen = cdictEnd - dict->ms.window.dictLimit; - if (cdictLen == 0) return 0; - s->ms.dictMatchState = &dict->ms; - if (s->ms.window.dictLimit < cdictEnd) { - s->ms.window.nextSrc = s->ms.window.base + cdictEnd; - ZSTD_window_clear(&s->ms.window); - } - s->ms.loadedDictEnd = s->ms.window.dictLimit; - return 1; -} - -/* The start of ZSTD_compressContinue_internal. Returns whether the input is contiguous. */ -int shim_update(Shim* s, const void* src, size_t srcSize) -{ - ZSTD_MatchState_t* const ms = &s->ms; - if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { - ms->forceNonContiguous = 0; - ms->nextToUpdate = ms->window.dictLimit; - return 0; - } - return 1; -} - -/* What ZSTD_compress_frameChunk does before each block. */ -void shim_pre_block(Shim* s, const void* src, size_t blockSize) -{ - ZSTD_MatchState_t* const ms = &s->ms; - const BYTE* const ip = (const BYTE*)src; - U32 const maxDist = (U32)1 << ms->cParams.windowLog; - ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); - ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); - if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; -} - -size_t shim_block(Shim* s, U32* rep, const void* src, size_t srcSize) -{ - ZSTD_MatchState_t* const ms = &s->ms; - ZSTD_dictMode_e const mode = ZSTD_matchState_dictMode(ms); - ZSTD_resetSeqStore(&s->seqStore); - if (ms->cParams.strategy == ZSTD_fast) { - switch (mode) { - case ZSTD_noDict: return ZSTD_compressBlock_fast(ms, &s->seqStore, rep, src, srcSize); - case ZSTD_extDict: return ZSTD_compressBlock_fast_extDict(ms, &s->seqStore, rep, src, srcSize); - default: return ZSTD_compressBlock_fast_dictMatchState(ms, &s->seqStore, rep, src, srcSize); - } - } - switch (mode) { - case ZSTD_noDict: return ZSTD_compressBlock_doubleFast(ms, &s->seqStore, rep, src, srcSize); - case ZSTD_extDict: return ZSTD_compressBlock_doubleFast_extDict(ms, &s->seqStore, rep, src, srcSize); - default: return ZSTD_compressBlock_doubleFast_dictMatchState(ms, &s->seqStore, rep, src, srcSize); - } -} - -size_t shim_nb_seq(const Shim* s) { return (size_t)(s->seqStore.sequences - s->seqStore.sequencesStart); } -const SeqDef* shim_seqs(const Shim* s) { return s->seqStore.sequencesStart; } -size_t shim_lit_len(const Shim* s) { return (size_t)(s->seqStore.lit - s->seqStore.litStart); } -const BYTE* shim_lits(const Shim* s) { return s->seqStore.litStart; } -U32 shim_long_type(const Shim* s) { return (U32)s->seqStore.longLengthType; } -U32 shim_long_pos(const Shim* s) { return s->seqStore.longLengthPos; } - -/* From the harness of the lazy strategies: the state of a whole CCtx. */ -#include "shim_cctx.inc" diff --git a/src/zstd/wip/work/fast/harness/c/shim_cctx.inc b/src/zstd/wip/work/fast/harness/c/shim_cctx.inc deleted file mode 100644 index 25c2831c1206..000000000000 --- a/src/zstd/wip/work/fast/harness/c/shim_cctx.inc +++ /dev/null @@ -1,30 +0,0 @@ - -typedef struct { - unsigned cp[7]; - unsigned useRow; - unsigned hasDms; - unsigned nextToUpdate, loadedDictEnd, lowLimit, dictLimit, rowHashLog, hashSaltEntropy, lazySkipping; - unsigned long long hashSalt; - unsigned hashCache[8]; - unsigned rep[3]; - const unsigned* hashTable; - const unsigned* chainTable; - const unsigned char* tagTable; -} ShimCCtxState; - -void shim_cctx_state(const ZSTD_CCtx* c, ShimCCtxState* s) { - const ZSTD_MatchState_t* ms = &c->blockState.matchState; - const ZSTD_compressionParameters* p = &c->appliedParams.cParams; - int i; - s->cp[0] = p->windowLog; s->cp[1] = p->chainLog; s->cp[2] = p->hashLog; s->cp[3] = p->searchLog; - s->cp[4] = p->minMatch; s->cp[5] = p->targetLength; s->cp[6] = (unsigned)p->strategy; - s->useRow = c->appliedParams.useRowMatchFinder == ZSTD_ps_enable; - s->hasDms = ms->dictMatchState != NULL; - s->nextToUpdate = ms->nextToUpdate; s->loadedDictEnd = ms->loadedDictEnd; - s->lowLimit = ms->window.lowLimit; s->dictLimit = ms->window.dictLimit; - s->rowHashLog = ms->rowHashLog; s->hashSaltEntropy = ms->hashSaltEntropy; s->lazySkipping = (unsigned)ms->lazySkipping; - s->hashSalt = ms->hashSalt; - for (i = 0; i < 8; i++) s->hashCache[i] = ms->hashCache[i]; - for (i = 0; i < 3; i++) s->rep[i] = c->blockState.prevCBlock->rep[i]; - s->hashTable = ms->hashTable; s->chainTable = ms->chainTable; s->tagTable = ms->tagTable; -} diff --git a/src/zstd/wip/work/fast/harness/rust-toolchain.toml b/src/zstd/wip/work/fast/harness/rust-toolchain.toml deleted file mode 100644 index c71778e8ccd3..000000000000 --- a/src/zstd/wip/work/fast/harness/rust-toolchain.toml +++ /dev/null @@ -1,35 +0,0 @@ -[toolchain] -channel = "nightly-2026-09-15" -# rust-src is needed for -Zbuild-std (Tier 3 targets like -# aarch64-unknown-freebsd have no prebuilt std). miri is for -# `bun run rust:miri`. targets ensures the -# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain -# (CI's images install this same channel with these components and -# targets, from scripts/build/ci-images/spec.ts; a source lint keeps -# the two files in step). Note: rustup ignores this whole file when -# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree -# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` -# self-heals that path. Keep this list as a fallback for direct -# `cargo build`/`cargo check`/rust-analyzer use. -# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd -# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). -# darwin/windows triples build natively on their own agents, so the host std -# comes with the toolchain install regardless; they're listed here so a dev -# running `bun run rust:check-all` or `cargo check --target` from any host -# still gets prebuilt std for every Tier 1/2 triple. -# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at -# what it emits. The build does not use them. -components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] -targets = [ - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "x86_64-unknown-linux-musl", - "aarch64-linux-android", - "x86_64-linux-android", - "x86_64-unknown-freebsd", - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-pc-windows-msvc", - "x86_64-pc-windows-msvc", -] diff --git a/src/zstd/wip/work/fast/harness/src/c.rs b/src/zstd/wip/work/fast/harness/src/c.rs deleted file mode 100644 index d5a82e1f6b78..000000000000 --- a/src/zstd/wip/work/fast/harness/src/c.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! The reference C library. -#![allow(non_camel_case_types, dead_code)] -use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; - -#[repr(C)] -pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } -#[repr(C)] -pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } -pub enum ZSTD_CCtx {} -pub enum ZSTD_DCtx {} -pub enum ZSTD_DDict {} - -unsafe extern "C" { - pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; - pub fn ZSTD_compressBound(size: usize) -> usize; - pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_isError(code: usize) -> c_uint; - pub fn ZSTD_getErrorCode(code: usize) -> c_uint; - pub fn ZSTD_getErrorName(code: usize) -> *const c_char; - pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; - pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; - pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; - pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; - pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; - pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; - pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; - pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; - pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; - pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; - pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; - pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; - pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; - pub fn ZSTD_defaultCLevel() -> c_int; - pub fn ZSTD_minCLevel() -> c_int; - pub fn ZSTD_maxCLevel() -> c_int; -} - -/// `Ok(value)` or `Err(error code)`. -pub fn res(code: usize) -> Result { - unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } -} - -pub struct CCtx(pub *mut ZSTD_CCtx); -impl CCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } - pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - /// Returns (result, out pos, in pos). - pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); - (r, o.pos, i.pos) - } -} -impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } - -pub struct DCtx(pub *mut ZSTD_DCtx); -impl DCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } - pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } - pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); - (r, o.pos, i.pos) - } -} -impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } - -pub fn compress(src: &[u8], level: i32) -> Vec { - let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; - let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); - dst.truncate(n); - dst -} -pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) -} diff --git a/src/zstd/wip/work/fast/harness/src/data.rs b/src/zstd/wip/work/fast/harness/src/data.rs deleted file mode 100644 index c48a645e1b5d..000000000000 --- a/src/zstd/wip/work/fast/harness/src/data.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Deterministic inputs. -pub struct Rng(pub u64); -impl Rng { - pub fn next(&mut self) -> u64 { - // splitmix64 - self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); - z ^ (z >> 31) - } - pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } - pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } - pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } - pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } - /// Skewed towards small values. - pub fn size(&mut self, max: usize) -> usize { - let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); - (self.next() as usize & ((1usize << bits) - 1)).min(max) - } -} - -static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; - -/// One buffer of about `len` bytes, of a kind picked by `rng`. -pub fn data(rng: &mut Rng, len: usize) -> Vec { - let mut v = Vec::with_capacity(len + 64); - match rng.below(9) { - 0 => v.resize(len, rng.next() as u8), - 1 => while v.len() < len { v.push(rng.next() as u8) }, - 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, - 3 => { // few symbols, skewed - let n = rng.range(2, 40); - while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } - } - 4 => { // LZ-like: copies of earlier parts, at all distances - while v.len() < len { - if v.len() > 8 && rng.chance(3, 4) { - let dist = 1 + rng.size(v.len() - 1); - let n = 3 + rng.size(300); - for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } - } else { - for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } - } - } - } - 5 => { // periodic with noise - let period = rng.range(1, 300); - let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); - while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } - } - 6 => { // sections of different kinds - while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } - } - 7 => { // numbers as text - let mut x = rng.next() % 1000; - while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } - } - _ => { // long runs - while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } - } - } - v.truncate(len); - v -} diff --git a/src/zstd/wip/work/fast/harness/src/frames.rs b/src/zstd/wip/work/fast/harness/src/frames.rs deleted file mode 100644 index eeb73a560cae..000000000000 --- a/src/zstd/wip/work/fast/harness/src/frames.rs +++ /dev/null @@ -1,178 +0,0 @@ -//! A frame through the Rust block compressors and through the C library (`ZSTD_generateSequences`). -use crate::c; -use crate::data::{Rng, data}; -use bun_zstd::testing_frame::{Dict, State}; -use std::ffi::{c_uint, c_void}; - -#[repr(C)] -#[derive(Clone, Copy, Default, Debug)] -struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } -#[repr(C)] -#[derive(Clone, Copy, Debug)] -struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } -#[repr(C)] -#[derive(Clone, Copy)] -struct CustomMem { a: *const c_void, b: *const c_void, c: *const c_void } -#[repr(C)] -struct ShimState { - cp: [c_uint; 7], use_row: c_uint, has_dms: c_uint, - next_to_update: c_uint, loaded_dict_end: c_uint, low_limit: c_uint, dict_limit: c_uint, row_hash_log: c_uint, hash_salt_entropy: c_uint, lazy_skipping: c_uint, - hash_salt: u64, hash_cache: [c_uint; 8], rep: [c_uint; 3], - hash_table: *const u32, chain_table: *const u32, tag_table: *const u8, -} -enum CDict {} -enum CCtxParams {} -unsafe extern "C" { - fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; - fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; - fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; - fn ZSTD_createCCtxParams() -> *mut CCtxParams; - fn ZSTD_freeCCtxParams(p: *mut CCtxParams) -> usize; - fn ZSTD_CCtxParams_setParameter(p: *mut CCtxParams, param: c_uint, v: i32) -> usize; - fn ZSTD_createCDict_advanced2(d: *const c_void, n: usize, load: c_uint, ty: c_uint, p: *const CCtxParams, mem: CustomMem) -> *mut CDict; - fn ZSTD_freeCDict(d: *mut CDict) -> usize; - fn ZSTD_CCtx_refCDict(c: *mut c::ZSTD_CCtx, d: *const CDict) -> usize; - fn ZSTD_getCParamsFromCDict(d: *const CDict) -> CParams; - fn shim_cctx_state(c: *const c::ZSTD_CCtx, s: *mut ShimState); -} - -fn arr(p: CParams) -> [u32; 7] { [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy] } - -#[derive(Clone, Copy, PartialEq, Debug)] -pub enum Mode { NoDict, Prefix, CDict } - -struct COut { seqs: Vec<(u32, u32, u32, u32)>, cp: [u32; 7], use_row: bool, cdict_cp: [u32; 7], state: State } - -/// `None` if the C library refuses (a last block that is too small). -fn c_frame(cp: [u32; 7], use_row: bool, mode: Mode, dict: &[u8], src: &[u8]) -> Option { - let mut cctx = c::CCtx::new(); - let mut cdict = std::ptr::null_mut(); - let mut cdict_cp = [0; 7]; - cctx.set(1010, 2).unwrap(); // no block splitter - if mode == Mode::CDict { - unsafe { - let p = ZSTD_createCCtxParams(); - for (i, v) in cp.iter().enumerate() { c::res(ZSTD_CCtxParams_setParameter(p, 101 + i as u32, *v as i32)).unwrap(); } - c::res(ZSTD_CCtxParams_setParameter(p, 1011, if use_row { 1 } else { 2 })).unwrap(); - let null = std::ptr::null(); - cdict = ZSTD_createCDict_advanced2(dict.as_ptr().cast(), dict.len(), 1, 1, p, CustomMem { a: null, b: null, c: null }); - assert!(!cdict.is_null()); - ZSTD_freeCCtxParams(p); - c::res(ZSTD_CCtx_refCDict(cctx.0, cdict)).unwrap(); - cdict_cp = arr(ZSTD_getCParamsFromCDict(cdict)); - } - cctx.set(1001, 1).unwrap(); // force attach - cctx.set(101, cp[0] as i32).unwrap(); - } else { - for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } - cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder - if mode == Mode::Prefix { cctx.ref_prefix(dict).unwrap(); } - } - let mut out = vec![Sequence::default(); src.len() / 3 + src.len() / 1024 + 8]; - let n = c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }); - let r = n.ok().map(|n| { - let mut s: ShimState = unsafe { std::mem::zeroed() }; - unsafe { shim_cctx_state(cctx.0, &mut s) }; - let row_used = s.use_row != 0 && (3..=5).contains(&s.cp[6]); - let table = |p: *const u32, n: usize| if n == 0 { Vec::new() } else { unsafe { std::slice::from_raw_parts(p, n) }.to_vec() }; - let state = State { - has_dms: s.has_dms != 0, next_to_update: s.next_to_update, loaded_dict_end: s.loaded_dict_end, low_limit: s.low_limit, dict_limit: s.dict_limit, - hash_salt_entropy: s.hash_salt_entropy, lazy_skipping: s.lazy_skipping != 0, hash_cache: s.hash_cache, rep: s.rep, - hash_table: table(s.hash_table, 1 << s.cp[2]), - chain_table: table(s.chain_table, if s.cp[6] != 1 && !row_used { 1 << s.cp[1] } else { 0 }), - tag_table: if row_used { unsafe { std::slice::from_raw_parts(s.tag_table, 1 << s.cp[2]) }.to_vec() } else { Vec::new() }, - }; - COut { - seqs: out[..n].iter().map(|s| (s.lit_length, s.match_length, s.rep, if s.rep == 0 { s.offset } else { 0 })).collect(), - cp: s.cp, use_row: s.use_row != 0, cdict_cp, state, - } - }); - if !cdict.is_null() { unsafe { ZSTD_freeCDict(cdict) }; } - r -} - -fn first_diff(a: &[T], b: &[T]) -> Option { - if a.len() != b.len() { return Some(a.len().min(b.len())); } - a.iter().zip(b).position(|(a, b)| a != b) -} - -/// `strategies`: which of 1..=9 to test. `modes`: which kinds of dictionary. -pub fn run(seed: u64, iters: u64, strategies: &[u32], modes: &[Mode]) { - let mut skipped = 0u64; - let mut dms_blocks = 0u64; - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if i % 2000 == 0 { eprintln!("seqs iter {i} (seed {s})"); } - let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071, 400_000]); - let len = rng.range(7, max); - let mut src = data(&mut rng, len); - let mode = *rng.pick(modes); - let dict = if mode != Mode::NoDict { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; - let mode = if dict.is_empty() { Mode::NoDict } else { mode }; - if !dict.is_empty() && rng.chance(2, 3) { - for _ in 0..rng.range(1, 6) { - let n = rng.size(dict.len().min(src.len())); - let at = if rng.chance(1, 3) { dict.len() - n } else { rng.below(dict.len() - n + 1) }; - let to = if rng.chance(1, 4) { 0 } else { rng.below(src.len() - n + 1) }; - src[to..to + n].copy_from_slice(&dict[at..at + n]); - } - } - let strategy = *rng.pick(strategies); - let p = if rng.chance(1, 3) { - // the parameters of a level, with the strategy forced - let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; - p.strategy = strategy; - p - } else { - CParams { - window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, - search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, - target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, - } - }; - let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; - let use_row = rng.chance(1, 2); - let Some(c) = c_frame(arr(p), use_row, mode, &dict, &src) else { skipped += 1; continue }; - let d = match mode { Mode::NoDict => Dict::None, Mode::Prefix => Dict::Prefix(&dict), Mode::CDict => Dict::CDict(&dict, c.cdict_cp) }; - let what = format!("seed {s}: {mode:?} cp {:?} cdict cp {:?} row {} dict {} src {}", c.cp, c.cdict_cp, c.use_row, dict.len(), src.len()); - let (got, state) = bun_zstd::testing_frame::frame(c.cp, c.use_row, d, &src).unwrap_or_else(|e| panic!("{what}: error {e:?}")); - if mode == Mode::CDict && dict.len() >= 8 { dms_blocks += 1; } - if let Some(at) = first_diff(&got, &c.seqs) { - panic!("{what}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", got.get(at), c.seqs.get(at), got.len(), c.seqs.len()); - } - if state != c.state { - let (a, b) = (&state, &c.state); - panic!("{what}: state differs: hash table {:?} chain table {:?} tag table {:?}\n got {:?}\n exp {:?}", - first_diff(&a.hash_table, &b.hash_table), first_diff(&a.chain_table, &b.chain_table), first_diff(&a.tag_table, &b.tag_table), - (a.has_dms, a.next_to_update, a.loaded_dict_end, a.low_limit, a.dict_limit, a.hash_salt_entropy, a.lazy_skipping, a.hash_cache, a.rep), - (b.has_dms, b.next_to_update, b.loaded_dict_end, b.low_limit, b.dict_limit, b.hash_salt_entropy, b.lazy_skipping, b.hash_cache, b.rep)); - } - } - println!("seqs ok: {iters} iterations from seed {seed} ({skipped} skipped, {dms_blocks} with an attached CDict), strategies {strategies:?}, modes {modes:?}"); -} - -/// Speed of the Rust block compressors against the C ones, on a file. -pub fn bench(path: &str, mode: Mode) { - let all = std::fs::read(path).unwrap(); - let (dict, src) = if mode == Mode::NoDict { (&[][..], &all[..]) } else { all.split_at(100_000.min(all.len() / 2)) }; - for (level, strategy, use_row) in [(5, 3, false), (5, 3, true), (7, 4, false), (7, 4, true), (10, 5, false), (10, 5, true), (12, 5, true), (13, 6, false), (15, 6, false)] { - let mut p = unsafe { ZSTD_getCParams(level, if mode == Mode::CDict { 100_000 } else { src.len() as u64 }, dict.len()) }; - p.strategy = strategy; - let mut best = [f64::MAX; 2]; - let mut nseq = [0usize; 2]; - for _ in 0..3 { - let t = std::time::Instant::now(); - let c = c_frame(arr(p), use_row, mode, dict, src).unwrap(); - best[0] = best[0].min(t.elapsed().as_secs_f64()); - nseq[0] = c.seqs.len(); - let d = match mode { Mode::NoDict => Dict::None, Mode::Prefix => Dict::Prefix(dict), Mode::CDict => Dict::CDict(dict, c.cdict_cp) }; - let t = std::time::Instant::now(); - let (got, _) = bun_zstd::testing_frame::frame(c.cp, c.use_row, d, src).unwrap(); - best[1] = best[1].min(t.elapsed().as_secs_f64()); - nseq[1] = got.len(); - } - let mb = src.len() as f64 / 1e6; - println!("{mode:?} level {level} strategy {strategy} row {use_row}: C {:.1} MB/s, Rust {:.1} MB/s ({:.0}%) seqs {} {}", mb / best[0], mb / best[1], 100.0 * best[0] / best[1], nseq[0], nseq[1]); - } -} diff --git a/src/zstd/wip/work/fast/harness/src/main.rs b/src/zstd/wip/work/fast/harness/src/main.rs deleted file mode 100644 index 9e62db4a65c6..000000000000 --- a/src/zstd/wip/work/fast/harness/src/main.rs +++ /dev/null @@ -1,44 +0,0 @@ -mod c; -mod data; -mod frames; -mod seqs; -mod stream; - -fn main() { - let args: Vec = std::env::args().collect(); - let cmd = args.get(1).map(String::as_str).unwrap_or(""); - let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); - let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); - match cmd { - // seqs - "seqs" => { - let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); - seqs::run(seed, iters, &strategies, args.get(5).is_some_and(|s| s == "1")); - } - // frames <0: no dictionary, 1: a prefix or none, 2: an attached CDict or none> - "frames" => { - let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); - let modes: &[frames::Mode] = match args.get(5).map(String::as_str) { - Some("1") => &[frames::Mode::NoDict, frames::Mode::Prefix], - Some("2") => &[frames::Mode::CDict, frames::Mode::CDict, frames::Mode::CDict, frames::Mode::NoDict], - _ => &[frames::Mode::NoDict], - }; - frames::run(seed, iters, &strategies, modes); - } - // bench - "bench" => { - let cp: Vec = args[3].split(',').map(|s| s.parse().unwrap()).collect(); - stream::bench(&args[2], cp.try_into().unwrap()); - } - // stream - "stream" => { - let hook = std::panic::take_hook(); - std::panic::set_hook(Box::new(move |info| { - eprintln!("stream seed {}", stream::CURRENT.load(std::sync::atomic::Ordering::Relaxed)); - hook(info); - })); - stream::run(seed, iters); - } - _ => eprintln!("unknown command"), - } -} diff --git a/src/zstd/wip/work/fast/harness/src/seqs.rs b/src/zstd/wip/work/fast/harness/src/seqs.rs deleted file mode 100644 index 4a41a2c60adb..000000000000 --- a/src/zstd/wip/work/fast/harness/src/seqs.rs +++ /dev/null @@ -1,78 +0,0 @@ -//! One block through a Rust block compressor and through the C library (`ZSTD_generateSequences`). -use crate::c; -use crate::data::{Rng, data}; -use std::ffi::{c_uint, c_void}; - -#[repr(C)] -#[derive(Clone, Copy, Default, Debug)] -struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } -#[repr(C)] -#[derive(Clone, Copy, Debug)] -struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } -unsafe extern "C" { - fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; - fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; - fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; -} - -fn c_sequences(cp: [u32; 7], use_row: bool, dict: &[u8], src: &[u8]) -> Vec<(u32, u32, u32)> { - let mut cctx = c::CCtx::new(); - // windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy - for (id, v) in [101, 103, 102, 104, 105, 106, 107].iter().zip(cp) { cctx.set(*id, v as i32).unwrap(); } - cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder - cctx.set(1010, 2).unwrap(); // no block splitter - if !dict.is_empty() { cctx.ref_prefix(dict).unwrap(); } - let mut out = vec![Sequence::default(); src.len() / 3 + 8]; - let n = c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }).unwrap(); - out[..n].iter().map(|s| (s.lit_length, s.match_length, s.offset)).collect() -} - -/// `strategies`: which of 1..=9 to test. `with_dict`: also test with a prefix, which is the extDict variants. -pub fn run(seed: u64, iters: u64, strategies: &[u32], with_dict: bool) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if i % 500 == 0 { eprintln!("seqs iter {i} (seed {s})"); } - let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071]); - let len = rng.range(7, max); - let mut src = data(&mut rng, len); - let dict = if with_dict && rng.chance(1, 2) { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; - if !dict.is_empty() && rng.chance(2, 3) { - for _ in 0..rng.range(1, 6) { - let n = rng.size(dict.len().min(src.len())); - let at = rng.below(dict.len() - n + 1); - let to = rng.below(src.len() - n + 1); - src[to..to + n].copy_from_slice(&dict[at..at + n]); - } - } - let strategy = *rng.pick(strategies); - let p = if rng.chance(1, 3) { - // the parameters of a level, with the strategy forced - let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; - p.strategy = strategy; - p - } else { - CParams { - window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, - search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, - target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, - } - }; - let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; - // The C library refuses to give the sequences of a block of less than 7 bytes. - let block_size = (1usize << p.window_log).min(128 << 10); - if src.len() % block_size != 0 && src.len() % block_size < 7 { src.truncate(src.len() - src.len() % block_size); } - let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; - let cp = [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy]; - let use_row = rng.chance(1, 2); - let expected = c_sequences(cp, use_row, &dict, &src); - let got = bun_zstd::testing::first_block(cp, use_row, &dict, &src); - let got = got.unwrap_or_else(|e| panic!("seed {s}: error {e:?} (cp {cp:?} row {use_row} dict {} src {})", dict.len(), src.len())); - if got != expected { - let at = got.iter().zip(&expected).position(|(a, b)| a != b).unwrap_or(got.len().min(expected.len())); - panic!("seed {s}: cp {cp:?} row {use_row} dict {} src {}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", - dict.len(), src.len(), got.get(at), expected.get(at), got.len(), expected.len()); - } - } - println!("seqs ok: {iters} iterations from seed {seed}, strategies {strategies:?}, dict {with_dict}"); -} diff --git a/src/zstd/wip/work/fast/harness/src/stream.rs b/src/zstd/wip/work/fast/harness/src/stream.rs deleted file mode 100644 index e75619deaa4d..000000000000 --- a/src/zstd/wip/work/fast/harness/src/stream.rs +++ /dev/null @@ -1,301 +0,0 @@ -//! The block compressors and the table fillers of `fast` and `dfast`, call by call, against the C functions -//! (`c/shim.c`), on match states made by hand: dictionaries that are loaded, attached (dictMatchState) or none, input -//! in chunks that are contiguous or not or that overwrite earlier input, blocks of any size, indices that start high. -//! After each call the sequences, the literals, the repeat offsets and the tables are compared. -use crate::data::{Rng, data}; -use bun_zstd::testing::TestMs; -use std::ffi::c_void; - -pub enum Shim {} -#[repr(C)] -#[derive(Clone, Copy, Default, Debug)] -pub struct State { - base: u64, dict_base: u64, next_src: u64, - dict_limit: u32, low_limit: u32, loaded_dict_end: u32, next_to_update: u32, dict_mode: u32, pad: u32, -} -#[repr(C)] -#[derive(Clone, Copy, Debug)] -struct SeqDef { off_base: u32, lit_length: u16, ml_base: u16 } - -unsafe extern "C" { - fn shim_new(cp: *const u32, max_block: usize) -> *mut Shim; - fn shim_free(s: *mut Shim); - fn shim_state(s: *const Shim, st: *mut State); - fn shim_hash_table(s: *const Shim) -> *const u32; - fn shim_chain_table(s: *const Shim) -> *const u32; - fn shim_bump(s: *mut Shim, index: u32); - fn shim_load_dict(s: *mut Shim, src: *const c_void, size: usize, dtlm: i32, tfp: i32, force: i32, before: *mut State) -> i32; - fn shim_attach(s: *mut Shim, dict: *const Shim) -> i32; - fn shim_update(s: *mut Shim, src: *const c_void, size: usize) -> i32; - fn shim_pre_block(s: *mut Shim, src: *const c_void, size: usize); - fn shim_block(s: *mut Shim, rep: *mut u32, src: *const c_void, size: usize) -> usize; - fn shim_nb_seq(s: *const Shim) -> usize; - fn shim_seqs(s: *const Shim) -> *const SeqDef; - fn shim_lit_len(s: *const Shim) -> usize; - fn shim_lits(s: *const Shim) -> *const u8; - fn shim_long_type(s: *const Shim) -> u32; - fn shim_long_pos(s: *const Shim) -> u32; -} - -/// The same match state on both sides. -struct Pair { c: *mut Shim, r: TestMs, cp: [u32; 7] } -impl Pair { - fn new(cp: [u32; 7], max_block: usize) -> Self { Self { c: unsafe { shim_new(cp.as_ptr(), max_block) }, r: TestMs::new(cp, max_block), cp } } - fn state(&self) -> State { let mut st = State::default(); unsafe { shim_state(self.c, &mut st) }; st } - fn sync(&mut self, st: &State) { self.r.set_state(st.dict_limit, st.low_limit, st.loaded_dict_end, st.next_to_update); } - fn check_tables(&self, what: &str) { - let (hash, chain) = self.r.tables(); - let c_hash = unsafe { std::slice::from_raw_parts(shim_hash_table(self.c), 1 << self.cp[2]) }; - let c_chain = unsafe { std::slice::from_raw_parts(shim_chain_table(self.c), 1 << self.cp[1]) }; - if let Some(at) = hash.iter().zip(c_hash).position(|(a, b)| a != b) { panic!("{what}: hash table differs at {at}: got {}, expected {}", hash[at], c_hash[at]); } - if let Some(at) = chain.iter().zip(c_chain).position(|(a, b)| a != b) { panic!("{what}: chain table differs at {at}: got {}, expected {}", chain[at], c_chain[at]); } - } - /// Returns the state after the load. - fn load_dict(&mut self, dict: &[u8], full: bool, for_cdict: bool, force: bool) -> State { - let mut before = State::default(); - let filled = unsafe { shim_load_dict(self.c, dict.as_ptr().cast(), dict.len(), full as i32, for_cdict as i32, force as i32, &mut before) }; - if filled != 0 { - self.sync(&before); - let base = (before.base as usize).wrapping_sub(dict.as_ptr() as usize); - self.r.fill(dict, base, dict.len(), full, for_cdict).unwrap_or_else(|e| panic!("fill: error {e:?}")); - self.check_tables("fill"); - } - self.state() - } -} -impl Drop for Pair { fn drop(&mut self) { unsafe { shim_free(self.c) } } } - -#[derive(Default)] -pub struct Stats { pub blocks: [[u64; 3]; 2], pub seqs: [[u64; 3]; 2], pub fills: [[u64; 4]; 2] } - -fn splice(rng: &mut Rng, from: &[u8], to: &mut [u8], times: usize) { - if from.is_empty() || to.is_empty() { return; } - for _ in 0..times { - let n = rng.size(from.len().min(to.len())); - let at = rng.below(from.len() - n + 1); - let dst = rng.below(to.len() - n + 1); - to[dst..dst + n].copy_from_slice(&from[at..at + n]); - } -} - -fn one(rng: &mut Rng, stats: &mut Stats) { - let strategy = rng.range(1, 2) as u32; - let si = strategy as usize - 1; - let window_log = *rng.pick(&[10u32, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20]); - let log = |rng: &mut Rng| if rng.chance(1, 4) { rng.range(6, 9) } else { rng.range(8, 17) } as u32; - let min_match = rng.range(3, 7) as u32; - let target_length = *rng.pick(&[0u32, 0, 1, 2, 3, 4, 5, 6, 7, 8, 16, 32, 100, 999]); - let cp = [window_log, log(rng), log(rng), 1, min_match, target_length, strategy]; - let mut block_max = (1usize << window_log).min(128 << 10); - if rng.chance(1, 3) { block_max = block_max.min(1 << rng.range(10, 17)); } - - let total = { let m = *rng.pick(&[60usize, 600, 6000, 40_000, 40_000, 300_000]); rng.range(1, m) }; - let mut corpus = data(rng, total); - let kind = rng.below(5); // 0, 1: no dictionary. 2: loaded. 3, 4: attached. - let dict_len = if kind >= 2 { let m = *rng.pick(&[12usize, 100, 2000, 30_000, 200_000]); rng.range(1, m) } else { 0 }; - let mut dict = data(rng, dict_len); - if rng.chance(1, 2) { let t = rng.range(1, 6); splice(rng, &corpus, &mut dict, t); } - if rng.chance(3, 4) { let t = rng.range(1, 12); splice(rng, &dict, &mut corpus, t); } - if rng.chance(1, 2) { let t = rng.range(1, 8); let copy = corpus.clone(); splice(rng, ©, &mut corpus, t); } - - let mut pair = Pair::new(cp, block_max); - if rng.chance(1, 4) { - let index = *rng.pick(&[100u32, 70_000, 1 << 24, 1 << 31, 3_600_000_000]); - unsafe { shim_bump(pair.c, rng.range(2, index as usize) as u32) }; - } - - // Each region is a buffer of its own. `cur` is the one of the current segment of the window, `ext` of the older. - let mut regions: Vec> = Vec::new(); - let mut cur: Option = None; - let mut cur_end = 0usize; - let mut ext: Option = None; - let mut rep = [1u32, 4, 8]; - - let mut dict_pair: Option = None; - let mut dms_state = State::default(); - let mut attached = false; - if kind == 2 { - let room = if rng.chance(1, 2) { rng.size(total) + rng.below(20) } else { 0 }; - let mut mem = dict.clone(); - mem.resize(dict.len() + room, 0); - regions.push(mem); - let full = rng.chance(1, 3); - let force = rng.chance(1, 4); - let filled_before = pair.state(); - let _ = filled_before; - pair.load_dict(®ions[0][..dict.len()], full, false, force); - stats.fills[si][full as usize] += 1; - cur = Some(0); - cur_end = dict.len(); - } else if kind >= 3 { - let mut dcp = cp; - dcp[1] = log(rng); - dcp[2] = log(rng); - if rng.chance(1, 8) { dcp[4] = rng.range(3, 7) as u32; } - let mut d = Pair::new(dcp, 16); - let full = rng.chance(4, 5); - dms_state = d.load_dict(&dict, full, true, false); - stats.fills[si][2 + full as usize] += 1; - attached = unsafe { shim_attach(pair.c, d.c) } != 0; - dict_pair = Some(d); - if rng.chance(1, 3) { rep = [rng.range(1, dict.len()) as u32, rng.range(1, dict.len()) as u32, rng.range(1, dict.len()) as u32]; } - } - if kind < 3 && rng.chance(1, 4) { - let mut r = |rng: &mut Rng| { let m = *rng.pick(&[10usize, 1000, 100_000, u32::MAX as usize]); rng.range(1, m) as u32 }; - rep = [r(rng), r(rng), r(rng)]; - } - - let mut consumed = 0usize; - while consumed < corpus.len() { - let left = corpus.len() - consumed; - let n = if rng.chance(1, 3) { left } else { rng.range(1, left) }; - let chunk = &corpus[consumed..consumed + n]; - consumed += n; - - // Where the chunk goes. - let choice = rng.below(10); - let (region, at) = match cur { - Some(r) if choice < 6 && cur_end + n <= regions[r].len() => (r, cur_end), - Some(r) if choice >= 8 && n <= regions[r].len() => (r, if rng.chance(1, 2) { 0 } else { rng.below(regions[r].len() - n + 1) }), - _ => { - let room = if rng.chance(2, 3) { rng.size(left - n + 64) } else { 0 }; - let lead = if rng.chance(1, 4) { rng.below(40) } else { 0 }; - regions.push(vec![0; lead + n + room]); - (regions.len() - 1, lead) - } - }; - regions[region][at..at + n].copy_from_slice(chunk); - let ptr = regions[region].as_ptr(); - let contiguous = unsafe { shim_update(pair.c, ptr.add(at).cast(), n) } != 0; - if !contiguous { ext = cur; cur = Some(region); } - assert_eq!(cur, Some(region)); - cur_end = at + n; - - let mut pos = at; - while pos < cur_end { - let size = (cur_end - pos).min(if rng.chance(1, 4) { rng.range(1, block_max) } else { block_max }); - unsafe { shim_pre_block(pair.c, ptr.add(pos).cast(), size) }; - let st = pair.state(); - pair.sync(&st); - attached &= st.dict_mode == 2 || (st.dict_mode == 1 && st.loaded_dict_end != 0); - let buf = ®ions[region][..]; - let dict_buf: &[u8] = ext.map_or(&[], |r| ®ions[r][..]); - let base = (st.base as usize).wrapping_sub(buf.as_ptr() as usize); - let dict_base = (st.dict_base as usize).wrapping_sub(dict_buf.as_ptr() as usize); - let dms = if st.dict_mode == 2 { - let d = dict_pair.as_ref().unwrap(); - Some((&d.r, &dict[..], (dms_state.base as usize).wrapping_sub(dict.as_ptr() as usize), (dms_state.next_src as usize).wrapping_sub(dict.as_ptr() as usize))) - } else { None }; - - // Repeat offsets at the limit of what is valid. - if rng.chance(1, 5) { - let curr = (ptr as usize + pos).wrapping_sub(st.base as usize) as u32; - let reach = if st.dict_mode == 2 { - let dict_size = (dms_state.next_src - dms_state.base) as u32 - dms_state.dict_limit; - (curr - st.dict_limit + dict_size).saturating_sub(rng.below(3) as u32) - } else { - let lowest_valid = if rng.chance(1, 2) { st.low_limit } else { st.dict_limit }; - let from = if rng.chance(1, 2) { curr } else { curr + size as u32 }; - let low = if st.loaded_dict_end == 0 && from - lowest_valid > 1 << window_log { from - (1 << window_log) } else { lowest_valid }; - (curr - low + rng.size(64) as u32).saturating_sub(2) - }; - if reach != 0 { rep[rng.below(2)] = reach; } - } - let (mut rep_c, mut rep_r) = (rep, rep); - let expected = unsafe { shim_block(pair.c, rep_c.as_mut_ptr(), ptr.add(pos).cast(), size) }; - let what = format!("block {pos}..{} of region {region} (mode {}, {st:?})", pos + size, st.dict_mode); - let got = pair.r.block(&mut rep_r, st.dict_mode, buf, base, dict_buf, dict_base, pos, pos + size, dms) - .unwrap_or_else(|e| panic!("{what}: error {e:?}")); - - let c_seqs: Vec<(u32, u16, u16)> = unsafe { std::slice::from_raw_parts(shim_seqs(pair.c), shim_nb_seq(pair.c)) } - .iter().map(|s| (s.off_base, s.lit_length, s.ml_base)).collect(); - let r_seqs = pair.r.sequences(); - if r_seqs != c_seqs { - let i = r_seqs.iter().zip(&c_seqs).position(|(a, b)| a != b).unwrap_or(r_seqs.len().min(c_seqs.len())); - panic!("{what}: sequence {i} differs: got {:?}, expected {:?} ({} vs {})", r_seqs.get(i), c_seqs.get(i), r_seqs.len(), c_seqs.len()); - } - assert_eq!(got, expected, "{what}: last literals"); - assert_eq!(rep_r, rep_c, "{what}: rep"); - let c_lits = unsafe { std::slice::from_raw_parts(shim_lits(pair.c), shim_lit_len(pair.c)) }; - assert!(pair.r.literals() == c_lits, "{what}: literals"); - let long = unsafe { (shim_long_type(pair.c), shim_long_pos(pair.c)) }; - if long.0 != 0 || pair.r.long_length().0 != 0 { assert_eq!(pair.r.long_length(), long, "{what}: long length"); } - pair.check_tables(&what); - assert_eq!(pair.state().next_to_update, st.next_to_update); - - stats.blocks[si][st.dict_mode as usize] += 1; - stats.seqs[si][st.dict_mode as usize] += c_seqs.len() as u64; - if rng.chance(7, 8) { rep = rep_c; } - pos += size; - } - } - let _ = attached; -} - -pub fn run(seed: u64, iters: u64) { - let mut stats = Stats::default(); - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - if i % 2000 == 0 { eprintln!("stream iter {i} (seed {s})"); } - if std::env::var("ONLY").is_ok_and(|o| o != s.to_string()) { continue; } - CURRENT.store(s, std::sync::atomic::Ordering::Relaxed); - one(&mut Rng(s), &mut stats); - } - println!("stream ok: {iters} iterations from seed {seed}"); - for (si, name) in ["fast", "dfast"].iter().enumerate() { - println!(" {name}: blocks noDict/extDict/dictMatchState {:?}, sequences {:?}, fills CCtx-fast/CCtx-full/CDict-fast/CDict-full {:?}", - stats.blocks[si], stats.seqs[si], stats.fills[si]); - } -} - -pub static CURRENT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - -/// bench : the time both sides take to find the sequences of the file, in blocks of 128 KB, with the -/// parameters of the level (which has to be of the `fast` or `dfast` strategy), without and with an attached dictionary. -pub fn bench(path: &str, cp: [u32; 7]) { - use std::time::{Duration, Instant}; - let all = std::fs::read(path).unwrap(); - for with_dms in [false, true] { - let (dict, src) = if with_dms { all.split_at(100_000.min(all.len() / 2)) } else { all.split_at(0) }; - let mut cp = cp; - if with_dms { cp[0] = 27; } - let (mut best_c, mut best_r) = (Duration::MAX, Duration::MAX); - let mut nb_seq = 0usize; - for _ in 0..5 { - let mut pair = Pair::new(cp, 128 << 10); - let mut d = Pair::new(cp, 16); - let mut dms_state = State::default(); - if with_dms { - dms_state = d.load_dict(dict, true, true, false); - unsafe { shim_attach(pair.c, d.c) }; - } - unsafe { shim_update(pair.c, src.as_ptr().cast(), src.len()) }; - let (mut time_c, mut time_r) = (Duration::ZERO, Duration::ZERO); - let (mut rep_c, mut rep_r) = ([1u32, 4, 8], [1u32, 4, 8]); - nb_seq = 0; - let mut pos = 0; - while pos < src.len() { - let size = (src.len() - pos).min(128 << 10); - unsafe { shim_pre_block(pair.c, src.as_ptr().add(pos).cast(), size) }; - let st = pair.state(); - pair.sync(&st); - let base = (st.base as usize).wrapping_sub(src.as_ptr() as usize); - let dms = (st.dict_mode == 2).then(|| (&d.r, dict, (dms_state.base as usize).wrapping_sub(dict.as_ptr() as usize), (dms_state.next_src as usize).wrapping_sub(dict.as_ptr() as usize))); - let t = Instant::now(); - let a = unsafe { shim_block(pair.c, rep_c.as_mut_ptr(), src.as_ptr().add(pos).cast(), size) }; - time_c += t.elapsed(); - let t = Instant::now(); - let b = pair.r.block(&mut rep_r, st.dict_mode, src, base, &[], 0, pos, pos + size, dms).unwrap(); - time_r += t.elapsed(); - assert_eq!(a, b); - nb_seq += unsafe { shim_nb_seq(pair.c) }; - pos += size; - } - best_c = best_c.min(time_c); - best_r = best_r.min(time_r); - } - let mbs = |d: Duration| src.len() as f64 / 1e6 / d.as_secs_f64(); - println!("cp {cp:?} dms {with_dms}: {} bytes, {nb_seq} sequences: C {:.0} MB/s, Rust {:.0} MB/s ({:.2}x the time)", - src.len(), mbs(best_c), mbs(best_r), best_r.as_secs_f64() / best_c.as_secs_f64()); - } -} diff --git a/src/zstd/wip/work/fast/zstd/lib.rs b/src/zstd/wip/work/fast/zstd/lib.rs deleted file mode 100644 index c658d011170d..000000000000 --- a/src/zstd/wip/work/fast/zstd/lib.rs +++ /dev/null @@ -1,386 +0,0 @@ -//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the -//! same bytes in, in both directions, and the same errors. -#![forbid(unsafe_code)] -// Frames come from anywhere and a panic aborts the process. -#![deny( - clippy::indexing_slicing, - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::unreachable -)] -#![warn(unused_must_use)] - -mod common; -mod compress; -mod decompress; -mod error; -mod util; - -pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; -pub use decompress::{ - DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, - find_frame_compressed_size, get_frame_content_size, -}; -pub use error::ErrorCode; - -/// `ZSTD_inBuffer` -pub struct InBuffer<'a> { - pub src: &'a [u8], - /// How much of `src` is read. - pub pos: usize, -} - -/// `ZSTD_outBuffer` -pub struct OutBuffer<'a> { - pub dst: &'a mut [u8], - /// How much of `dst` is written. - pub pos: usize, -} - -/// `ZSTD_ResetDirective` -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum ResetDirective { - SessionOnly = 1, - Parameters = 2, - SessionAndParameters = 3, -} - -// ----------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] -pub enum ZstdError { - /// The output, or the decoder state the frame's window size dictates, could not be allocated. - OutOfMemory, - InvalidZstdData, - DecompressionFailed, - ZstdDecompressionError, - ShortRead, -} - -impl core::fmt::Display for ZstdError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.write_str(<&'static str>::from(self)) - } -} -impl core::error::Error for ZstdError {} - -impl ZstdError { - /// The error for a failed decompression call; `other` is the non-allocation failure. - fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { - if code == ErrorCode::MemoryAllocation { - ZstdError::OutOfMemory - } else { - other - } - } -} - -/// `len` zeros for a decoder to write to. -fn output_buffer(len: usize) -> Result, ZstdError> { - util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) -} - -/// Decompress data, automatically allocating the output buffer. -/// Handles both frames with known and unknown content sizes. -/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. -/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. -pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { - let size = find_decompressed_size(src); - - const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit - - if size == ZSTD_CONTENTSIZE_ERROR { - return Err(ZstdError::InvalidZstdData); - } - - // Use streaming decompression if: - // 1. Content size is unknown, OR - // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) - if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { - let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { - // A frame's output is rarely smaller than its input. - src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) - } else { - // The header size is untrusted: reserve no more than the fast path below would. - MAX_PREALLOCATE_SIZE - }; - let mut list: Vec = Vec::new(); - list.try_reserve_exact(initial_capacity) - .map_err(|_| ZstdError::OutOfMemory)?; - StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; - return Ok(list); - } - - // Fast path: size is known and within reasonable limits - let mut output = output_buffer(size as usize)?; - let written = decompress(&mut output, src) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is -/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. -pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { - const MAX_SIZE: u64 = 16 * 1024 * 1024; - let size = find_decompressed_size(src); - // Also if it is unknown or invalid, which are the two largest values. - if size > MAX_SIZE { - return Err(ZstdError::InvalidZstdData); - } - let mut output = output_buffer(size as usize)?; - let written = DDict::prefix(prefix) - .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum State { - Uninitialized, - Inflating, - End, - Error, -} - -/// Minimum spare output capacity offered to the decoder per call. -const STREAMING_OUTPUT_STEP: usize = 4096; - -// ────────────────────────────────────────────────────────────────────────── -// StreamingDecoder -// ────────────────────────────────────────────────────────────────────────── - -/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) -/// per call, so callers can hold the decoder across multiple body chunks. -pub struct StreamingDecoder { - stream: DCtx, - state: State, - /// The decoder filled its last window and may hold more. `max_output` can end a call there. - output_full: bool, -} - -impl Default for StreamingDecoder { - fn default() -> Self { - Self::new() - } -} - -impl StreamingDecoder { - pub fn new() -> Self { - Self { - stream: DCtx::new(), - state: State::Uninitialized, - output_full: false, - } - } - - #[inline] - pub fn is_inflating(&self) -> bool { - matches!(self.state, State::Inflating) - } - - /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is - /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. - /// Returns `ShortRead` when more input is required and `is_done` is false. - pub fn decompress( - &mut self, - input: &[u8], - out: &mut Vec, - max_output: usize, - is_done: bool, - ) -> Result { - if matches!(self.state, State::End | State::Error) { - return Ok(input.len()); - } - - let mut in_buf = InBuffer { src: input, pos: 0 }; - while matches!(self.state, State::Uninitialized | State::Inflating) { - let next_in_len = input.len() - in_buf.pos; - - // Call the decoder again with no input until it leaves the window short. - if next_in_len == 0 && !self.output_full { - if is_done { - if self.state == State::Inflating { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - self.state = State::End; - } - return Ok(in_buf.pos); - } - - if out.len() >= max_output { - return Ok(in_buf.pos); - } - - if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { - self.state = State::Error; - return Err(ZstdError::OutOfMemory); - } - // The decoder writes to initialized memory only. - let written_before = out.len(); - let window = (out.capacity() - written_before).min(max_output - written_before); - out.resize(written_before + window, 0); - let mut out_buf = OutBuffer { - dst: out.get_mut(written_before..).unwrap_or_default(), - pos: 0, - }; - let read_before = in_buf.pos; - - let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); - let bytes_written = out_buf.pos; - out.truncate(written_before + bytes_written); - let remaining = match result { - Ok(remaining) => remaining, - Err(code) => { - self.state = State::Error; - return Err(ZstdError::for_decompression( - code, - ZstdError::ZstdDecompressionError, - )); - } - }; - - self.output_full = bytes_written == window; - - if remaining == 0 { - // Frame complete, and fully flushed. - self.state = State::Uninitialized; - self.output_full = false; - if in_buf.pos >= input.len() { - if is_done { - self.state = State::End; - } - return Ok(in_buf.pos); - } - // More input available — reinitialize for the next frame. - self.stream.init_stream(); - continue; - } - - self.state = State::Inflating; - - if in_buf.pos - read_before == next_in_len { - if self.output_full { - continue; - } - if is_done { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - return Err(ZstdError::ShortRead); - } - } - Ok(in_buf.pos) - } -} - -// ── compressed embedded assets ──────────────────────────────────────────── - -/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes -/// Bun never executes or parses itself: the shell completion scripts and the -/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, -/// error overlay/page). Anything that runs inside Bun stays uncompressed. -/// -/// The exception is text that only `bun lint`, `bun fmt` and `bun check` -/// read. It may be compressed because it isn't on the path of running a program. -/// -/// Release (`bun_codegen_embed`) builds include the `.zst` twin that -/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to -/// `/compressed/.zst`; other builds evaluate `$fallback`. -#[macro_export] -macro_rules! embed_compressed { - // A codegen output (`/$sub`); debug builds read it at runtime. - (codegen $sub:literal) => { - $crate::embed_compressed!( - ("codegen/", $sub), - ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() - ) - }; - // Same, with a trailing NUL included in the slice (for C-string consumers). - (codegen_nul $sub:literal) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/codegen/", - $sub, - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); - &__COPY.get_or_init(|| { - let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); - let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); - copy.extend_from_slice(text); - copy.push(0); - copy.into_boxed_slice() - })[..] - } - }; - __bytes - }}; - // A file under `src/`; debug builds read it from the source tree at runtime. - (src $sub:literal) => { - $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) - }; - ($name:literal, $fallback:expr) => { - $crate::embed_compressed!(($name), $fallback) - }; - (($($name:literal),+), $fallback:expr) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/", - $($name,)+ - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - $fallback - } - }; - __bytes - }}; -} - -/// Cold, shared body of [`embed_compressed!`]'s release arm. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { - decompress_alloc(compressed) - .expect("embedded asset: invalid zstd frame") - .into_boxed_slice() -} - -/// [`inflate_embedded`] plus a trailing NUL byte. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { - let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); - inflated.reserve_exact(1); - inflated.push(0); - inflated.into_boxed_slice() -} -pub mod testing; -pub mod testing_frame; diff --git a/src/zstd/wip/work/fast/zstd/testing.rs b/src/zstd/wip/work/fast/zstd/testing.rs deleted file mode 100644 index 9f6051d0ef68..000000000000 --- a/src/zstd/wip/work/fast/zstd/testing.rs +++ /dev/null @@ -1,364 +0,0 @@ -//! Hooks for the differential tests. Only in scratch copies of the crate. -#![allow(clippy::all, dead_code, unreachable_pub)] - -use crate::ErrorCode; -use crate::compress::internal::*; -use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; - -fn bitmix(mut val: u64, len: u64) -> u64 { - val ^= val.rotate_right(49) ^ val.rotate_right(24); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^= (val >> 35).wrapping_add(len); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^ (val >> 28) -} - -/// The sequences of the first block of a frame, as (literal length, match length, offset), and then the literals -/// that are left as (length, 0, 0). -/// -/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, already adjusted to the sizes. -/// `dict` is a prefix (raw content, as `ZSTD_CCtx_refPrefix`), or empty. `src` is from 7 to 131071 bytes. -pub fn first_block( - cp: [u32; 7], - use_row: bool, - dict: &[u8], - src: &[u8], -) -> Result, ErrorCode> { - let strategy = Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?; - let c_params = CompressionParameters { - window_log: cp[0], - chain_log: cp[1], - hash_log: cp[2], - search_log: cp[3], - min_match: cp[4], - target_length: cp[5], - strategy: Some(strategy), - }; - let row_used = row_match_finder_supported(strategy) && use_row; - - // ZSTD_reset_matchState - let mut ms = MatchState::new(); - ms.hash_log3 = if c_params.min_match == 3 { - c_params.window_log.min(17) - } else { - 0 - }; - ms.window.clear(); - ms.next_to_update = ms.window.dict_limit; - ms.hash_table = vec![0; 1 << c_params.hash_log]; - ms.chain_table = vec![ - 0; - if strategy != Strategy::Fast && !row_used { - 1 << c_params.chain_log - } else { - 0 - } - ]; - ms.hash_table3 = vec![ - 0; - if ms.hash_log3 != 0 { - 1 << ms.hash_log3 - } else { - 0 - } - ]; - if row_used { - ms.tag_table = vec![0; 1 << c_params.hash_log]; - ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4); - ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); - } - if strategy >= Strategy::BtOpt { - ms.opt.lit_freq = vec![0; 256]; - ms.opt.lit_length_freq = vec![0; 36]; - ms.opt.match_length_freq = vec![0; 53]; - ms.opt.off_code_freq = vec![0; 32]; - ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; - ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; - } - ms.c_params = c_params; - - let block_size = (1usize << c_params.window_log).min(128 << 10); - let mut seq_store = SeqStore::new(); - seq_store.reserve( - block_size / if c_params.min_match == 3 { 3 } else { 4 }, - block_size, - )?; - - let bufs = Buffers { - in_buff: &[], - user: src, - dict, - }; - - // ZSTD_loadDictionaryContent(dtlm_fast, tfp_forCCtx) - if !dict.is_empty() { - let iend = dict.len(); - ms.window - .update(BufId::Dict.start(), BufId::Dict, dict.len(), false); - let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); - let ip = iend - dict.len().min(max_dict_size); - let prefix = Prefix { - buf: dict, - base: ms.window.base.wrapping_sub(BufId::Dict.start()), - }; - ms.next_to_update = prefix.index_of(ip); - ms.loaded_dict_end = prefix.index_of(iend); - if iend - ip > HASH_READ_SIZE { - match strategy { - Strategy::Fast => fast::fill_hash_table( - &mut ms, - prefix, - iend, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - )?, - Strategy::DFast => double_fast::fill_double_hash_table( - &mut ms, - prefix, - iend, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - )?, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { - if use_row { - ms.tag_table.fill(0); - lazy::row_update(&mut ms, prefix, iend - HASH_READ_SIZE)?; - } else { - lazy::insert_and_find_first_index(&mut ms, prefix, iend - HASH_READ_SIZE)?; - } - } - _ => opt::update_tree(&mut ms, prefix, iend - HASH_READ_SIZE, iend)?, - } - ms.next_to_update = prefix.index_of(iend); - } - } - - // ZSTD_compressContinue_internal - let src_addr = BufId::User.start(); - if !ms - .window - .update(src_addr, BufId::User, src.len(), ms.force_non_contiguous) - { - ms.force_non_contiguous = false; - ms.next_to_update = ms.window.dict_limit; - } - // ZSTD_compress_frameChunk - let max_dist = 1u32 << c_params.window_log; - let mut prev = CompressedBlockState::new(); - let mut out = Vec::new(); - let mut start = 0usize; - while start < src.len() { - let end = src.len().min(start + block_size); - if ms - .window - .dict_is_invalid(src_addr + end, max_dist, ms.loaded_dict_end) - { - ms.loaded_dict_end = 0; - } - if ms - .window - .enforce_max_dist(src_addr + start, max_dist, ms.loaded_dict_end) - { - ms.loaded_dict_end = 0; - } - if ms.next_to_update < ms.window.low_limit { - ms.next_to_update = ms.window.low_limit; - } - // ZSTD_buildSeqStore - seq_store.reset(); - let block = BlockSrc { - buf: bufs.get(ms.window.base_buf), - base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), - dict_buf: bufs.get(ms.window.dict_buf), - dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), - start, - end, - dms: None, - symbol_costs: &prev.entropy, - ldm_seq_store: None, - }; - let curr = block.index_of(start); - if curr > ms.next_to_update + 384 { - ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); - } - let dict_mode = ms.dict_mode(None); - if std::env::var("DBG").is_ok() { - eprintln!( - "block {start}..{end} mode {dict_mode:?} window {:?} lde {} ntu {}", - ms.window, ms.loaded_dict_end, ms.next_to_update - ); - } - let mut rep = prev.rep; - let last_ll = compress::compress_block( - &mut ms, - &mut seq_store, - &mut rep, - &block, - strategy, - use_row, - dict_mode, - )?; - - let mut reps = prev.rep; - for (i, seq) in seq_store.sequences.iter().enumerate() { - let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); - if seq_store.long_length_pos as usize == i { - match seq_store.long_length_type { - LongLengthType::LiteralLength => ll += 0x10000, - LongLengthType::MatchLength => ml += 0x10000, - LongLengthType::None => {} - } - } - update_rep(&mut reps, seq.off_base, ll == 0); - out.push((ll, ml, reps[0])); - } - out.push((last_ll as u32, 0, 0)); - assert_eq!( - reps[..2], - rep[..2], - "the repeat offsets the block compressor leaves are those of its sequences" - ); - prev.rep = rep; - start = end; - } - Ok(out) -} - -/// A match state whose window is set by hand, to run the block compressors of `fast` and `dfast` one call at a time. -pub struct TestMs { - ms: MatchState, - seq_store: SeqStore, - prev: CompressedBlockState, -} - -/// The dictionary's match state, its buffer, `window.base` and `window.nextSrc` as positions in it. -pub type TestDms<'a> = (&'a TestMs, &'a [u8], usize, usize); - -impl TestMs { - /// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy. - pub fn new(cp: [u32; 7], max_block: usize) -> Self { - let mut ms = MatchState::new(); - ms.c_params = CompressionParameters { - window_log: cp[0], - chain_log: cp[1], - hash_log: cp[2], - search_log: cp[3], - min_match: cp[4], - target_length: cp[5], - strategy: Strategy::from_int(cp[6] as i32), - }; - ms.hash_table = vec![0; 1 << cp[2]]; - ms.chain_table = vec![0; 1 << cp[1]]; - let mut seq_store = SeqStore::new(); - seq_store.reserve(max_block / 3 + 1, max_block).unwrap(); - Self { - ms, - seq_store, - prev: CompressedBlockState::new(), - } - } - - pub fn set_state( - &mut self, - dict_limit: u32, - low_limit: u32, - loaded_dict_end: u32, - next_to_update: u32, - ) { - self.ms.window.dict_limit = dict_limit; - self.ms.window.low_limit = low_limit; - self.ms.loaded_dict_end = loaded_dict_end; - self.ms.next_to_update = next_to_update; - } - - pub fn fill( - &mut self, - buf: &[u8], - base: usize, - end: usize, - full: bool, - for_cdict: bool, - ) -> Result<(), ErrorCode> { - let prefix = Prefix { buf, base }; - let dtlm = if full { - DictTableLoadMethod::Full - } else { - DictTableLoadMethod::Fast - }; - let tfp = if for_cdict { - TableFillPurpose::ForCDict - } else { - TableFillPurpose::ForCCtx - }; - match self.ms.c_params.strat() { - Strategy::Fast => fast::fill_hash_table(&mut self.ms, prefix, end, dtlm, tfp), - _ => double_fast::fill_double_hash_table(&mut self.ms, prefix, end, dtlm, tfp), - } - } - - /// `mode` is a `ZSTD_dictMode_e`. - pub fn block( - &mut self, - rep: &mut [u32; 3], - mode: u32, - buf: &[u8], - base: usize, - dict_buf: &[u8], - dict_base: usize, - start: usize, - end: usize, - dms: Option>, - ) -> Result { - self.seq_store.reset(); - let dms = dms.map(|(dict, buf, base, end)| DictMatchState { - ms: &dict.ms, - buf, - base, - end, - }); - let block = BlockSrc { - buf, - base, - dict_buf, - dict_base, - start, - end, - dms, - symbol_costs: &self.prev.entropy, - ldm_seq_store: None, - }; - let dict_mode = self.ms.dict_mode(dms.as_ref()); - assert_eq!(dict_mode as u32, mode); - let strategy = self.ms.c_params.strat(); - compress::compress_block( - &mut self.ms, - &mut self.seq_store, - rep, - &block, - strategy, - false, - dict_mode, - ) - } - - /// offBase, litLength, mlBase - pub fn sequences(&self) -> Vec<(u32, u16, u16)> { - self.seq_store - .sequences - .iter() - .map(|s| (s.off_base, s.lit_length, s.ml_base)) - .collect() - } - pub fn literals(&self) -> &[u8] { - self.seq_store.literals() - } - pub fn long_length(&self) -> (u32, u32) { - ( - self.seq_store.long_length_type as u32, - self.seq_store.long_length_pos, - ) - } - pub fn tables(&self) -> (&[u32], &[u32]) { - (&self.ms.hash_table, &self.ms.chain_table) - } -} diff --git a/src/zstd/wip/work/lazy/STATUS.md b/src/zstd/wip/work/lazy/STATUS.md deleted file mode 100644 index 6884f955acc4..000000000000 --- a/src/zstd/wip/work/lazy/STATUS.md +++ /dev/null @@ -1,53 +0,0 @@ -# lazy (zstd_lazy.c -> compress/lazy.rs): status - -## Deliverable -`/root/zstd-ref/work/lazy/zstd/compress/lazy.rs` (same file saved as `/root/zstd-ref/work/lazy/lazy.v2.rs`). -Needs `#![feature(hint_prefetch)]` in lib.rs (safe `core::hint::prefetch_read`). `cargo fmt --check` ok, `cargo clippy`: no -finding in lazy.rs. Public API of the stub unchanged, plus `pub(crate) const ZSTD_ROW_HASH_TAG_BITS` (from zstd_lazy.h; params/cctx -need it for `maxRowHashLog`). Private `prefetch_l1` at the top of the file, to be moved to util.rs. -`lazy.v1.rs` = the same without any prefetch (builds without the feature). - -## Structure -- `ZSTD_compressBlock_lazy_generic` and `..._extDict_generic` are ONE fn `compress_block_lazy_generic` - (`#[inline(never)]`, 21 instances). The only differences in C are kept: the initial `ip += ...`, the saved rep offsets (noDict) and the - lazy-skipping threshold (`step > 8` with step one less in extDict). -- The 4 copies of the repcode check are `rep_length` (0 = no match). `DictSegment` = what an index below dictLimit leads to - (older segment, or the dms with `base - dictIndexDelta`). -- Search fns are `#[inline(never)]` const-generic: `hc_find_best_match`, `bt_find_best_match`, - `row_find_best_match`; `search_max` switches on mls/rowLog at each call as C does. -- `dummy32` is done by aliasing the two slots (`smaller_ptr = larger_ptr`) so the final clear writes only the other one. -- If `iend < 8` (or 16 with rows) as a position in the buffer, `ilimit` would be negative: early return (no search, no hashing, rep unchanged). - -## Tests (all against C, sequences AND final match state) -Harness: `/root/zstd-ref/work/lazy/harness` (+ `shim.c`, reads cctx internals), driver `/root/zstd-ref/work/lazy/zstd/testing.rs` (`frame`). -Scratch-only: `zstd/compress/opt.rs::update_tree` is a throw-away noDict port, only to load dictionaries for btlazy2. - cd /root/zstd-ref/work/lazy/harness && RUSTFLAGS="--cap-lints=warn" cargo build --release - ./target/release/zstd_harness seqs 3,4,5,6 -mode 0 no dict, 1 prefix (extDict), 2 attached CDict (dictMatchState), 3 through ZSTD_compressStream2's wrapping input buffer -(extDict with dict_buf == buf), mixed with none/prefix/CDict. Frames are multi-block (up to 400 KB, windowLog 10..22). -Compared after each frame: every sequence (ll, ml, repcode/offset), hash/chain/tag tables, hashCache, nextToUpdate, hashSaltEntropy, -lazySkipping, lowLimit, dictLimit, loadedDictEnd, rep, whether the dms is still attached. - ./target/release/zstd_harness bench <0|1|2> (build with --profile bench for real numbers) - -## Verified (no mismatch, no error, no overflow panic; harness has overflow checks + debug assertions on) -- First version (no prefetch, plain mask loop), complete runs: mode 0: 320,000 frames; mode 1: 320,000; mode 2: 320,000 (134,306 with a CDict attached). -- Final file (prefetch, mask by chunks of 16), runs stopped by a time limit, not by a failure (logs/final_*.log): mode 0: >=152,000; mode 1: >=140,000; - mode 2: >=306,000 (2 seeds); mode 3: >=284,000 (2 seeds). Plus 3,000 to 8,000 of each mode on other seeds. -- Line coverage (llvm-cov) over modes 0-3: every line of lazy.rs is hit except `?` error paths, `_ => Err(BUG)` and the never-false `off_base != 0`. - `insert_dubt1` with `curr < dict_limit` and the extDict "offset beyond the window" test are only reached by mode 3. - -## Not tested -- Index overflow correction (ZSTD_window_correctOverflow / reduceIndex, DUBT mark preservation): not in this file, not exercised. -- Reusing a context for several frames (hash salt != first value, stale tag table). hashSaltEntropy is compared, the salted second frame is not run. -- CDict copied rather than attached; ldm + lazy (`ZSTD_ldm_blockCompress` calling these); aarch64 only inspected as asm, not run. - -## Speed (7 MB of source text, sequences only, Rust as % of C; logs/bench0.txt) -greedy HC 115, greedy row 108, lazy HC 96, lazy row 88, lazy2 HC 100, lazy2 row L10 77-82, L12 75-78, btlazy2 97-98. -Without prefetch the row finder was 97/78/62/59. -Open: the rest of the row gap. Known causes seen in the asm of `row_find_best_match`: -1. the first 16 tag bytes are assembled from pieces (byte 0 is also loaded as the scalar `head` in an earlier block, and SLP reuses it) instead of - one movdqu: about 15 extra shuffles per search, on aarch64 too. In isolation the fn compiles to movdqu+pcmpeqb+pmovmskb. - Tried without success: `.max(1)` on nb_attempts; loading the chunk as u128 (slower). -2. `match_buffer` is zeroed on each call (64 to 256 bytes); 3. two bounds checks per 4-byte read in the candidate loop (`read_le32(ip + ml - 3)` could - be kept in a variable that changes with `ml`); 4. the rare "skip > 384 positions" path is inlined and costs registers (could be `#[cold]`). -Tried and rejected: prefetch emulated by real loads (`black_box`: +5% at best; early filtering load of the candidates' first 4 bytes: slower). diff --git a/src/zstd/wip/work/lazy/harness/Cargo.toml b/src/zstd/wip/work/lazy/harness/Cargo.toml deleted file mode 100644 index ac7a85d4b3bf..000000000000 --- a/src/zstd/wip/work/lazy/harness/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "zstd_harness" -version = "0.0.0" -edition = "2024" - -[dependencies] -bun_zstd = { path = "/root/zstd-ref/work/lazy/zstd" } - -[profile.release] -debug = "line-tables-only" -overflow-checks = true -debug-assertions = true -panic = "abort" - -[profile.bench] -inherits = "release" -overflow-checks = false -debug-assertions = false - -[workspace] diff --git a/src/zstd/wip/work/lazy/harness/build.rs b/src/zstd/wip/work/lazy/harness/build.rs deleted file mode 100644 index 5dd59e7fcdfd..000000000000 --- a/src/zstd/wip/work/lazy/harness/build.rs +++ /dev/null @@ -1,18 +0,0 @@ -use std::process::Command; -fn main() { - let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); - println!("cargo:rerun-if-env-changed=ZSTD_REF"); - println!("cargo:rerun-if-changed=shim.c"); - let out = std::env::var("OUT_DIR").unwrap(); - let ok = Command::new("clang-23") - .args(["-c", "-O2", "-fPIC", "-march=nehalem", "-DZSTD_MULTITHREAD", "-DZSTD_LEGACY_SUPPORT=0", "-DXXH_NAMESPACE=ZSTD_", - "-I/root/zstd-ref/zstd/lib", "-I/root/zstd-ref/zstd/lib/common", "shim.c", "-o"]) - .arg(format!("{out}/shim.o")).status().unwrap().success(); - assert!(ok); - let _ = std::fs::remove_file(format!("{out}/libshim.a")); - assert!(Command::new("llvm-ar-23").args(["rcs", &format!("{out}/libshim.a"), &format!("{out}/shim.o")]).status().unwrap().success()); - println!("cargo:rustc-link-search=native={out}"); - println!("cargo:rustc-link-lib=static=shim"); - println!("cargo:rustc-link-search=native={}", if variant.starts_with("/") { variant.clone() } else { format!("/root/zstd-ref/build/{variant}") }); - println!("cargo:rustc-link-lib=static=zstd"); -} diff --git a/src/zstd/wip/work/lazy/harness/rust-toolchain.toml b/src/zstd/wip/work/lazy/harness/rust-toolchain.toml deleted file mode 100644 index c71778e8ccd3..000000000000 --- a/src/zstd/wip/work/lazy/harness/rust-toolchain.toml +++ /dev/null @@ -1,35 +0,0 @@ -[toolchain] -channel = "nightly-2026-09-15" -# rust-src is needed for -Zbuild-std (Tier 3 targets like -# aarch64-unknown-freebsd have no prebuilt std). miri is for -# `bun run rust:miri`. targets ensures the -# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain -# (CI's images install this same channel with these components and -# targets, from scripts/build/ci-images/spec.ts; a source lint keeps -# the two files in step). Note: rustup ignores this whole file when -# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree -# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` -# self-heals that path. Keep this list as a fallback for direct -# `cargo build`/`cargo check`/rust-analyzer use. -# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd -# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). -# darwin/windows triples build natively on their own agents, so the host std -# comes with the toolchain install regardless; they're listed here so a dev -# running `bun run rust:check-all` or `cargo check --target` from any host -# still gets prebuilt std for every Tier 1/2 triple. -# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at -# what it emits. The build does not use them. -components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] -targets = [ - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "x86_64-unknown-linux-musl", - "aarch64-linux-android", - "x86_64-linux-android", - "x86_64-unknown-freebsd", - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-pc-windows-msvc", - "x86_64-pc-windows-msvc", -] diff --git a/src/zstd/wip/work/lazy/harness/shim.c b/src/zstd/wip/work/lazy/harness/shim.c deleted file mode 100644 index 08a216c4ed2f..000000000000 --- a/src/zstd/wip/work/lazy/harness/shim.c +++ /dev/null @@ -1,41 +0,0 @@ -// Exposes the internal state of the C library to the differential tests. -#define ZSTD_STATIC_LINKING_ONLY -#include "compress/zstd_compress_internal.h" - -typedef struct { - unsigned cp[7]; - unsigned useRow; - unsigned hasDms; - unsigned nextToUpdate, loadedDictEnd, lowLimit, dictLimit, rowHashLog, hashSaltEntropy, lazySkipping; - unsigned long long hashSalt; - unsigned hashCache[8]; - unsigned rep[3]; - const unsigned* hashTable; - const unsigned* chainTable; - const unsigned char* tagTable; -} ShimState; - -void shim_state(const ZSTD_CCtx* c, ShimState* s) { - const ZSTD_MatchState_t* ms = &c->blockState.matchState; - const ZSTD_compressionParameters* p = &c->appliedParams.cParams; - int i; - s->cp[0] = p->windowLog; s->cp[1] = p->chainLog; s->cp[2] = p->hashLog; s->cp[3] = p->searchLog; - s->cp[4] = p->minMatch; s->cp[5] = p->targetLength; s->cp[6] = (unsigned)p->strategy; - s->useRow = c->appliedParams.useRowMatchFinder == ZSTD_ps_enable; - s->hasDms = ms->dictMatchState != NULL; - s->nextToUpdate = ms->nextToUpdate; s->loadedDictEnd = ms->loadedDictEnd; - s->lowLimit = ms->window.lowLimit; s->dictLimit = ms->window.dictLimit; - s->rowHashLog = ms->rowHashLog; s->hashSaltEntropy = ms->hashSaltEntropy; s->lazySkipping = (unsigned)ms->lazySkipping; - s->hashSalt = ms->hashSalt; - for (i = 0; i < 8; i++) s->hashCache[i] = ms->hashCache[i]; - for (i = 0; i < 3; i++) s->rep[i] = c->blockState.prevCBlock->rep[i]; - s->hashTable = ms->hashTable; s->chainTable = ms->chainTable; s->tagTable = ms->tagTable; -} - -void shim_collect(ZSTD_CCtx* c, ZSTD_Sequence* seqs, size_t cap) { - c->seqCollector.collectSequences = 1; - c->seqCollector.seqStart = seqs; - c->seqCollector.seqIndex = 0; - c->seqCollector.maxSequences = cap; -} -size_t shim_collected(const ZSTD_CCtx* c) { return c->seqCollector.seqIndex; } diff --git a/src/zstd/wip/work/lazy/harness/src/c.rs b/src/zstd/wip/work/lazy/harness/src/c.rs deleted file mode 100644 index d5a82e1f6b78..000000000000 --- a/src/zstd/wip/work/lazy/harness/src/c.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! The reference C library. -#![allow(non_camel_case_types, dead_code)] -use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; - -#[repr(C)] -pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } -#[repr(C)] -pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } -pub enum ZSTD_CCtx {} -pub enum ZSTD_DCtx {} -pub enum ZSTD_DDict {} - -unsafe extern "C" { - pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; - pub fn ZSTD_compressBound(size: usize) -> usize; - pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_isError(code: usize) -> c_uint; - pub fn ZSTD_getErrorCode(code: usize) -> c_uint; - pub fn ZSTD_getErrorName(code: usize) -> *const c_char; - pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; - pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; - pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; - pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; - pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; - pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; - pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; - pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; - pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; - pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; - pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; - pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; - pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; - pub fn ZSTD_defaultCLevel() -> c_int; - pub fn ZSTD_minCLevel() -> c_int; - pub fn ZSTD_maxCLevel() -> c_int; -} - -/// `Ok(value)` or `Err(error code)`. -pub fn res(code: usize) -> Result { - unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } -} - -pub struct CCtx(pub *mut ZSTD_CCtx); -impl CCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } - pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - /// Returns (result, out pos, in pos). - pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); - (r, o.pos, i.pos) - } -} -impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } - -pub struct DCtx(pub *mut ZSTD_DCtx); -impl DCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } - pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } - pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); - (r, o.pos, i.pos) - } -} -impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } - -pub fn compress(src: &[u8], level: i32) -> Vec { - let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; - let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); - dst.truncate(n); - dst -} -pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) -} diff --git a/src/zstd/wip/work/lazy/harness/src/data.rs b/src/zstd/wip/work/lazy/harness/src/data.rs deleted file mode 100644 index c48a645e1b5d..000000000000 --- a/src/zstd/wip/work/lazy/harness/src/data.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Deterministic inputs. -pub struct Rng(pub u64); -impl Rng { - pub fn next(&mut self) -> u64 { - // splitmix64 - self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); - z ^ (z >> 31) - } - pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } - pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } - pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } - pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } - /// Skewed towards small values. - pub fn size(&mut self, max: usize) -> usize { - let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); - (self.next() as usize & ((1usize << bits) - 1)).min(max) - } -} - -static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; - -/// One buffer of about `len` bytes, of a kind picked by `rng`. -pub fn data(rng: &mut Rng, len: usize) -> Vec { - let mut v = Vec::with_capacity(len + 64); - match rng.below(9) { - 0 => v.resize(len, rng.next() as u8), - 1 => while v.len() < len { v.push(rng.next() as u8) }, - 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, - 3 => { // few symbols, skewed - let n = rng.range(2, 40); - while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } - } - 4 => { // LZ-like: copies of earlier parts, at all distances - while v.len() < len { - if v.len() > 8 && rng.chance(3, 4) { - let dist = 1 + rng.size(v.len() - 1); - let n = 3 + rng.size(300); - for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } - } else { - for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } - } - } - } - 5 => { // periodic with noise - let period = rng.range(1, 300); - let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); - while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } - } - 6 => { // sections of different kinds - while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } - } - 7 => { // numbers as text - let mut x = rng.next() % 1000; - while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } - } - _ => { // long runs - while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } - } - } - v.truncate(len); - v -} diff --git a/src/zstd/wip/work/lazy/harness/src/main.rs b/src/zstd/wip/work/lazy/harness/src/main.rs deleted file mode 100644 index 41d45b62720d..000000000000 --- a/src/zstd/wip/work/lazy/harness/src/main.rs +++ /dev/null @@ -1,27 +0,0 @@ -mod c; -mod data; -mod seqs; - -fn main() { - let args: Vec = std::env::args().collect(); - let cmd = args.get(1).map(String::as_str).unwrap_or(""); - let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); - let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); - match cmd { - // seqs - "seqs" => { - let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); - // 0: no dictionary, 1: a prefix or none, 2: an attached CDict or none - let modes: &[seqs::Mode] = match args.get(5).map(String::as_str) { - Some("1") => &[seqs::Mode::NoDict, seqs::Mode::Prefix], - Some("2") => &[seqs::Mode::CDict, seqs::Mode::CDict, seqs::Mode::CDict, seqs::Mode::NoDict], - Some("3") => &[seqs::Mode::NoDict, seqs::Mode::NoDict, seqs::Mode::Prefix, seqs::Mode::CDict], - _ => &[seqs::Mode::NoDict], - }; - // 3: through the buffer of ZSTD_compressStream2, which wraps around - seqs::run(seed, iters, &strategies, modes, args.get(5).is_some_and(|s| s == "3")); - } - "bench" => seqs::bench(&args[2], match args.get(3).map(String::as_str) { Some("1") => seqs::Mode::Prefix, Some("2") => seqs::Mode::CDict, _ => seqs::Mode::NoDict }), - _ => eprintln!("unknown command"), - } -} diff --git a/src/zstd/wip/work/lazy/harness/src/seqs.rs b/src/zstd/wip/work/lazy/harness/src/seqs.rs deleted file mode 100644 index fb7d35f839e4..000000000000 --- a/src/zstd/wip/work/lazy/harness/src/seqs.rs +++ /dev/null @@ -1,193 +0,0 @@ -//! A frame through the Rust block compressors and through the C library (`ZSTD_generateSequences`). -use crate::c; -use crate::data::{Rng, data}; -use bun_zstd::testing::{Dict, State}; -use std::ffi::{c_uint, c_void}; - -#[repr(C)] -#[derive(Clone, Copy, Default, Debug)] -struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } -#[repr(C)] -#[derive(Clone, Copy, Debug)] -struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } -#[repr(C)] -#[derive(Clone, Copy)] -struct CustomMem { a: *const c_void, b: *const c_void, c: *const c_void } -#[repr(C)] -struct ShimState { - cp: [c_uint; 7], use_row: c_uint, has_dms: c_uint, - next_to_update: c_uint, loaded_dict_end: c_uint, low_limit: c_uint, dict_limit: c_uint, row_hash_log: c_uint, hash_salt_entropy: c_uint, lazy_skipping: c_uint, - hash_salt: u64, hash_cache: [c_uint; 8], rep: [c_uint; 3], - hash_table: *const u32, chain_table: *const u32, tag_table: *const u8, -} -enum CDict {} -enum CCtxParams {} -unsafe extern "C" { - fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; - fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; - fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; - fn ZSTD_createCCtxParams() -> *mut CCtxParams; - fn ZSTD_freeCCtxParams(p: *mut CCtxParams) -> usize; - fn ZSTD_CCtxParams_setParameter(p: *mut CCtxParams, param: c_uint, v: i32) -> usize; - fn ZSTD_createCDict_advanced2(d: *const c_void, n: usize, load: c_uint, ty: c_uint, p: *const CCtxParams, mem: CustomMem) -> *mut CDict; - fn ZSTD_freeCDict(d: *mut CDict) -> usize; - fn ZSTD_CCtx_refCDict(c: *mut c::ZSTD_CCtx, d: *const CDict) -> usize; - fn ZSTD_getCParamsFromCDict(d: *const CDict) -> CParams; - fn shim_state(c: *const c::ZSTD_CCtx, s: *mut ShimState); - fn shim_collect(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize); - fn shim_collected(c: *const c::ZSTD_CCtx) -> usize; -} - -fn arr(p: CParams) -> [u32; 7] { [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy] } - -#[derive(Clone, Copy, PartialEq, Debug)] -pub enum Mode { NoDict, Prefix, CDict } - -struct COut { seqs: Vec<(u32, u32, u32, u32)>, cp: [u32; 7], use_row: bool, cdict_cp: [u32; 7], state: State } - -/// `None` if the C library refuses (a last block that is too small). -fn c_frame(cp: [u32; 7], use_row: bool, mode: Mode, dict: &[u8], src: &[u8], streaming: bool) -> Option { - let mut cctx = c::CCtx::new(); - let mut cdict = std::ptr::null_mut(); - let mut cdict_cp = [0; 7]; - cctx.set(1010, 2).unwrap(); // no block splitter - if mode == Mode::CDict { - unsafe { - let p = ZSTD_createCCtxParams(); - for (i, v) in cp.iter().enumerate() { c::res(ZSTD_CCtxParams_setParameter(p, 101 + i as u32, *v as i32)).unwrap(); } - c::res(ZSTD_CCtxParams_setParameter(p, 1011, if use_row { 1 } else { 2 })).unwrap(); - let null = std::ptr::null(); - cdict = ZSTD_createCDict_advanced2(dict.as_ptr().cast(), dict.len(), 1, 1, p, CustomMem { a: null, b: null, c: null }); - assert!(!cdict.is_null()); - ZSTD_freeCCtxParams(p); - c::res(ZSTD_CCtx_refCDict(cctx.0, cdict)).unwrap(); - cdict_cp = arr(ZSTD_getCParamsFromCDict(cdict)); - } - cctx.set(1001, 1).unwrap(); // force attach - cctx.set(101, cp[0] as i32).unwrap(); - } else { - for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } - cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder - if mode == Mode::Prefix { cctx.ref_prefix(dict).unwrap(); } - } - let mut out = vec![Sequence::default(); src.len() / 3 + src.len() / 1024 + 8]; - let n = if streaming { - unsafe { shim_collect(cctx.0, out.as_mut_ptr(), out.len()) }; - let mut dst = vec![0u8; src.len() + src.len() / 64 + 1024]; - let (mut dpos, mut spos) = (0, 0); - let mut r = Ok(0); - while spos < src.len() && r.is_ok() { - (r, dpos, spos) = cctx.stream2(&mut dst, dpos, src, spos, 0); - } - if r.is_ok() { (r, _, _) = cctx.stream2(&mut dst, dpos, src, spos, 2); assert!(r != Ok(1) || true); } - r.map(|left| { assert_eq!(left, 0); unsafe { shim_collected(cctx.0) } }) - } else { - c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }) - }; - let r = n.ok().map(|n| { - let mut s: ShimState = unsafe { std::mem::zeroed() }; - unsafe { shim_state(cctx.0, &mut s) }; - let row_used = s.use_row != 0 && (3..=5).contains(&s.cp[6]); - let table = |p: *const u32, n: usize| if n == 0 { Vec::new() } else { unsafe { std::slice::from_raw_parts(p, n) }.to_vec() }; - let state = State { - has_dms: s.has_dms != 0, next_to_update: s.next_to_update, loaded_dict_end: s.loaded_dict_end, low_limit: s.low_limit, dict_limit: s.dict_limit, - hash_salt_entropy: s.hash_salt_entropy, lazy_skipping: s.lazy_skipping != 0, hash_cache: s.hash_cache, rep: s.rep, - hash_table: table(s.hash_table, 1 << s.cp[2]), - chain_table: table(s.chain_table, if s.cp[6] != 1 && !row_used { 1 << s.cp[1] } else { 0 }), - tag_table: if row_used { unsafe { std::slice::from_raw_parts(s.tag_table, 1 << s.cp[2]) }.to_vec() } else { Vec::new() }, - }; - COut { - seqs: out[..n].iter().map(|s| (s.lit_length, s.match_length, s.rep, if s.rep == 0 { s.offset } else { 0 })).collect(), - cp: s.cp, use_row: s.use_row != 0, cdict_cp, state, - } - }); - if !cdict.is_null() { unsafe { ZSTD_freeCDict(cdict) }; } - r -} - -fn first_diff(a: &[T], b: &[T]) -> Option { - if a.len() != b.len() { return Some(a.len().min(b.len())); } - a.iter().zip(b).position(|(a, b)| a != b) -} - -/// `strategies`: which of 1..=9 to test. `modes`: which kinds of dictionary. -pub fn run(seed: u64, iters: u64, strategies: &[u32], modes: &[Mode], stream: bool) { - let mut skipped = 0u64; - let mut dms_blocks = 0u64; - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if i % 2000 == 0 { eprintln!("seqs iter {i} (seed {s})"); } - let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071, 400_000]); - let len = rng.range(7, max); - let mut src = data(&mut rng, len); - let mode = *rng.pick(modes); - let dict = if mode != Mode::NoDict { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; - let mode = if dict.is_empty() { Mode::NoDict } else { mode }; - if !dict.is_empty() && rng.chance(2, 3) { - for _ in 0..rng.range(1, 6) { - let n = rng.size(dict.len().min(src.len())); - let at = if rng.chance(1, 3) { dict.len() - n } else { rng.below(dict.len() - n + 1) }; - let to = if rng.chance(1, 4) { 0 } else { rng.below(src.len() - n + 1) }; - src[to..to + n].copy_from_slice(&dict[at..at + n]); - } - } - let strategy = *rng.pick(strategies); - let p = if rng.chance(1, 3) { - // the parameters of a level, with the strategy forced - let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; - p.strategy = strategy; - p - } else { - CParams { - window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, - search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, - target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, - } - }; - let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; - let use_row = rng.chance(1, 2); - let streaming = stream && rng.chance(3, 4); - let Some(c) = c_frame(arr(p), use_row, mode, &dict, &src, streaming) else { skipped += 1; continue }; - let d = match mode { Mode::NoDict => Dict::None, Mode::Prefix => Dict::Prefix(&dict), Mode::CDict => Dict::CDict(&dict, c.cdict_cp) }; - let what = format!("seed {s}: {mode:?} streaming {streaming} cp {:?} cdict cp {:?} row {} dict {} src {}", c.cp, c.cdict_cp, c.use_row, dict.len(), src.len()); - let (got, state) = bun_zstd::testing::frame(c.cp, c.use_row, d, &src, streaming).unwrap_or_else(|e| panic!("{what}: error {e:?}")); - if mode == Mode::CDict && dict.len() >= 8 { dms_blocks += 1; } - if let Some(at) = first_diff(&got, &c.seqs) { - panic!("{what}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", got.get(at), c.seqs.get(at), got.len(), c.seqs.len()); - } - if state != c.state { - let (a, b) = (&state, &c.state); - panic!("{what}: state differs: hash table {:?} chain table {:?} tag table {:?}\n got {:?}\n exp {:?}", - first_diff(&a.hash_table, &b.hash_table), first_diff(&a.chain_table, &b.chain_table), first_diff(&a.tag_table, &b.tag_table), - (a.has_dms, a.next_to_update, a.loaded_dict_end, a.low_limit, a.dict_limit, a.hash_salt_entropy, a.lazy_skipping, a.hash_cache, a.rep), - (b.has_dms, b.next_to_update, b.loaded_dict_end, b.low_limit, b.dict_limit, b.hash_salt_entropy, b.lazy_skipping, b.hash_cache, b.rep)); - } - } - println!("seqs ok: {iters} iterations from seed {seed} ({skipped} skipped, {dms_blocks} with an attached CDict), strategies {strategies:?}, modes {modes:?}, streaming {stream}"); -} - -/// Speed of the Rust block compressors against the C ones, on a file. -pub fn bench(path: &str, mode: Mode) { - let all = std::fs::read(path).unwrap(); - let (dict, src) = if mode == Mode::NoDict { (&[][..], &all[..]) } else { all.split_at(100_000.min(all.len() / 2)) }; - for (level, strategy, use_row) in [(5, 3, false), (5, 3, true), (7, 4, false), (7, 4, true), (10, 5, false), (10, 5, true), (12, 5, true), (13, 6, false), (15, 6, false)] { - let mut p = unsafe { ZSTD_getCParams(level, if mode == Mode::CDict { 100_000 } else { src.len() as u64 }, dict.len()) }; - p.strategy = strategy; - let mut best = [f64::MAX; 2]; - let mut nseq = [0usize; 2]; - for _ in 0..3 { - let t = std::time::Instant::now(); - let c = c_frame(arr(p), use_row, mode, dict, src, false).unwrap(); - best[0] = best[0].min(t.elapsed().as_secs_f64()); - nseq[0] = c.seqs.len(); - let d = match mode { Mode::NoDict => Dict::None, Mode::Prefix => Dict::Prefix(dict), Mode::CDict => Dict::CDict(dict, c.cdict_cp) }; - let t = std::time::Instant::now(); - let (got, _) = bun_zstd::testing::frame(c.cp, c.use_row, d, src, false).unwrap(); - best[1] = best[1].min(t.elapsed().as_secs_f64()); - nseq[1] = got.len(); - } - let mb = src.len() as f64 / 1e6; - println!("{mode:?} level {level} strategy {strategy} row {use_row}: C {:.1} MB/s, Rust {:.1} MB/s ({:.0}%) seqs {} {}", mb / best[0], mb / best[1], 100.0 * best[0] / best[1], nseq[0], nseq[1]); - } -} diff --git a/src/zstd/wip/work/lazy/lazy.exp-filter.rs b/src/zstd/wip/work/lazy/lazy.exp-filter.rs deleted file mode 100644 index 916221b41ed5..000000000000 --- a/src/zstd/wip/work/lazy/lazy.exp-filter.rs +++ /dev/null @@ -1,1360 +0,0 @@ -//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. - -use super::internal::{ - BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, - SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, - hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, - offset_to_offbase, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32}; - -const K_LAZY_SKIPPING_STEP: usize = 8; - -// `ZSTD_dictMode_e`, as an argument of a const generic. -const NO_DICT: u8 = DictMode::NoDict as u8; -const EXT_DICT: u8 = DictMode::ExtDict as u8; -const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; - -/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. -#[inline(always)] -const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { - dms.end.wrapping_sub(dms.base) as u32 -} - -// ── binary tree search ──────────────────────────────────────────────────── - -/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. -#[inline(always)] -fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { - let hash_log = ms.c_params.hash_log; - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let bt = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); - *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; - } - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. -fn insert_dubt1( - ms: &mut MatchState, - src: &BlockSrc<'_>, - curr: u32, - input_end: usize, - mut nb_compares: u32, - bt_low: u32, -) -> Result<()> { - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let (ip_buf, ip, iend) = if curr >= dict_limit { - (src.buf, src.pos_of(curr), input_end) - } else { - (src.dict_buf, src.dict_pos_of(curr), dict_end) - }; - let window_valid = ms.window.low_limit; - let max_distance = 1u32 << ms.c_params.window_log; - let window_low = if curr.wrapping_sub(window_valid) > max_distance { - curr - max_distance - } else { - window_valid - }; - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - // The next sorted candidate. The other slot has the previous unsorted one. - let mut match_index = bt.at(smaller_ptr)?; - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let in_prefix = - DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; - // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = if in_prefix { - (src.buf, src.pos_of(match_index)) - } else { - (src.dict_buf, src.dict_pos_of(match_index)) - }; - - if in_prefix || curr < dict_limit { - // Both are in the same segment. - match_length += count( - ip_buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - match_length += count_2segments( - ip_buf, - ip + match_length, - match_buf, - match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if match_index as usize + match_length >= dict_limit as usize { - (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); - } - } - - if ip + match_length == iend { - // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. - break; - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0) -} - -/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. -fn dubt_find_better_dict_match( - ms: &MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mut best_length: usize, - mut nb_compares: u32, - mls: u32, -) -> Result { - let dms = src.dms.ok_or(BUG)?; - let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(h)?; - - let prefix_start = src.pos_of(ms.window.dict_limit); - let curr = src.index_of(ip); - let dict_high_limit = dms_end_index(&dms); - let dict_low_limit = dms.ms.window.low_limit; - let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); - - let dict_bt = dms.ms.chain_table.as_slice(); - let bt_log = dms.ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { - dict_low_limit - } else { - dict_high_limit - bt_mask - }; - - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - - while nb_compares != 0 && dict_match_index > dict_low_limit { - let next_ptr = 2 * (dict_match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let match_index = dict_match_index.wrapping_add(dict_index_delta); - match_length += count_2segments( - src.buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - iend, - dms.end, - prefix_start, - )?; - - if match_length > best_length { - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(offset_ptr.wrapping_add(1)) as i32 - { - best_length = match_length; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // No way to know which is smaller. - break; - } - } - - if dict_match_index <= bt_low { - // Beyond the tree. - break; - } - let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { - let match_pos = src.pos_of(match_index); - src.buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? - }; - if match_byte < src.buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dict_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dict_bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - Ok(best_length) -} - -/// `ZSTD_DUBT_findBestMatch` -#[inline(always)] -fn dubt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - off_base_ptr: &mut u32, - mls: u32, -) -> Result { - let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; - let mut match_index = ms.hash_table.at(h)?; - - let curr = src.index_of(ip); - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = curr.saturating_sub(bt_mask); - let unsort_limit = bt_low.max(window_low); - let mut nb_compares = 1u32 << ms.c_params.search_log; - let mut nb_candidates = nb_compares; - let mut previous_candidate = 0u32; - - // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. - let bt = ms.chain_table.as_mut_slice(); - let mut next_candidate = 2 * (match_index & bt_mask) as usize; - while match_index > unsort_limit - && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK - && nb_candidates > 1 - { - bt.put(next_candidate + 1, previous_candidate)?; - previous_candidate = match_index; - match_index = bt.at(next_candidate)?; - next_candidate = 2 * (match_index & bt_mask) as usize; - nb_candidates -= 1; - } - - // If the last candidate is still unsorted, cut the list there. - if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { - *bt.array_at_mut(next_candidate)? = [0, 0]; - } - - // Sort the candidates, the oldest first. - match_index = previous_candidate; - while match_index != 0 { - let next_candidate_idx = ms - .chain_table - .at(2 * (match_index & bt_mask) as usize + 1)?; - insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; - match_index = next_candidate_idx; - nb_candidates += 1; - } - - // Find the longest match. - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - let mut match_end_idx = curr + 8 + 1; - let mut best_length = 0usize; - - match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); - - if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - src.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - let dict_match_pos = src.dict_pos_of(match_index); - match_length += count_2segments( - src.buf, - ip + match_length, - src.dict_buf, - dict_match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - (match_buf, match_pos) = (src.dict_buf, dict_match_pos); - } - } - - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index + match_length as u32; - } - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(*off_base_ptr) as i32 - { - best_length = match_length; - *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // Equal: no way to know which is smaller. - if DICT_MODE == DICT_MATCH_STATE { - // Nothing in the dictionary can be longer. - nb_compares = 0; - } - break; - } - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0)?; - - if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { - best_length = dubt_find_better_dict_match( - ms, - src, - ip, - iend, - off_base_ptr, - best_length, - nb_compares, - mls, - )?; - } - - // Skip repetitive patterns. - ms.next_to_update = match_end_idx - 8; - Ok(best_length) -} - -/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. -/// `off_base_ptr` is written to if there is one. -#[inline(never)] -fn bt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - off_base_ptr: &mut u32, -) -> Result { - if ip < src.pos_of(ms.next_to_update) { - // In an area that is skipped. - return Ok(0); - } - update_dubt(ms, src.prefix(), ip, MLS)?; - dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) -} - -// ── hash chains ─────────────────────────────────────────────────────────── - -/// `ZSTD_insertAndFindFirstIndex_internal` -#[inline(always)] -fn insert_and_find_first_index_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - lazy_skipping: bool, -) -> Result { - let hash_log = ms.c_params.hash_log; - let chain_mask = (1u32 << ms.c_params.chain_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let chain_table = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); - chain_table.put((idx & chain_mask) as usize, next_in_chain)?; - // While skipping, not every position is inserted. - if lazy_skipping { - break; - } - } - - ms.next_to_update = target; - hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) -} - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) -} - -/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn hc_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let chain_size = 1u32 << ms.c_params.chain_log; - let chain_mask = chain_size - 1; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let min_chain = curr.saturating_sub(chain_size); - let mut nb_attempts = 1u32 << ms.c_params.search_log; - let mut ml = 4 - 1; - - let mut match_index = - insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; - let chain_table = ms.chain_table.as_slice(); - - while match_index >= low_limit && nb_attempts > 0 { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - - if match_index <= min_chain { - break; - } - match_index = chain_table.at((match_index & chain_mask) as usize)?; - nb_attempts -= 1; - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_chain_table = dms.ms.chain_table.as_slice(); - let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; - let dms_chain_mask = dms_chain_size - 1; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_size = dms_end_index(&dms); - let dms_index_delta = dict_limit.wrapping_sub(dms_size); - let dms_min_chain = dms_size.saturating_sub(dms_chain_size); - - let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; - match_index = dms.ms.hash_table.at(h)?; - - while match_index >= dms_lowest_index && nb_attempts > 0 { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - - if match_index <= dms_min_chain { - break; - } - match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; - nb_attempts -= 1; - } - } - - Ok(ml) -} - -// ── row-based match finder ──────────────────────────────────────────────── -// -// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits -// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag -// table is the "head": which entry was written last. - -pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; -const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; -const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; - -/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. -#[inline(always)] -fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { - let mut next = u32::from(*head).wrapping_sub(1) & row_mask; - if next == 0 { - next = row_mask; - } - *head = next as u8; - next -} - -/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. -#[inline(always)] -fn row_fill_hash_cache( - ms: &mut MatchState, - prefix: Prefix<'_>, - mls: u32, - idx: u32, - i_limit: usize, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); - let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; - - for idx in idx..lim { - let hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - ms.hash_salt, - )? as u32; - let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; - ms.hash_cache.put(cached, hash)?; - } - Ok(()) -} - -/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes -/// further takes its place. -#[inline(always)] -fn row_next_cached_hash( - cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], - prefix: Prefix<'_>, - idx: u32, - hash_log: u32, - mls: u32, - hash_salt: u64, -) -> Result { - let new_hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32; - let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; - Ok(core::mem::replace(cached, new_hash)) -} - -/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. -#[inline(always)] -fn row_update_internal_impl( - ms: &mut MatchState, - prefix: Prefix<'_>, - update_start_idx: u32, - update_end_idx: u32, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let hash_salt = ms.hash_salt; - let hash_table = ms.hash_table.as_mut_slice(); - let tag_table = ms.tag_table.as_mut_slice(); - - for idx in update_start_idx..update_end_idx { - let hash = if use_cache { - row_next_cached_hash(&mut ms.hash_cache, prefix, idx, hash_log, mls, hash_salt)? - } else { - hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32 - }; - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; - tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; - hash_table.put(rel_row + pos, idx)?; - } - Ok(()) -} - -/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is -/// when compressing, not all of them if they are many. -#[inline(always)] -fn row_update_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - const K_SKIP_THRESHOLD: u32 = 384; - const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; - const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; - - let mut idx = ms.next_to_update; - let target = prefix.index_of(ip); - - if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { - let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; - row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; - idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; - row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; - } - row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the -/// row-based match finder. -pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { - let row_log = ms.c_params.search_log.clamp(4, 6); - let row_mask = (1u32 << row_log) - 1; - let mls = ms.c_params.min_match.min(6); - row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) -} - -/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. -#[inline(always)] -fn row_get_match_mask( - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - head: u32, -) -> u64 { - let mut matches = 0u64; - let (chunks, _) = tag_row.as_chunks::<16>(); - for (n, chunk) in chunks.iter().enumerate() { - let mut chunk_matches = 0u16; - for (i, &entry) in chunk.iter().enumerate() { - chunk_matches |= u16::from(entry == tag) << i; - } - matches |= u64::from(chunk_matches) << (16 * n); - } - match ROW_ENTRIES { - 16 => u64::from((matches as u16).rotate_right(head)), - 32 => u64::from((matches as u32).rotate_right(head)), - _ => matches.rotate_right(head), - } -} - -/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, -/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than -/// `low_limit`. Returns how many there are. -#[inline(always)] -fn row_collect_matches( - row: &[u32; ROW_ENTRIES], - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - low_limit: u32, - nb_attempts: &mut u32, - match_buffer: &mut [u32; ROW_ENTRIES], - read_match: impl Fn(u32) -> Result, - ip_bytes: u32, -) -> Result { - let row_mask = ROW_ENTRIES as u32 - 1; - let head = u32::from(tag_row.at(0)?) & row_mask; - let mut num_matches = 0usize; - let mut matches = row_get_match_mask(tag_row, tag, head); - - while matches > 0 && *nb_attempts > 0 { - let match_pos = (head + matches.trailing_zeros()) & row_mask; - matches &= matches - 1; - if match_pos == 0 { - // The head, which happens to be the same as the tag. - continue; - } - let match_index = row.at(match_pos as usize)?; - if match_index < low_limit { - break; - } - match_buffer.put(num_matches, match_index)?; - num_matches += usize::from(read_match(match_index)? == ip_bytes); - *nb_attempts -= 1; - } - Ok(num_matches) -} - -/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn row_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let hash_log = ms.row_hash_log; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let row_log = ROW_ENTRIES.trailing_zeros(); - let row_mask = ROW_ENTRIES as u32 - 1; - // There are no more searches than entries in a row. - let capped_search_log = ms.c_params.search_log.min(row_log); - let hash_salt = ms.hash_salt; - let mut nb_attempts = 1u32 << capped_search_log; - let mut ml = 4 - 1; - let mut match_buffer = [0u32; ROW_ENTRIES]; - let ip_bytes = read_le32(buf, ip)?; - - // Update the tables up to, and not including, `ip`. - let hash = if ms.lazy_skipping { - // While skipping, not every position is inserted. - ms.next_to_update = curr; - hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 - } else { - let prefix = src.prefix(); - row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; - row_next_cached_hash(&mut ms.hash_cache, prefix, curr, hash_log, MLS, hash_salt)? - }; - ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); - - { - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; - let row = ms.hash_table.array_at_mut::(rel_row)?; - let tag_row = ms.tag_table.array_at_mut::(rel_row)?; - let num_matches = row_collect_matches( - row, - tag_row, - tag, - low_limit, - &mut nb_attempts, - &mut match_buffer, - |match_index| { - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - read_le32(buf, src.pos_of(match_index)) - } else { - read_le32(src.dict_buf, src.dict_pos_of(match_index)) - } - }, - ip_bytes, - )?; - - // Insert `ip`. - let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; - tag_row.put(pos, tag)?; - row.put(pos, ms.next_to_update)?; - ms.next_to_update += 1; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - } - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); - - let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; - let dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; - let dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let num_matches = row_collect_matches( - dms.ms.hash_table.array_at::(dms_rel_row)?, - dms.ms.tag_table.array_at::(dms_rel_row)?, - (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, - dms_lowest_index, - &mut nb_attempts, - &mut match_buffer, - |match_index| read_le32(dms.buf, dms.pos_of(match_index)), - ip_bytes, - )?; - - for &match_index in match_buffer.sub(..num_matches)? { - let current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - dms.pos_of(match_index) + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - } - } - - Ok(ml) -} - -// ── the parsers ─────────────────────────────────────────────────────────── - -// `searchMethod_e`, as an argument of a const generic. -const SEARCH_HASH_CHAIN: u8 = 0; -const SEARCH_BINARY_TREE: u8 = 1; -const SEARCH_ROW_HASH: u8 = 2; - -/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. -#[inline(always)] -fn search_max( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mls: u32, - row_log: u32, -) -> Result { - match SEARCH_METHOD { - SEARCH_HASH_CHAIN => match mls { - 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - SEARCH_BINARY_TREE => match mls { - 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - _ => match (mls, row_log) { - (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - } -} - -/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match -/// state is searched. -#[derive(Clone, Copy)] -struct DictSegment<'a> { - buf: &'a [u8], - /// Its sum with an index of the window is a position in `buf`. It wraps around. - base: usize, - /// Where the segment starts in `buf`. - start: usize, - /// Where it ends. - end: usize, -} - -/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. -#[inline(always)] -fn rep_length( - ms: &MatchState, - src: &BlockSrc<'_>, - dict: &DictSegment<'_>, - ip: usize, - offset: u32, -) -> Result { - let buf = src.buf; - let iend = src.end; - if DICT_MODE == NO_DICT { - let rep_match = ip.wrapping_sub(offset as usize); - if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { - return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); - } - return Ok(0); - } - - let dict_limit = ms.window.dict_limit; - let curr = src.index_of(ip); - let rep_index = curr.wrapping_sub(offset); - if !index_overlap_check(dict_limit, rep_index) { - return Ok(0); - } - if DICT_MODE == EXT_DICT { - // The offset may lead to what is no longer in the window. - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - if offset > curr.wrapping_sub(window_low) { - return Ok(0); - } - } - let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { - let rep_match = dict.base.wrapping_add(rep_index as usize); - (dict.buf, rep_match, dict.end) - } else { - (buf, src.pos_of(rep_index), iend) - }; - if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { - return Ok(0); - } - let prefix_start = src.pos_of(dict_limit); - Ok(count_2segments( - buf, - ip + 4, - rep_buf, - rep_match + 4, - iend, - rep_end, - prefix_start, - )? + 4) -} - -/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions -/// further a better match is looked for, once there is one. -#[inline(never)] -fn compress_block_lazy_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let buf = src.buf; - let istart = src.start; - let mut ip = istart; - let mut anchor = istart; - let iend = src.end; - let prefix_lowest_index = ms.window.dict_limit; - let prefix_lowest = src.pos_of(prefix_lowest_index); - let mls = ms.c_params.min_match.clamp(4, 6); - let row_log = ms.c_params.search_log.clamp(4, 6); - - let [rep_0, rep_1, _] = rep; - let mut offset_1 = *rep_0; - let mut offset_2 = *rep_1; - let mut offset_saved_1 = 0u32; - let mut offset_saved_2 = 0u32; - - ms.lazy_skipping = false; - - let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { - 8 + ZSTD_ROW_HASH_CACHE_SIZE - } else { - 8 - }; - let Some(ilimit) = iend.checked_sub(margin) else { - // There is no position to search, nor to hash. - return Ok(iend - istart); - }; - - let dict = match DICT_MODE { - NO_DICT => DictSegment { - buf: &[], - base: 0, - start: 0, - end: 0, - }, - EXT_DICT => DictSegment { - buf: src.dict_buf, - base: src.dict_base, - start: src.dict_pos_of(ms.window.low_limit), - end: src.dict_pos_of(prefix_lowest_index), - }, - _ => { - let dms = src.dms.ok_or(BUG)?; - let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); - DictSegment { - buf: dms.buf, - base: dms.base.wrapping_sub(dict_index_delta as usize), - start: dms.pos_of(dms.ms.window.dict_limit), - end: dms.end, - } - } - }; - - if DICT_MODE == EXT_DICT { - ip += usize::from(ip == prefix_lowest); - } else { - let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); - ip += usize::from(dict_and_prefix_length == 0); - } - if DICT_MODE == NO_DICT { - let curr = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); - let max_rep = curr - window_low; - if offset_2 > max_rep { - offset_saved_2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved_1 = offset_1; - offset_1 = 0; - } - } - - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; - } - - 'parse: while ip < ilimit { - let mut match_length; - let mut off_base = REPCODE1_TO_OFFBASE; - let mut start = ip + 1; - - 'found: { - // Check the repeat offset. - match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; - if DEPTH == 0 && match_length != 0 { - break 'found; - } - - // First search. - { - let mut off_base_found = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut off_base_found, - mls, - row_log, - )?; - if ml2 > match_length { - match_length = ml2; - start = ip; - off_base = off_base_found; - } - } - - if match_length < 4 { - // Jump faster over incompressible sections. - let step = (ip - anchor) >> K_SEARCH_STRENGTH; - ip += step + 1; - // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of - // no use out of the tables. As in the C code, it does not start at the same step in all modes. - ms.lazy_skipping = if DICT_MODE == EXT_DICT { - step > K_LAZY_SKIPPING_STEP - } else { - step + 1 > K_LAZY_SKIPPING_STEP - }; - continue 'parse; - } - - // Look for something better at the next positions. - if DEPTH >= 1 { - while ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 3) as i32; - let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - - // Nothing better: try one position further. - if DEPTH == 2 && ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 4) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - } - break; - } - } - - // Catch up: the match may start earlier. - if offbase_is_offset(off_base) { - let offset = offbase_to_offset(off_base); - let match_index = src.index_of(start).wrapping_sub(offset); - let (match_buf, mut match_pos, m_start) = - if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { - let match_pos = dict.base.wrapping_add(match_index as usize); - (dict.buf, match_pos, dict.start) - } else { - (buf, src.pos_of(match_index), prefix_lowest) - }; - while start > anchor - && match_pos > m_start - && buf.at(start - 1)? == match_buf.at(match_pos - 1)? - { - start -= 1; - match_pos -= 1; - match_length += 1; - } - offset_2 = offset_1; - offset_1 = offset; - } - } - - // Store the sequence. - seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; - ip = start + match_length; - anchor = ip; - - if ms.lazy_skipping { - // There was a match: every position is inserted again. - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; - } - ms.lazy_skipping = false; - } - - // Check the second repeat offset right away. - while ip <= ilimit { - match_length = rep_length::(ms, src, &dict, ip, offset_2)?; - if match_length == 0 { - break; - } - (offset_1, offset_2) = (offset_2, offset_1); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; - ip += match_length; - anchor = ip; - } - } - - // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. - if offset_saved_1 != 0 && offset_1 != 0 { - offset_saved_2 = offset_saved_1; - } - - // Save the repeat offsets for the next block. - *rep_0 = if offset_1 != 0 { - offset_1 - } else { - offset_saved_1 - }; - *rep_1 = if offset_2 != 0 { - offset_2 - } else { - offset_saved_2 - }; - - // The size of the last literals. - Ok(iend - anchor) -} - -/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. -#[inline(always)] -fn compress_block_lazy_any_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - match dict_mode { - DictMode::NoDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::ExtDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - compress_block_lazy_generic::( - ms, seq_store, rep, src, - ) - } - } -} - -/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: -/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. -pub(crate) fn compress_block_lazy( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row: bool, - dict_mode: DictMode, -) -> Result { - match (strategy, use_row) { - (Strategy::Greedy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Greedy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( - ms, seq_store, rep, src, dict_mode, - ), - _ => Err(BUG), - } -} diff --git a/src/zstd/wip/work/lazy/lazy.exp-prefetch.rs b/src/zstd/wip/work/lazy/lazy.exp-prefetch.rs deleted file mode 100644 index 267b0e7c9577..000000000000 --- a/src/zstd/wip/work/lazy/lazy.exp-prefetch.rs +++ /dev/null @@ -1,1384 +0,0 @@ -//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. - -use super::internal::{ - BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, - SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, - hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, - offset_to_offbase, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32}; - -const K_LAZY_SKIPPING_STEP: usize = 8; - -// `ZSTD_dictMode_e`, as an argument of a const generic. -const NO_DICT: u8 = DictMode::NoDict as u8; -const EXT_DICT: u8 = DictMode::ExtDict as u8; -const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; - -/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. -#[inline(always)] -const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { - dms.end.wrapping_sub(dms.base) as u32 -} - -// ── binary tree search ──────────────────────────────────────────────────── - -/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. -#[inline(always)] -fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { - let hash_log = ms.c_params.hash_log; - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let bt = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); - *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; - } - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. -fn insert_dubt1( - ms: &mut MatchState, - src: &BlockSrc<'_>, - curr: u32, - input_end: usize, - mut nb_compares: u32, - bt_low: u32, -) -> Result<()> { - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let (ip_buf, ip, iend) = if curr >= dict_limit { - (src.buf, src.pos_of(curr), input_end) - } else { - (src.dict_buf, src.dict_pos_of(curr), dict_end) - }; - let window_valid = ms.window.low_limit; - let max_distance = 1u32 << ms.c_params.window_log; - let window_low = if curr.wrapping_sub(window_valid) > max_distance { - curr - max_distance - } else { - window_valid - }; - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - // The next sorted candidate. The other slot has the previous unsorted one. - let mut match_index = bt.at(smaller_ptr)?; - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let in_prefix = - DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; - // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = if in_prefix { - (src.buf, src.pos_of(match_index)) - } else { - (src.dict_buf, src.dict_pos_of(match_index)) - }; - - if in_prefix || curr < dict_limit { - // Both are in the same segment. - match_length += count( - ip_buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - match_length += count_2segments( - ip_buf, - ip + match_length, - match_buf, - match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if match_index as usize + match_length >= dict_limit as usize { - (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); - } - } - - if ip + match_length == iend { - // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. - break; - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0) -} - -/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. -fn dubt_find_better_dict_match( - ms: &MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mut best_length: usize, - mut nb_compares: u32, - mls: u32, -) -> Result { - let dms = src.dms.ok_or(BUG)?; - let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(h)?; - - let prefix_start = src.pos_of(ms.window.dict_limit); - let curr = src.index_of(ip); - let dict_high_limit = dms_end_index(&dms); - let dict_low_limit = dms.ms.window.low_limit; - let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); - - let dict_bt = dms.ms.chain_table.as_slice(); - let bt_log = dms.ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { - dict_low_limit - } else { - dict_high_limit - bt_mask - }; - - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - - while nb_compares != 0 && dict_match_index > dict_low_limit { - let next_ptr = 2 * (dict_match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let match_index = dict_match_index.wrapping_add(dict_index_delta); - match_length += count_2segments( - src.buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - iend, - dms.end, - prefix_start, - )?; - - if match_length > best_length { - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(offset_ptr.wrapping_add(1)) as i32 - { - best_length = match_length; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // No way to know which is smaller. - break; - } - } - - if dict_match_index <= bt_low { - // Beyond the tree. - break; - } - let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { - let match_pos = src.pos_of(match_index); - src.buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? - }; - if match_byte < src.buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dict_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dict_bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - Ok(best_length) -} - -/// `ZSTD_DUBT_findBestMatch` -#[inline(always)] -fn dubt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - off_base_ptr: &mut u32, - mls: u32, -) -> Result { - let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; - let mut match_index = ms.hash_table.at(h)?; - - let curr = src.index_of(ip); - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = curr.saturating_sub(bt_mask); - let unsort_limit = bt_low.max(window_low); - let mut nb_compares = 1u32 << ms.c_params.search_log; - let mut nb_candidates = nb_compares; - let mut previous_candidate = 0u32; - - // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. - let bt = ms.chain_table.as_mut_slice(); - let mut next_candidate = 2 * (match_index & bt_mask) as usize; - while match_index > unsort_limit - && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK - && nb_candidates > 1 - { - bt.put(next_candidate + 1, previous_candidate)?; - previous_candidate = match_index; - match_index = bt.at(next_candidate)?; - next_candidate = 2 * (match_index & bt_mask) as usize; - nb_candidates -= 1; - } - - // If the last candidate is still unsorted, cut the list there. - if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { - *bt.array_at_mut(next_candidate)? = [0, 0]; - } - - // Sort the candidates, the oldest first. - match_index = previous_candidate; - while match_index != 0 { - let next_candidate_idx = ms - .chain_table - .at(2 * (match_index & bt_mask) as usize + 1)?; - insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; - match_index = next_candidate_idx; - nb_candidates += 1; - } - - // Find the longest match. - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - let mut match_end_idx = curr + 8 + 1; - let mut best_length = 0usize; - - match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); - - if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - src.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - let dict_match_pos = src.dict_pos_of(match_index); - match_length += count_2segments( - src.buf, - ip + match_length, - src.dict_buf, - dict_match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - (match_buf, match_pos) = (src.dict_buf, dict_match_pos); - } - } - - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index + match_length as u32; - } - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(*off_base_ptr) as i32 - { - best_length = match_length; - *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // Equal: no way to know which is smaller. - if DICT_MODE == DICT_MATCH_STATE { - // Nothing in the dictionary can be longer. - nb_compares = 0; - } - break; - } - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0)?; - - if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { - best_length = dubt_find_better_dict_match( - ms, - src, - ip, - iend, - off_base_ptr, - best_length, - nb_compares, - mls, - )?; - } - - // Skip repetitive patterns. - ms.next_to_update = match_end_idx - 8; - Ok(best_length) -} - -/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. -/// `off_base_ptr` is written to if there is one. -#[inline(never)] -fn bt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - off_base_ptr: &mut u32, -) -> Result { - if ip < src.pos_of(ms.next_to_update) { - // In an area that is skipped. - return Ok(0); - } - update_dubt(ms, src.prefix(), ip, MLS)?; - dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) -} - -// ── hash chains ─────────────────────────────────────────────────────────── - -/// `ZSTD_insertAndFindFirstIndex_internal` -#[inline(always)] -fn insert_and_find_first_index_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - lazy_skipping: bool, -) -> Result { - let hash_log = ms.c_params.hash_log; - let chain_mask = (1u32 << ms.c_params.chain_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let chain_table = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); - chain_table.put((idx & chain_mask) as usize, next_in_chain)?; - // While skipping, not every position is inserted. - if lazy_skipping { - break; - } - } - - ms.next_to_update = target; - hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) -} - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) -} - -/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn hc_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let chain_size = 1u32 << ms.c_params.chain_log; - let chain_mask = chain_size - 1; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let min_chain = curr.saturating_sub(chain_size); - let mut nb_attempts = 1u32 << ms.c_params.search_log; - let mut ml = 4 - 1; - - let mut match_index = - insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; - let chain_table = ms.chain_table.as_slice(); - - while match_index >= low_limit && nb_attempts > 0 { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - - if match_index <= min_chain { - break; - } - match_index = chain_table.at((match_index & chain_mask) as usize)?; - nb_attempts -= 1; - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_chain_table = dms.ms.chain_table.as_slice(); - let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; - let dms_chain_mask = dms_chain_size - 1; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_size = dms_end_index(&dms); - let dms_index_delta = dict_limit.wrapping_sub(dms_size); - let dms_min_chain = dms_size.saturating_sub(dms_chain_size); - - let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; - match_index = dms.ms.hash_table.at(h)?; - - while match_index >= dms_lowest_index && nb_attempts > 0 { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - - if match_index <= dms_min_chain { - break; - } - match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; - nb_attempts -= 1; - } - } - - Ok(ml) -} - -// ── row-based match finder ──────────────────────────────────────────────── -// -// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits -// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag -// table is the "head": which entry was written last. - -pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; -const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; -const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; - -/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. -#[inline(always)] -fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { - let mut next = u32::from(*head).wrapping_sub(1) & row_mask; - if next == 0 { - next = row_mask; - } - *head = next as u8; - next -} - -#[inline(always)] -fn prefetch_l1(slice: &[T], index: usize) { - core::hint::prefetch_read(slice.as_ptr().wrapping_add(index), core::hint::Locality::L1); -} - -#[inline(always)] -fn row_prefetch(hash_table: &[u32], tag_table: &[u8], rel_row: usize, row_log: u32) { - prefetch_l1(hash_table, rel_row); - if row_log >= 5 { - prefetch_l1(hash_table, rel_row + 16); - } - prefetch_l1(tag_table, rel_row); - if row_log == 6 { - prefetch_l1(tag_table, rel_row + 32); - } -} - -/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. -#[inline(always)] -fn row_fill_hash_cache( - ms: &mut MatchState, - prefix: Prefix<'_>, - mls: u32, - idx: u32, - i_limit: usize, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); - let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; - - for idx in idx..lim { - let hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - ms.hash_salt, - )? as u32; - let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; - ms.hash_cache.put(cached, hash)?; - } - Ok(()) -} - -/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes -/// further takes its place. -#[inline(always)] -fn row_next_cached_hash( - cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], - hash_table: &[u32], - tag_table: &[u8], - row_log: u32, - prefix: Prefix<'_>, - idx: u32, - hash_log: u32, - mls: u32, - hash_salt: u64, -) -> Result { - let new_hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32; - row_prefetch(hash_table, tag_table, ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize, row_log); - let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; - Ok(core::mem::replace(cached, new_hash)) -} - -/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. -#[inline(always)] -fn row_update_internal_impl( - ms: &mut MatchState, - prefix: Prefix<'_>, - update_start_idx: u32, - update_end_idx: u32, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let hash_salt = ms.hash_salt; - let hash_table = ms.hash_table.as_mut_slice(); - let tag_table = ms.tag_table.as_mut_slice(); - - for idx in update_start_idx..update_end_idx { - let hash = if use_cache { - row_next_cached_hash(&mut ms.hash_cache, hash_table, tag_table, row_log, prefix, idx, hash_log, mls, hash_salt)? - } else { - hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32 - }; - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; - tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; - hash_table.put(rel_row + pos, idx)?; - } - Ok(()) -} - -/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is -/// when compressing, not all of them if they are many. -#[inline(always)] -fn row_update_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - const K_SKIP_THRESHOLD: u32 = 384; - const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; - const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; - - let mut idx = ms.next_to_update; - let target = prefix.index_of(ip); - - if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { - let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; - row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; - idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; - row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; - } - row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the -/// row-based match finder. -pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { - let row_log = ms.c_params.search_log.clamp(4, 6); - let row_mask = (1u32 << row_log) - 1; - let mls = ms.c_params.min_match.min(6); - row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) -} - -/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. -#[inline(always)] -fn row_get_match_mask( - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - head: u32, -) -> u64 { - let mut matches = 0u64; - let (chunks, _) = tag_row.as_chunks::<16>(); - for (n, chunk) in chunks.iter().enumerate() { - let mut chunk_matches = 0u16; - for (i, &entry) in chunk.iter().enumerate() { - chunk_matches |= u16::from(entry == tag) << i; - } - matches |= u64::from(chunk_matches) << (16 * n); - } - match ROW_ENTRIES { - 16 => u64::from((matches as u16).rotate_right(head)), - 32 => u64::from((matches as u32).rotate_right(head)), - _ => matches.rotate_right(head), - } -} - -/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, -/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than -/// `low_limit`. Returns how many there are. -#[inline(always)] -fn row_collect_matches( - row: &[u32; ROW_ENTRIES], - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - low_limit: u32, - nb_attempts: &mut u32, - match_buffer: &mut [u32; ROW_ENTRIES], - prefetch: impl Fn(u32), -) -> Result { - let row_mask = ROW_ENTRIES as u32 - 1; - let head = u32::from(tag_row.at(0)?) & row_mask; - let mut num_matches = 0usize; - let mut matches = row_get_match_mask(tag_row, tag, head); - - while matches > 0 && *nb_attempts > 0 { - let match_pos = (head + matches.trailing_zeros()) & row_mask; - matches &= matches - 1; - if match_pos == 0 { - // The head, which happens to be the same as the tag. - continue; - } - let match_index = row.at(match_pos as usize)?; - if match_index < low_limit { - break; - } - prefetch(match_index); - match_buffer.put(num_matches, match_index)?; - num_matches += 1; - *nb_attempts -= 1; - } - Ok(num_matches) -} - -/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn row_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let hash_log = ms.row_hash_log; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let row_log = ROW_ENTRIES.trailing_zeros(); - let row_mask = ROW_ENTRIES as u32 - 1; - // There are no more searches than entries in a row. - let capped_search_log = ms.c_params.search_log.min(row_log); - let hash_salt = ms.hash_salt; - let mut nb_attempts = 1u32 << capped_search_log; - let mut ml = 4 - 1; - let mut match_buffer = [0u32; ROW_ENTRIES]; - - // Update the tables up to, and not including, `ip`. - let hash = if ms.lazy_skipping { - // While skipping, not every position is inserted. - ms.next_to_update = curr; - hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 - } else { - let prefix = src.prefix(); - row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; - row_next_cached_hash(&mut ms.hash_cache, &ms.hash_table, &ms.tag_table, row_log, prefix, curr, hash_log, MLS, hash_salt)? - }; - ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); - - { - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; - let row = ms.hash_table.array_at_mut::(rel_row)?; - let tag_row = ms.tag_table.array_at_mut::(rel_row)?; - let num_matches = row_collect_matches( - row, - tag_row, - tag, - low_limit, - &mut nb_attempts, - &mut match_buffer, - |match_index| { - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - prefetch_l1(buf, src.pos_of(match_index)); - } else { - prefetch_l1(src.dict_buf, src.dict_pos_of(match_index)); - } - }, - )?; - - // Insert `ip`. - let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; - tag_row.put(pos, tag)?; - row.put(pos, ms.next_to_update)?; - ms.next_to_update += 1; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - } - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); - - let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; - let dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; - let dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let num_matches = row_collect_matches( - dms.ms.hash_table.array_at::(dms_rel_row)?, - dms.ms.tag_table.array_at::(dms_rel_row)?, - (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, - dms_lowest_index, - &mut nb_attempts, - &mut match_buffer, - |match_index| prefetch_l1(dms.buf, dms.pos_of(match_index)), - )?; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - } - } - - Ok(ml) -} - -// ── the parsers ─────────────────────────────────────────────────────────── - -// `searchMethod_e`, as an argument of a const generic. -const SEARCH_HASH_CHAIN: u8 = 0; -const SEARCH_BINARY_TREE: u8 = 1; -const SEARCH_ROW_HASH: u8 = 2; - -/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. -#[inline(always)] -fn search_max( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mls: u32, - row_log: u32, -) -> Result { - match SEARCH_METHOD { - SEARCH_HASH_CHAIN => match mls { - 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - SEARCH_BINARY_TREE => match mls { - 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - _ => match (mls, row_log) { - (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - } -} - -/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match -/// state is searched. -#[derive(Clone, Copy)] -struct DictSegment<'a> { - buf: &'a [u8], - /// Its sum with an index of the window is a position in `buf`. It wraps around. - base: usize, - /// Where the segment starts in `buf`. - start: usize, - /// Where it ends. - end: usize, -} - -/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. -#[inline(always)] -fn rep_length( - ms: &MatchState, - src: &BlockSrc<'_>, - dict: &DictSegment<'_>, - ip: usize, - offset: u32, -) -> Result { - let buf = src.buf; - let iend = src.end; - if DICT_MODE == NO_DICT { - let rep_match = ip.wrapping_sub(offset as usize); - if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { - return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); - } - return Ok(0); - } - - let dict_limit = ms.window.dict_limit; - let curr = src.index_of(ip); - let rep_index = curr.wrapping_sub(offset); - if !index_overlap_check(dict_limit, rep_index) { - return Ok(0); - } - if DICT_MODE == EXT_DICT { - // The offset may lead to what is no longer in the window. - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - if offset > curr.wrapping_sub(window_low) { - return Ok(0); - } - } - let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { - let rep_match = dict.base.wrapping_add(rep_index as usize); - (dict.buf, rep_match, dict.end) - } else { - (buf, src.pos_of(rep_index), iend) - }; - if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { - return Ok(0); - } - let prefix_start = src.pos_of(dict_limit); - Ok(count_2segments( - buf, - ip + 4, - rep_buf, - rep_match + 4, - iend, - rep_end, - prefix_start, - )? + 4) -} - -/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions -/// further a better match is looked for, once there is one. -#[inline(never)] -fn compress_block_lazy_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let buf = src.buf; - let istart = src.start; - let mut ip = istart; - let mut anchor = istart; - let iend = src.end; - let prefix_lowest_index = ms.window.dict_limit; - let prefix_lowest = src.pos_of(prefix_lowest_index); - let mls = ms.c_params.min_match.clamp(4, 6); - let row_log = ms.c_params.search_log.clamp(4, 6); - - let [rep_0, rep_1, _] = rep; - let mut offset_1 = *rep_0; - let mut offset_2 = *rep_1; - let mut offset_saved_1 = 0u32; - let mut offset_saved_2 = 0u32; - - ms.lazy_skipping = false; - - let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { - 8 + ZSTD_ROW_HASH_CACHE_SIZE - } else { - 8 - }; - let Some(ilimit) = iend.checked_sub(margin) else { - // There is no position to search, nor to hash. - return Ok(iend - istart); - }; - - let dict = match DICT_MODE { - NO_DICT => DictSegment { - buf: &[], - base: 0, - start: 0, - end: 0, - }, - EXT_DICT => DictSegment { - buf: src.dict_buf, - base: src.dict_base, - start: src.dict_pos_of(ms.window.low_limit), - end: src.dict_pos_of(prefix_lowest_index), - }, - _ => { - let dms = src.dms.ok_or(BUG)?; - let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); - DictSegment { - buf: dms.buf, - base: dms.base.wrapping_sub(dict_index_delta as usize), - start: dms.pos_of(dms.ms.window.dict_limit), - end: dms.end, - } - } - }; - - if DICT_MODE == EXT_DICT { - ip += usize::from(ip == prefix_lowest); - } else { - let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); - ip += usize::from(dict_and_prefix_length == 0); - } - if DICT_MODE == NO_DICT { - let curr = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); - let max_rep = curr - window_low; - if offset_2 > max_rep { - offset_saved_2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved_1 = offset_1; - offset_1 = 0; - } - } - - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; - } - - 'parse: while ip < ilimit { - let mut match_length; - let mut off_base = REPCODE1_TO_OFFBASE; - let mut start = ip + 1; - - 'found: { - // Check the repeat offset. - match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; - if DEPTH == 0 && match_length != 0 { - break 'found; - } - - // First search. - { - let mut off_base_found = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut off_base_found, - mls, - row_log, - )?; - if ml2 > match_length { - match_length = ml2; - start = ip; - off_base = off_base_found; - } - } - - if match_length < 4 { - // Jump faster over incompressible sections. - let step = (ip - anchor) >> K_SEARCH_STRENGTH; - ip += step + 1; - // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of - // no use out of the tables. As in the C code, it does not start at the same step in all modes. - ms.lazy_skipping = if DICT_MODE == EXT_DICT { - step > K_LAZY_SKIPPING_STEP - } else { - step + 1 > K_LAZY_SKIPPING_STEP - }; - continue 'parse; - } - - // Look for something better at the next positions. - if DEPTH >= 1 { - while ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 3) as i32; - let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - - // Nothing better: try one position further. - if DEPTH == 2 && ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 4) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - } - break; - } - } - - // Catch up: the match may start earlier. - if offbase_is_offset(off_base) { - let offset = offbase_to_offset(off_base); - let match_index = src.index_of(start).wrapping_sub(offset); - let (match_buf, mut match_pos, m_start) = - if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { - let match_pos = dict.base.wrapping_add(match_index as usize); - (dict.buf, match_pos, dict.start) - } else { - (buf, src.pos_of(match_index), prefix_lowest) - }; - while start > anchor - && match_pos > m_start - && buf.at(start - 1)? == match_buf.at(match_pos - 1)? - { - start -= 1; - match_pos -= 1; - match_length += 1; - } - offset_2 = offset_1; - offset_1 = offset; - } - } - - // Store the sequence. - seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; - ip = start + match_length; - anchor = ip; - - if ms.lazy_skipping { - // There was a match: every position is inserted again. - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; - } - ms.lazy_skipping = false; - } - - // Check the second repeat offset right away. - while ip <= ilimit { - match_length = rep_length::(ms, src, &dict, ip, offset_2)?; - if match_length == 0 { - break; - } - (offset_1, offset_2) = (offset_2, offset_1); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; - ip += match_length; - anchor = ip; - } - } - - // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. - if offset_saved_1 != 0 && offset_1 != 0 { - offset_saved_2 = offset_saved_1; - } - - // Save the repeat offsets for the next block. - *rep_0 = if offset_1 != 0 { - offset_1 - } else { - offset_saved_1 - }; - *rep_1 = if offset_2 != 0 { - offset_2 - } else { - offset_saved_2 - }; - - // The size of the last literals. - Ok(iend - anchor) -} - -/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. -#[inline(always)] -fn compress_block_lazy_any_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - match dict_mode { - DictMode::NoDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::ExtDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - compress_block_lazy_generic::( - ms, seq_store, rep, src, - ) - } - } -} - -/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: -/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. -pub(crate) fn compress_block_lazy( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row: bool, - dict_mode: DictMode, -) -> Result { - match (strategy, use_row) { - (Strategy::Greedy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Greedy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( - ms, seq_store, rep, src, dict_mode, - ), - _ => Err(BUG), - } -} diff --git a/src/zstd/wip/work/lazy/lazy.v1.rs b/src/zstd/wip/work/lazy/lazy.v1.rs deleted file mode 100644 index 94dbe308ce79..000000000000 --- a/src/zstd/wip/work/lazy/lazy.v1.rs +++ /dev/null @@ -1,1353 +0,0 @@ -//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. - -use super::internal::{ - BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, - SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, - hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, - offset_to_offbase, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32}; - -const K_LAZY_SKIPPING_STEP: usize = 8; - -// `ZSTD_dictMode_e`, as an argument of a const generic. -const NO_DICT: u8 = DictMode::NoDict as u8; -const EXT_DICT: u8 = DictMode::ExtDict as u8; -const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; - -/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. -#[inline(always)] -const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { - dms.end.wrapping_sub(dms.base) as u32 -} - -// ── binary tree search ──────────────────────────────────────────────────── - -/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. -#[inline(always)] -fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { - let hash_log = ms.c_params.hash_log; - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let bt = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); - *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; - } - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. -fn insert_dubt1( - ms: &mut MatchState, - src: &BlockSrc<'_>, - curr: u32, - input_end: usize, - mut nb_compares: u32, - bt_low: u32, -) -> Result<()> { - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let (ip_buf, ip, iend) = if curr >= dict_limit { - (src.buf, src.pos_of(curr), input_end) - } else { - (src.dict_buf, src.dict_pos_of(curr), dict_end) - }; - let window_valid = ms.window.low_limit; - let max_distance = 1u32 << ms.c_params.window_log; - let window_low = if curr.wrapping_sub(window_valid) > max_distance { - curr - max_distance - } else { - window_valid - }; - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - // The next sorted candidate. The other slot has the previous unsorted one. - let mut match_index = bt.at(smaller_ptr)?; - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let in_prefix = - DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; - // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = if in_prefix { - (src.buf, src.pos_of(match_index)) - } else { - (src.dict_buf, src.dict_pos_of(match_index)) - }; - - if in_prefix || curr < dict_limit { - // Both are in the same segment. - match_length += count( - ip_buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - match_length += count_2segments( - ip_buf, - ip + match_length, - match_buf, - match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if match_index as usize + match_length >= dict_limit as usize { - (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); - } - } - - if ip + match_length == iend { - // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. - break; - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0) -} - -/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. -fn dubt_find_better_dict_match( - ms: &MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mut best_length: usize, - mut nb_compares: u32, - mls: u32, -) -> Result { - let dms = src.dms.ok_or(BUG)?; - let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(h)?; - - let prefix_start = src.pos_of(ms.window.dict_limit); - let curr = src.index_of(ip); - let dict_high_limit = dms_end_index(&dms); - let dict_low_limit = dms.ms.window.low_limit; - let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); - - let dict_bt = dms.ms.chain_table.as_slice(); - let bt_log = dms.ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { - dict_low_limit - } else { - dict_high_limit - bt_mask - }; - - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - - while nb_compares != 0 && dict_match_index > dict_low_limit { - let next_ptr = 2 * (dict_match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let match_index = dict_match_index.wrapping_add(dict_index_delta); - match_length += count_2segments( - src.buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - iend, - dms.end, - prefix_start, - )?; - - if match_length > best_length { - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(offset_ptr.wrapping_add(1)) as i32 - { - best_length = match_length; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // No way to know which is smaller. - break; - } - } - - if dict_match_index <= bt_low { - // Beyond the tree. - break; - } - let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { - let match_pos = src.pos_of(match_index); - src.buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? - }; - if match_byte < src.buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dict_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dict_bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - Ok(best_length) -} - -/// `ZSTD_DUBT_findBestMatch` -#[inline(always)] -fn dubt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - off_base_ptr: &mut u32, - mls: u32, -) -> Result { - let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; - let mut match_index = ms.hash_table.at(h)?; - - let curr = src.index_of(ip); - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = curr.saturating_sub(bt_mask); - let unsort_limit = bt_low.max(window_low); - let mut nb_compares = 1u32 << ms.c_params.search_log; - let mut nb_candidates = nb_compares; - let mut previous_candidate = 0u32; - - // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. - let bt = ms.chain_table.as_mut_slice(); - let mut next_candidate = 2 * (match_index & bt_mask) as usize; - while match_index > unsort_limit - && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK - && nb_candidates > 1 - { - bt.put(next_candidate + 1, previous_candidate)?; - previous_candidate = match_index; - match_index = bt.at(next_candidate)?; - next_candidate = 2 * (match_index & bt_mask) as usize; - nb_candidates -= 1; - } - - // If the last candidate is still unsorted, cut the list there. - if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { - *bt.array_at_mut(next_candidate)? = [0, 0]; - } - - // Sort the candidates, the oldest first. - match_index = previous_candidate; - while match_index != 0 { - let next_candidate_idx = ms - .chain_table - .at(2 * (match_index & bt_mask) as usize + 1)?; - insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; - match_index = next_candidate_idx; - nb_candidates += 1; - } - - // Find the longest match. - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - let mut match_end_idx = curr + 8 + 1; - let mut best_length = 0usize; - - match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); - - if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - src.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - let dict_match_pos = src.dict_pos_of(match_index); - match_length += count_2segments( - src.buf, - ip + match_length, - src.dict_buf, - dict_match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - (match_buf, match_pos) = (src.dict_buf, dict_match_pos); - } - } - - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index + match_length as u32; - } - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(*off_base_ptr) as i32 - { - best_length = match_length; - *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // Equal: no way to know which is smaller. - if DICT_MODE == DICT_MATCH_STATE { - // Nothing in the dictionary can be longer. - nb_compares = 0; - } - break; - } - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0)?; - - if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { - best_length = dubt_find_better_dict_match( - ms, - src, - ip, - iend, - off_base_ptr, - best_length, - nb_compares, - mls, - )?; - } - - // Skip repetitive patterns. - ms.next_to_update = match_end_idx - 8; - Ok(best_length) -} - -/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. -/// `off_base_ptr` is written to if there is one. -#[inline(never)] -fn bt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - off_base_ptr: &mut u32, -) -> Result { - if ip < src.pos_of(ms.next_to_update) { - // In an area that is skipped. - return Ok(0); - } - update_dubt(ms, src.prefix(), ip, MLS)?; - dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) -} - -// ── hash chains ─────────────────────────────────────────────────────────── - -/// `ZSTD_insertAndFindFirstIndex_internal` -#[inline(always)] -fn insert_and_find_first_index_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - lazy_skipping: bool, -) -> Result { - let hash_log = ms.c_params.hash_log; - let chain_mask = (1u32 << ms.c_params.chain_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let chain_table = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); - chain_table.put((idx & chain_mask) as usize, next_in_chain)?; - // While skipping, not every position is inserted. - if lazy_skipping { - break; - } - } - - ms.next_to_update = target; - hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) -} - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) -} - -/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn hc_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let chain_size = 1u32 << ms.c_params.chain_log; - let chain_mask = chain_size - 1; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let min_chain = curr.saturating_sub(chain_size); - let mut nb_attempts = 1u32 << ms.c_params.search_log; - let mut ml = 4 - 1; - - let mut match_index = - insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; - let chain_table = ms.chain_table.as_slice(); - - while match_index >= low_limit && nb_attempts > 0 { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - - if match_index <= min_chain { - break; - } - match_index = chain_table.at((match_index & chain_mask) as usize)?; - nb_attempts -= 1; - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_chain_table = dms.ms.chain_table.as_slice(); - let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; - let dms_chain_mask = dms_chain_size - 1; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_size = dms_end_index(&dms); - let dms_index_delta = dict_limit.wrapping_sub(dms_size); - let dms_min_chain = dms_size.saturating_sub(dms_chain_size); - - let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; - match_index = dms.ms.hash_table.at(h)?; - - while match_index >= dms_lowest_index && nb_attempts > 0 { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - - if match_index <= dms_min_chain { - break; - } - match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; - nb_attempts -= 1; - } - } - - Ok(ml) -} - -// ── row-based match finder ──────────────────────────────────────────────── -// -// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits -// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag -// table is the "head": which entry was written last. - -pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; -const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; -const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; - -/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. -#[inline(always)] -fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { - let mut next = u32::from(*head).wrapping_sub(1) & row_mask; - if next == 0 { - next = row_mask; - } - *head = next as u8; - next -} - -/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. -#[inline(always)] -fn row_fill_hash_cache( - ms: &mut MatchState, - prefix: Prefix<'_>, - mls: u32, - idx: u32, - i_limit: usize, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); - let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; - - for idx in idx..lim { - let hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - ms.hash_salt, - )? as u32; - let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; - ms.hash_cache.put(cached, hash)?; - } - Ok(()) -} - -/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes -/// further takes its place. -#[inline(always)] -fn row_next_cached_hash( - cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], - prefix: Prefix<'_>, - idx: u32, - hash_log: u32, - mls: u32, - hash_salt: u64, -) -> Result { - let new_hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32; - let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; - Ok(core::mem::replace(cached, new_hash)) -} - -/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. -#[inline(always)] -fn row_update_internal_impl( - ms: &mut MatchState, - prefix: Prefix<'_>, - update_start_idx: u32, - update_end_idx: u32, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let hash_salt = ms.hash_salt; - let hash_table = ms.hash_table.as_mut_slice(); - let tag_table = ms.tag_table.as_mut_slice(); - - for idx in update_start_idx..update_end_idx { - let hash = if use_cache { - row_next_cached_hash(&mut ms.hash_cache, prefix, idx, hash_log, mls, hash_salt)? - } else { - hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32 - }; - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; - tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; - hash_table.put(rel_row + pos, idx)?; - } - Ok(()) -} - -/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is -/// when compressing, not all of them if they are many. -#[inline(always)] -fn row_update_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - const K_SKIP_THRESHOLD: u32 = 384; - const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; - const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; - - let mut idx = ms.next_to_update; - let target = prefix.index_of(ip); - - if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { - let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; - row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; - idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; - row_fill_hash_cache(ms, prefix, mls, idx, ip + 1)?; - } - row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the -/// row-based match finder. -pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { - let row_log = ms.c_params.search_log.clamp(4, 6); - let row_mask = (1u32 << row_log) - 1; - let mls = ms.c_params.min_match.min(6); - row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) -} - -/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. -#[inline(always)] -fn row_get_match_mask( - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - head: u32, -) -> u64 { - let mut matches = 0u64; - let (chunks, _) = tag_row.as_chunks::<16>(); - for (n, chunk) in chunks.iter().enumerate() { - let mut chunk_matches = 0u16; - for (i, &entry) in chunk.iter().enumerate() { - chunk_matches |= u16::from(entry == tag) << i; - } - matches |= u64::from(chunk_matches) << (16 * n); - } - match ROW_ENTRIES { - 16 => u64::from((matches as u16).rotate_right(head)), - 32 => u64::from((matches as u32).rotate_right(head)), - _ => matches.rotate_right(head), - } -} - -/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, -/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than -/// `low_limit`. Returns how many there are. -#[inline(always)] -fn row_collect_matches( - row: &[u32; ROW_ENTRIES], - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - low_limit: u32, - nb_attempts: &mut u32, - match_buffer: &mut [u32; ROW_ENTRIES], -) -> Result { - let row_mask = ROW_ENTRIES as u32 - 1; - let head = u32::from(tag_row.at(0)?) & row_mask; - let mut num_matches = 0usize; - let mut matches = row_get_match_mask(tag_row, tag, head); - - while matches > 0 && *nb_attempts > 0 { - let match_pos = (head + matches.trailing_zeros()) & row_mask; - matches &= matches - 1; - if match_pos == 0 { - // The head, which happens to be the same as the tag. - continue; - } - let match_index = row.at(match_pos as usize)?; - if match_index < low_limit { - break; - } - match_buffer.put(num_matches, match_index)?; - num_matches += 1; - *nb_attempts -= 1; - } - Ok(num_matches) -} - -/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn row_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let hash_log = ms.row_hash_log; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let row_log = ROW_ENTRIES.trailing_zeros(); - let row_mask = ROW_ENTRIES as u32 - 1; - // There are no more searches than entries in a row. - let capped_search_log = ms.c_params.search_log.min(row_log); - let hash_salt = ms.hash_salt; - let mut nb_attempts = 1u32 << capped_search_log; - let mut ml = 4 - 1; - let mut match_buffer = [0u32; ROW_ENTRIES]; - - // Update the tables up to, and not including, `ip`. - let hash = if ms.lazy_skipping { - // While skipping, not every position is inserted. - ms.next_to_update = curr; - hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 - } else { - let prefix = src.prefix(); - row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; - row_next_cached_hash(&mut ms.hash_cache, prefix, curr, hash_log, MLS, hash_salt)? - }; - ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); - - { - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; - let row = ms.hash_table.array_at_mut::(rel_row)?; - let tag_row = ms.tag_table.array_at_mut::(rel_row)?; - let num_matches = row_collect_matches( - row, - tag_row, - tag, - low_limit, - &mut nb_attempts, - &mut match_buffer, - )?; - - // Insert `ip`. - let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; - tag_row.put(pos, tag)?; - row.put(pos, ms.next_to_update)?; - ms.next_to_update += 1; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - } - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); - - let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; - let dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; - let dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let num_matches = row_collect_matches( - dms.ms.hash_table.array_at::(dms_rel_row)?, - dms.ms.tag_table.array_at::(dms_rel_row)?, - (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, - dms_lowest_index, - &mut nb_attempts, - &mut match_buffer, - )?; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - } - } - - Ok(ml) -} - -// ── the parsers ─────────────────────────────────────────────────────────── - -// `searchMethod_e`, as an argument of a const generic. -const SEARCH_HASH_CHAIN: u8 = 0; -const SEARCH_BINARY_TREE: u8 = 1; -const SEARCH_ROW_HASH: u8 = 2; - -/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. -#[inline(always)] -fn search_max( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mls: u32, - row_log: u32, -) -> Result { - match SEARCH_METHOD { - SEARCH_HASH_CHAIN => match mls { - 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - SEARCH_BINARY_TREE => match mls { - 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - _ => match (mls, row_log) { - (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - } -} - -/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match -/// state is searched. -#[derive(Clone, Copy)] -struct DictSegment<'a> { - buf: &'a [u8], - /// Its sum with an index of the window is a position in `buf`. It wraps around. - base: usize, - /// Where the segment starts in `buf`. - start: usize, - /// Where it ends. - end: usize, -} - -/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. -#[inline(always)] -fn rep_length( - ms: &MatchState, - src: &BlockSrc<'_>, - dict: &DictSegment<'_>, - ip: usize, - offset: u32, -) -> Result { - let buf = src.buf; - let iend = src.end; - if DICT_MODE == NO_DICT { - let rep_match = ip.wrapping_sub(offset as usize); - if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { - return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); - } - return Ok(0); - } - - let dict_limit = ms.window.dict_limit; - let curr = src.index_of(ip); - let rep_index = curr.wrapping_sub(offset); - if !index_overlap_check(dict_limit, rep_index) { - return Ok(0); - } - if DICT_MODE == EXT_DICT { - // The offset may lead to what is no longer in the window. - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - if offset > curr.wrapping_sub(window_low) { - return Ok(0); - } - } - let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { - let rep_match = dict.base.wrapping_add(rep_index as usize); - (dict.buf, rep_match, dict.end) - } else { - (buf, src.pos_of(rep_index), iend) - }; - if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { - return Ok(0); - } - let prefix_start = src.pos_of(dict_limit); - Ok(count_2segments( - buf, - ip + 4, - rep_buf, - rep_match + 4, - iend, - rep_end, - prefix_start, - )? + 4) -} - -/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions -/// further a better match is looked for, once there is one. -#[inline(never)] -fn compress_block_lazy_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let buf = src.buf; - let istart = src.start; - let mut ip = istart; - let mut anchor = istart; - let iend = src.end; - let prefix_lowest_index = ms.window.dict_limit; - let prefix_lowest = src.pos_of(prefix_lowest_index); - let mls = ms.c_params.min_match.clamp(4, 6); - let row_log = ms.c_params.search_log.clamp(4, 6); - - let [rep_0, rep_1, _] = rep; - let mut offset_1 = *rep_0; - let mut offset_2 = *rep_1; - let mut offset_saved_1 = 0u32; - let mut offset_saved_2 = 0u32; - - ms.lazy_skipping = false; - - let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { - 8 + ZSTD_ROW_HASH_CACHE_SIZE - } else { - 8 - }; - let Some(ilimit) = iend.checked_sub(margin) else { - // There is no position to search, nor to hash. - return Ok(iend - istart); - }; - - let dict = match DICT_MODE { - NO_DICT => DictSegment { - buf: &[], - base: 0, - start: 0, - end: 0, - }, - EXT_DICT => DictSegment { - buf: src.dict_buf, - base: src.dict_base, - start: src.dict_pos_of(ms.window.low_limit), - end: src.dict_pos_of(prefix_lowest_index), - }, - _ => { - let dms = src.dms.ok_or(BUG)?; - let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); - DictSegment { - buf: dms.buf, - base: dms.base.wrapping_sub(dict_index_delta as usize), - start: dms.pos_of(dms.ms.window.dict_limit), - end: dms.end, - } - } - }; - - if DICT_MODE == EXT_DICT { - ip += usize::from(ip == prefix_lowest); - } else { - let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); - ip += usize::from(dict_and_prefix_length == 0); - } - if DICT_MODE == NO_DICT { - let curr = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); - let max_rep = curr - window_low; - if offset_2 > max_rep { - offset_saved_2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved_1 = offset_1; - offset_1 = 0; - } - } - - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; - } - - 'parse: while ip < ilimit { - let mut match_length; - let mut off_base = REPCODE1_TO_OFFBASE; - let mut start = ip + 1; - - 'found: { - // Check the repeat offset. - match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; - if DEPTH == 0 && match_length != 0 { - break 'found; - } - - // First search. - { - let mut off_base_found = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut off_base_found, - mls, - row_log, - )?; - if ml2 > match_length { - match_length = ml2; - start = ip; - off_base = off_base_found; - } - } - - if match_length < 4 { - // Jump faster over incompressible sections. - let step = (ip - anchor) >> K_SEARCH_STRENGTH; - ip += step + 1; - // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of - // no use out of the tables. As in the C code, it does not start at the same step in all modes. - ms.lazy_skipping = if DICT_MODE == EXT_DICT { - step > K_LAZY_SKIPPING_STEP - } else { - step + 1 > K_LAZY_SKIPPING_STEP - }; - continue 'parse; - } - - // Look for something better at the next positions. - if DEPTH >= 1 { - while ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 3) as i32; - let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - - // Nothing better: try one position further. - if DEPTH == 2 && ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 4) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - } - break; - } - } - - // Catch up: the match may start earlier. - if offbase_is_offset(off_base) { - let offset = offbase_to_offset(off_base); - let match_index = src.index_of(start).wrapping_sub(offset); - let (match_buf, mut match_pos, m_start) = - if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { - let match_pos = dict.base.wrapping_add(match_index as usize); - (dict.buf, match_pos, dict.start) - } else { - (buf, src.pos_of(match_index), prefix_lowest) - }; - while start > anchor - && match_pos > m_start - && buf.at(start - 1)? == match_buf.at(match_pos - 1)? - { - start -= 1; - match_pos -= 1; - match_length += 1; - } - offset_2 = offset_1; - offset_1 = offset; - } - } - - // Store the sequence. - seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; - ip = start + match_length; - anchor = ip; - - if ms.lazy_skipping { - // There was a match: every position is inserted again. - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), mls, ms.next_to_update, ilimit)?; - } - ms.lazy_skipping = false; - } - - // Check the second repeat offset right away. - while ip <= ilimit { - match_length = rep_length::(ms, src, &dict, ip, offset_2)?; - if match_length == 0 { - break; - } - (offset_1, offset_2) = (offset_2, offset_1); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; - ip += match_length; - anchor = ip; - } - } - - // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. - if offset_saved_1 != 0 && offset_1 != 0 { - offset_saved_2 = offset_saved_1; - } - - // Save the repeat offsets for the next block. - *rep_0 = if offset_1 != 0 { - offset_1 - } else { - offset_saved_1 - }; - *rep_1 = if offset_2 != 0 { - offset_2 - } else { - offset_saved_2 - }; - - // The size of the last literals. - Ok(iend - anchor) -} - -/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. -#[inline(always)] -fn compress_block_lazy_any_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - match dict_mode { - DictMode::NoDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::ExtDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - compress_block_lazy_generic::( - ms, seq_store, rep, src, - ) - } - } -} - -/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: -/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. -pub(crate) fn compress_block_lazy( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row: bool, - dict_mode: DictMode, -) -> Result { - match (strategy, use_row) { - (Strategy::Greedy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Greedy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( - ms, seq_store, rep, src, dict_mode, - ), - _ => Err(BUG), - } -} diff --git a/src/zstd/wip/work/lazy/lazy.v2.rs b/src/zstd/wip/work/lazy/lazy.v2.rs deleted file mode 100644 index c19d790c6159..000000000000 --- a/src/zstd/wip/work/lazy/lazy.v2.rs +++ /dev/null @@ -1,1419 +0,0 @@ -//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. - -use core::hint::{Locality, prefetch_read}; - -use super::internal::{ - BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, - SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, - hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, - offset_to_offbase, -}; -use crate::common::ZSTD_REP_NUM; -use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32}; - -const K_LAZY_SKIPPING_STEP: usize = 8; - -// `ZSTD_dictMode_e`, as an argument of a const generic. -const NO_DICT: u8 = DictMode::NoDict as u8; -const EXT_DICT: u8 = DictMode::ExtDict as u8; -const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; - -/// `PREFETCH_L1`: a hint that `slice[index]` is about to be read. `index` may be anything. -#[inline(always)] -fn prefetch_l1(slice: &[T], index: usize) { - prefetch_read(slice.as_ptr().wrapping_add(index), Locality::L1); -} - -/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. -#[inline(always)] -const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { - dms.end.wrapping_sub(dms.base) as u32 -} - -// ── binary tree search ──────────────────────────────────────────────────── - -/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. -#[inline(always)] -fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { - let hash_log = ms.c_params.hash_log; - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let bt = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); - *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; - } - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. -fn insert_dubt1( - ms: &mut MatchState, - src: &BlockSrc<'_>, - curr: u32, - input_end: usize, - mut nb_compares: u32, - bt_low: u32, -) -> Result<()> { - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let (ip_buf, ip, iend) = if curr >= dict_limit { - (src.buf, src.pos_of(curr), input_end) - } else { - (src.dict_buf, src.dict_pos_of(curr), dict_end) - }; - let window_valid = ms.window.low_limit; - let max_distance = 1u32 << ms.c_params.window_log; - let window_low = if curr.wrapping_sub(window_valid) > max_distance { - curr - max_distance - } else { - window_valid - }; - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - // The next sorted candidate. The other slot has the previous unsorted one. - let mut match_index = bt.at(smaller_ptr)?; - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let in_prefix = - DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; - // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = if in_prefix { - (src.buf, src.pos_of(match_index)) - } else { - (src.dict_buf, src.dict_pos_of(match_index)) - }; - - if in_prefix || curr < dict_limit { - // Both are in the same segment. - match_length += count( - ip_buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - match_length += count_2segments( - ip_buf, - ip + match_length, - match_buf, - match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if match_index as usize + match_length >= dict_limit as usize { - (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); - } - } - - if ip + match_length == iend { - // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. - break; - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0) -} - -/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. -fn dubt_find_better_dict_match( - ms: &MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mut best_length: usize, - mut nb_compares: u32, - mls: u32, -) -> Result { - let dms = src.dms.ok_or(BUG)?; - let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(h)?; - - let prefix_start = src.pos_of(ms.window.dict_limit); - let curr = src.index_of(ip); - let dict_high_limit = dms_end_index(&dms); - let dict_low_limit = dms.ms.window.low_limit; - let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); - - let dict_bt = dms.ms.chain_table.as_slice(); - let bt_log = dms.ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { - dict_low_limit - } else { - dict_high_limit - bt_mask - }; - - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - - while nb_compares != 0 && dict_match_index > dict_low_limit { - let next_ptr = 2 * (dict_match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let match_index = dict_match_index.wrapping_add(dict_index_delta); - match_length += count_2segments( - src.buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - iend, - dms.end, - prefix_start, - )?; - - if match_length > best_length { - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(offset_ptr.wrapping_add(1)) as i32 - { - best_length = match_length; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // No way to know which is smaller. - break; - } - } - - if dict_match_index <= bt_low { - // Beyond the tree. - break; - } - let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { - let match_pos = src.pos_of(match_index); - src.buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? - }; - if match_byte < src.buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dict_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dict_bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - Ok(best_length) -} - -/// `ZSTD_DUBT_findBestMatch` -#[inline(always)] -fn dubt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - off_base_ptr: &mut u32, - mls: u32, -) -> Result { - let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; - let mut match_index = ms.hash_table.at(h)?; - - let curr = src.index_of(ip); - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = curr.saturating_sub(bt_mask); - let unsort_limit = bt_low.max(window_low); - let mut nb_compares = 1u32 << ms.c_params.search_log; - let mut nb_candidates = nb_compares; - let mut previous_candidate = 0u32; - - // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. - let bt = ms.chain_table.as_mut_slice(); - let mut next_candidate = 2 * (match_index & bt_mask) as usize; - while match_index > unsort_limit - && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK - && nb_candidates > 1 - { - bt.put(next_candidate + 1, previous_candidate)?; - previous_candidate = match_index; - match_index = bt.at(next_candidate)?; - next_candidate = 2 * (match_index & bt_mask) as usize; - nb_candidates -= 1; - } - - // If the last candidate is still unsorted, cut the list there. - if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { - *bt.array_at_mut(next_candidate)? = [0, 0]; - } - - // Sort the candidates, the oldest first. - match_index = previous_candidate; - while match_index != 0 { - let next_candidate_idx = ms - .chain_table - .at(2 * (match_index & bt_mask) as usize + 1)?; - insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; - match_index = next_candidate_idx; - nb_candidates += 1; - } - - // Find the longest match. - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - let mut match_end_idx = curr + 8 + 1; - let mut best_length = 0usize; - - match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); - - if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - src.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - let dict_match_pos = src.dict_pos_of(match_index); - match_length += count_2segments( - src.buf, - ip + match_length, - src.dict_buf, - dict_match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - (match_buf, match_pos) = (src.dict_buf, dict_match_pos); - } - } - - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index + match_length as u32; - } - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(*off_base_ptr) as i32 - { - best_length = match_length; - *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // Equal: no way to know which is smaller. - if DICT_MODE == DICT_MATCH_STATE { - // Nothing in the dictionary can be longer. - nb_compares = 0; - } - break; - } - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0)?; - - if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { - best_length = dubt_find_better_dict_match( - ms, - src, - ip, - iend, - off_base_ptr, - best_length, - nb_compares, - mls, - )?; - } - - // Skip repetitive patterns. - ms.next_to_update = match_end_idx - 8; - Ok(best_length) -} - -/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. -/// `off_base_ptr` is written to if there is one. -#[inline(never)] -fn bt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - off_base_ptr: &mut u32, -) -> Result { - if ip < src.pos_of(ms.next_to_update) { - // In an area that is skipped. - return Ok(0); - } - update_dubt(ms, src.prefix(), ip, MLS)?; - dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) -} - -// ── hash chains ─────────────────────────────────────────────────────────── - -/// `ZSTD_insertAndFindFirstIndex_internal` -#[inline(always)] -fn insert_and_find_first_index_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - lazy_skipping: bool, -) -> Result { - let hash_log = ms.c_params.hash_log; - let chain_mask = (1u32 << ms.c_params.chain_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let chain_table = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); - chain_table.put((idx & chain_mask) as usize, next_in_chain)?; - // While skipping, not every position is inserted. - if lazy_skipping { - break; - } - } - - ms.next_to_update = target; - hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) -} - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) -} - -/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn hc_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let chain_size = 1u32 << ms.c_params.chain_log; - let chain_mask = chain_size - 1; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let min_chain = curr.saturating_sub(chain_size); - let mut nb_attempts = 1u32 << ms.c_params.search_log; - let mut ml = 4 - 1; - - let mut match_index = - insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; - let chain_table = ms.chain_table.as_slice(); - - while match_index >= low_limit && nb_attempts > 0 { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - - if match_index <= min_chain { - break; - } - match_index = chain_table.at((match_index & chain_mask) as usize)?; - nb_attempts -= 1; - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_chain_table = dms.ms.chain_table.as_slice(); - let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; - let dms_chain_mask = dms_chain_size - 1; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_size = dms_end_index(&dms); - let dms_index_delta = dict_limit.wrapping_sub(dms_size); - let dms_min_chain = dms_size.saturating_sub(dms_chain_size); - - let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; - match_index = dms.ms.hash_table.at(h)?; - - while match_index >= dms_lowest_index && nb_attempts > 0 { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - - if match_index <= dms_min_chain { - break; - } - match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; - nb_attempts -= 1; - } - } - - Ok(ml) -} - -// ── row-based match finder ──────────────────────────────────────────────── -// -// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits -// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag -// table is the "head": which entry was written last. - -pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; -const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; -const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; - -/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. -#[inline(always)] -fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { - let mut next = u32::from(*head).wrapping_sub(1) & row_mask; - if next == 0 { - next = row_mask; - } - *head = next as u8; - next -} - -/// `ZSTD_row_prefetch`: of the row that starts at `rel_row`. -#[inline(always)] -fn row_prefetch(hash_table: &[u32], tag_table: &[u8], rel_row: usize, row_log: u32) { - prefetch_l1(hash_table, rel_row); - if row_log >= 5 { - prefetch_l1(hash_table, rel_row + 16); - } - prefetch_l1(tag_table, rel_row); - if row_log == 6 { - prefetch_l1(tag_table, rel_row + 32); - } -} - -/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. -#[inline(always)] -fn row_fill_hash_cache( - ms: &mut MatchState, - prefix: Prefix<'_>, - row_log: u32, - mls: u32, - idx: u32, - i_limit: usize, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); - let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; - - for idx in idx..lim { - let hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - ms.hash_salt, - )? as u32; - let row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - row_prefetch(&ms.hash_table, &ms.tag_table, row, row_log); - let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; - ms.hash_cache.put(cached, hash)?; - } - Ok(()) -} - -/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes -/// further takes its place. -#[inline(always)] -fn row_next_cached_hash( - cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], - hash_table: &[u32], - tag_table: &[u8], - prefix: Prefix<'_>, - idx: u32, - hash_log: u32, - row_log: u32, - mls: u32, - hash_salt: u64, -) -> Result { - let new_hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32; - let row = ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - row_prefetch(hash_table, tag_table, row, row_log); - let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; - Ok(core::mem::replace(cached, new_hash)) -} - -/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. -#[inline(always)] -fn row_update_internal_impl( - ms: &mut MatchState, - prefix: Prefix<'_>, - update_start_idx: u32, - update_end_idx: u32, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let hash_salt = ms.hash_salt; - let hash_table = ms.hash_table.as_mut_slice(); - let tag_table = ms.tag_table.as_mut_slice(); - - for idx in update_start_idx..update_end_idx { - let hash = if use_cache { - row_next_cached_hash( - &mut ms.hash_cache, - hash_table, - tag_table, - prefix, - idx, - hash_log, - row_log, - mls, - hash_salt, - )? - } else { - hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32 - }; - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; - tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; - hash_table.put(rel_row + pos, idx)?; - } - Ok(()) -} - -/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is -/// when compressing, not all of them if they are many. -#[inline(always)] -fn row_update_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - const K_SKIP_THRESHOLD: u32 = 384; - const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; - const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; - - let mut idx = ms.next_to_update; - let target = prefix.index_of(ip); - - if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { - let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; - row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; - idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; - row_fill_hash_cache(ms, prefix, row_log, mls, idx, ip + 1)?; - } - row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; - ms.next_to_update = target; - Ok(()) -} - -/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the -/// row-based match finder. -pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { - let row_log = ms.c_params.search_log.clamp(4, 6); - let row_mask = (1u32 << row_log) - 1; - let mls = ms.c_params.min_match.min(6); - row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) -} - -/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. -#[inline(always)] -fn row_get_match_mask( - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - head: u32, -) -> u64 { - let mut matches = 0u64; - let (chunks, _) = tag_row.as_chunks::<16>(); - for (n, chunk) in chunks.iter().enumerate() { - let mut chunk_matches = 0u16; - for (i, &entry) in chunk.iter().enumerate() { - chunk_matches |= u16::from(entry == tag) << i; - } - matches |= u64::from(chunk_matches) << (16 * n); - } - match ROW_ENTRIES { - 16 => u64::from((matches as u16).rotate_right(head)), - 32 => u64::from((matches as u32).rotate_right(head)), - _ => matches.rotate_right(head), - } -} - -/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, -/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than -/// `low_limit`. `prefetch` is called with each. Returns how many there are. -#[inline(always)] -fn row_collect_matches( - row: &[u32; ROW_ENTRIES], - tag_row: &[u8; ROW_ENTRIES], - tag: u8, - low_limit: u32, - nb_attempts: &mut u32, - match_buffer: &mut [u32; ROW_ENTRIES], - prefetch: impl Fn(u32), -) -> Result { - let row_mask = ROW_ENTRIES as u32 - 1; - let head = u32::from(tag_row.at(0)?) & row_mask; - let mut num_matches = 0usize; - let mut matches = row_get_match_mask(tag_row, tag, head); - - while matches > 0 && *nb_attempts > 0 { - let match_pos = (head + matches.trailing_zeros()) & row_mask; - matches &= matches - 1; - if match_pos == 0 { - // The head, which happens to be the same as the tag. - continue; - } - let match_index = row.at(match_pos as usize)?; - if match_index < low_limit { - break; - } - prefetch(match_index); - match_buffer.put(num_matches, match_index)?; - num_matches += 1; - *nb_attempts -= 1; - } - Ok(num_matches) -} - -/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn row_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let hash_log = ms.row_hash_log; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let row_log = ROW_ENTRIES.trailing_zeros(); - let row_mask = ROW_ENTRIES as u32 - 1; - // There are no more searches than entries in a row. - let capped_search_log = ms.c_params.search_log.min(row_log); - let hash_salt = ms.hash_salt; - let mut nb_attempts = 1u32 << capped_search_log; - let mut ml = 4 - 1; - let mut match_buffer = [0u32; ROW_ENTRIES]; - - let mut dms_hash = 0u32; - let mut dms_rel_row = 0usize; - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; - dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; - dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - row_prefetch(&dms.ms.hash_table, &dms.ms.tag_table, dms_rel_row, row_log); - } - - // Update the tables up to, and not including, `ip`. - let hash = if ms.lazy_skipping { - // While skipping, not every position is inserted. - ms.next_to_update = curr; - hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 - } else { - let prefix = src.prefix(); - row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; - row_next_cached_hash( - &mut ms.hash_cache, - &ms.hash_table, - &ms.tag_table, - prefix, - curr, - hash_log, - row_log, - MLS, - hash_salt, - )? - }; - ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); - - { - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; - let row = ms.hash_table.array_at_mut::(rel_row)?; - let tag_row = ms.tag_table.array_at_mut::(rel_row)?; - let num_matches = row_collect_matches( - row, - tag_row, - tag, - low_limit, - &mut nb_attempts, - &mut match_buffer, - |match_index| { - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - prefetch_l1(buf, src.pos_of(match_index)); - } else { - prefetch_l1(src.dict_buf, src.dict_pos_of(match_index)); - } - }, - )?; - - // Insert `ip`. - let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; - tag_row.put(pos, tag)?; - row.put(pos, ms.next_to_update)?; - ms.next_to_update += 1; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; - } - } - - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - } - } - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); - - let num_matches = row_collect_matches( - dms.ms.hash_table.array_at::(dms_rel_row)?, - dms.ms.tag_table.array_at::(dms_rel_row)?, - (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, - dms_lowest_index, - &mut nb_attempts, - &mut match_buffer, - |match_index| prefetch_l1(dms.buf, dms.pos_of(match_index)), - )?; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; - } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } - } - } - } - - Ok(ml) -} - -// ── the parsers ─────────────────────────────────────────────────────────── - -// `searchMethod_e`, as an argument of a const generic. -const SEARCH_HASH_CHAIN: u8 = 0; -const SEARCH_BINARY_TREE: u8 = 1; -const SEARCH_ROW_HASH: u8 = 2; - -/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. -#[inline(always)] -fn search_max( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mls: u32, - row_log: u32, -) -> Result { - match SEARCH_METHOD { - SEARCH_HASH_CHAIN => match mls { - 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - SEARCH_BINARY_TREE => match mls { - 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - _ => match (mls, row_log) { - (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - } -} - -/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match -/// state is searched. -#[derive(Clone, Copy)] -struct DictSegment<'a> { - buf: &'a [u8], - /// Its sum with an index of the window is a position in `buf`. It wraps around. - base: usize, - /// Where the segment starts in `buf`. - start: usize, - /// Where it ends. - end: usize, -} - -/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. -#[inline(always)] -fn rep_length( - ms: &MatchState, - src: &BlockSrc<'_>, - dict: &DictSegment<'_>, - ip: usize, - offset: u32, -) -> Result { - let buf = src.buf; - let iend = src.end; - if DICT_MODE == NO_DICT { - let rep_match = ip.wrapping_sub(offset as usize); - if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { - return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); - } - return Ok(0); - } - - let dict_limit = ms.window.dict_limit; - let curr = src.index_of(ip); - let rep_index = curr.wrapping_sub(offset); - if !index_overlap_check(dict_limit, rep_index) { - return Ok(0); - } - if DICT_MODE == EXT_DICT { - // The offset may lead to what is no longer in the window. - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - if offset > curr.wrapping_sub(window_low) { - return Ok(0); - } - } - let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { - let rep_match = dict.base.wrapping_add(rep_index as usize); - (dict.buf, rep_match, dict.end) - } else { - (buf, src.pos_of(rep_index), iend) - }; - if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { - return Ok(0); - } - let prefix_start = src.pos_of(dict_limit); - Ok(count_2segments( - buf, - ip + 4, - rep_buf, - rep_match + 4, - iend, - rep_end, - prefix_start, - )? + 4) -} - -/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions -/// further a better match is looked for, once there is one. -#[inline(never)] -fn compress_block_lazy_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let buf = src.buf; - let istart = src.start; - let mut ip = istart; - let mut anchor = istart; - let iend = src.end; - let prefix_lowest_index = ms.window.dict_limit; - let prefix_lowest = src.pos_of(prefix_lowest_index); - let mls = ms.c_params.min_match.clamp(4, 6); - let row_log = ms.c_params.search_log.clamp(4, 6); - - let [rep_0, rep_1, _] = rep; - let mut offset_1 = *rep_0; - let mut offset_2 = *rep_1; - let mut offset_saved_1 = 0u32; - let mut offset_saved_2 = 0u32; - - ms.lazy_skipping = false; - - let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { - 8 + ZSTD_ROW_HASH_CACHE_SIZE - } else { - 8 - }; - let Some(ilimit) = iend.checked_sub(margin) else { - // There is no position to search, nor to hash. - return Ok(iend - istart); - }; - - let dict = match DICT_MODE { - NO_DICT => DictSegment { - buf: &[], - base: 0, - start: 0, - end: 0, - }, - EXT_DICT => DictSegment { - buf: src.dict_buf, - base: src.dict_base, - start: src.dict_pos_of(ms.window.low_limit), - end: src.dict_pos_of(prefix_lowest_index), - }, - _ => { - let dms = src.dms.ok_or(BUG)?; - let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); - DictSegment { - buf: dms.buf, - base: dms.base.wrapping_sub(dict_index_delta as usize), - start: dms.pos_of(dms.ms.window.dict_limit), - end: dms.end, - } - } - }; - - if DICT_MODE == EXT_DICT { - ip += usize::from(ip == prefix_lowest); - } else { - let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); - ip += usize::from(dict_and_prefix_length == 0); - } - if DICT_MODE == NO_DICT { - let curr = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); - let max_rep = curr - window_low; - if offset_2 > max_rep { - offset_saved_2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved_1 = offset_1; - offset_1 = 0; - } - } - - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), row_log, mls, ms.next_to_update, ilimit)?; - } - - 'parse: while ip < ilimit { - let mut match_length; - let mut off_base = REPCODE1_TO_OFFBASE; - let mut start = ip + 1; - - 'found: { - // Check the repeat offset. - match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; - if DEPTH == 0 && match_length != 0 { - break 'found; - } - - // First search. - { - let mut off_base_found = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut off_base_found, - mls, - row_log, - )?; - if ml2 > match_length { - match_length = ml2; - start = ip; - off_base = off_base_found; - } - } - - if match_length < 4 { - // Jump faster over incompressible sections. - let step = (ip - anchor) >> K_SEARCH_STRENGTH; - ip += step + 1; - // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of - // no use out of the tables. As in the C code, it does not start at the same step in all modes. - ms.lazy_skipping = if DICT_MODE == EXT_DICT { - step > K_LAZY_SKIPPING_STEP - } else { - step + 1 > K_LAZY_SKIPPING_STEP - }; - continue 'parse; - } - - // Look for something better at the next positions. - if DEPTH >= 1 { - while ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 3) as i32; - let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - - // Nothing better: try one position further. - if DEPTH == 2 && ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 4) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - } - break; - } - } - - // Catch up: the match may start earlier. - if offbase_is_offset(off_base) { - let offset = offbase_to_offset(off_base); - let match_index = src.index_of(start).wrapping_sub(offset); - let (match_buf, mut match_pos, m_start) = - if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { - let match_pos = dict.base.wrapping_add(match_index as usize); - (dict.buf, match_pos, dict.start) - } else { - (buf, src.pos_of(match_index), prefix_lowest) - }; - while start > anchor - && match_pos > m_start - && buf.at(start - 1)? == match_buf.at(match_pos - 1)? - { - start -= 1; - match_pos -= 1; - match_length += 1; - } - offset_2 = offset_1; - offset_1 = offset; - } - } - - // Store the sequence. - seq_store.store_seq(start - anchor, buf, anchor, off_base, match_length)?; - ip = start + match_length; - anchor = ip; - - if ms.lazy_skipping { - // There was a match: every position is inserted again. - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), row_log, mls, ms.next_to_update, ilimit)?; - } - ms.lazy_skipping = false; - } - - // Check the second repeat offset right away. - while ip <= ilimit { - match_length = rep_length::(ms, src, &dict, ip, offset_2)?; - if match_length == 0 { - break; - } - (offset_1, offset_2) = (offset_2, offset_1); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; - ip += match_length; - anchor = ip; - } - } - - // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. - if offset_saved_1 != 0 && offset_1 != 0 { - offset_saved_2 = offset_saved_1; - } - - // Save the repeat offsets for the next block. - *rep_0 = if offset_1 != 0 { - offset_1 - } else { - offset_saved_1 - }; - *rep_1 = if offset_2 != 0 { - offset_2 - } else { - offset_saved_2 - }; - - // The size of the last literals. - Ok(iend - anchor) -} - -/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. -#[inline(always)] -fn compress_block_lazy_any_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, -) -> Result { - match dict_mode { - DictMode::NoDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::ExtDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) - } - DictMode::DictMatchState => { - compress_block_lazy_generic::( - ms, seq_store, rep, src, - ) - } - } -} - -/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: -/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. -pub(crate) fn compress_block_lazy( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - strategy: Strategy, - use_row: bool, - dict_mode: DictMode, -) -> Result { - match (strategy, use_row) { - (Strategy::Greedy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Greedy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( - ms, seq_store, rep, src, dict_mode, - ), - _ => Err(BUG), - } -} diff --git a/src/zstd/wip/work/lazy/lib.orig.rs b/src/zstd/wip/work/lazy/lib.orig.rs deleted file mode 100644 index 31c98c72e5e4..000000000000 --- a/src/zstd/wip/work/lazy/lib.orig.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the -//! same bytes in, in both directions, and the same errors. -#![forbid(unsafe_code)] -// Frames come from anywhere and a panic aborts the process. -#![deny( - clippy::indexing_slicing, - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::unreachable -)] -#![warn(unused_must_use)] - -mod common; -mod compress; -mod decompress; -mod error; -mod util; - -pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; -pub use decompress::{ - DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, - find_frame_compressed_size, get_frame_content_size, -}; -pub use error::ErrorCode; - -/// `ZSTD_inBuffer` -pub struct InBuffer<'a> { - pub src: &'a [u8], - /// How much of `src` is read. - pub pos: usize, -} - -/// `ZSTD_outBuffer` -pub struct OutBuffer<'a> { - pub dst: &'a mut [u8], - /// How much of `dst` is written. - pub pos: usize, -} - -/// `ZSTD_ResetDirective` -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum ResetDirective { - SessionOnly = 1, - Parameters = 2, - SessionAndParameters = 3, -} - -// ----------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] -pub enum ZstdError { - /// The output, or the decoder state the frame's window size dictates, could not be allocated. - OutOfMemory, - InvalidZstdData, - DecompressionFailed, - ZstdDecompressionError, - ShortRead, -} - -impl core::fmt::Display for ZstdError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.write_str(<&'static str>::from(self)) - } -} -impl core::error::Error for ZstdError {} - -impl ZstdError { - /// The error for a failed decompression call; `other` is the non-allocation failure. - fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { - if code == ErrorCode::MemoryAllocation { - ZstdError::OutOfMemory - } else { - other - } - } -} - -/// `len` zeros for a decoder to write to. -fn output_buffer(len: usize) -> Result, ZstdError> { - util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) -} - -/// Decompress data, automatically allocating the output buffer. -/// Handles both frames with known and unknown content sizes. -/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. -/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. -pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { - let size = find_decompressed_size(src); - - const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit - - if size == ZSTD_CONTENTSIZE_ERROR { - return Err(ZstdError::InvalidZstdData); - } - - // Use streaming decompression if: - // 1. Content size is unknown, OR - // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) - if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { - let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { - // A frame's output is rarely smaller than its input. - src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) - } else { - // The header size is untrusted: reserve no more than the fast path below would. - MAX_PREALLOCATE_SIZE - }; - let mut list: Vec = Vec::new(); - list.try_reserve_exact(initial_capacity) - .map_err(|_| ZstdError::OutOfMemory)?; - StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; - return Ok(list); - } - - // Fast path: size is known and within reasonable limits - let mut output = output_buffer(size as usize)?; - let written = decompress(&mut output, src) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is -/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. -pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { - const MAX_SIZE: u64 = 16 * 1024 * 1024; - let size = find_decompressed_size(src); - // Also if it is unknown or invalid, which are the two largest values. - if size > MAX_SIZE { - return Err(ZstdError::InvalidZstdData); - } - let mut output = output_buffer(size as usize)?; - let written = DDict::prefix(prefix) - .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum State { - Uninitialized, - Inflating, - End, - Error, -} - -/// Minimum spare output capacity offered to the decoder per call. -const STREAMING_OUTPUT_STEP: usize = 4096; - -// ────────────────────────────────────────────────────────────────────────── -// StreamingDecoder -// ────────────────────────────────────────────────────────────────────────── - -/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) -/// per call, so callers can hold the decoder across multiple body chunks. -pub struct StreamingDecoder { - stream: DCtx, - state: State, - /// The decoder filled its last window and may hold more. `max_output` can end a call there. - output_full: bool, -} - -impl Default for StreamingDecoder { - fn default() -> Self { - Self::new() - } -} - -impl StreamingDecoder { - pub fn new() -> Self { - Self { - stream: DCtx::new(), - state: State::Uninitialized, - output_full: false, - } - } - - #[inline] - pub fn is_inflating(&self) -> bool { - matches!(self.state, State::Inflating) - } - - /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is - /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. - /// Returns `ShortRead` when more input is required and `is_done` is false. - pub fn decompress( - &mut self, - input: &[u8], - out: &mut Vec, - max_output: usize, - is_done: bool, - ) -> Result { - if matches!(self.state, State::End | State::Error) { - return Ok(input.len()); - } - - let mut in_buf = InBuffer { src: input, pos: 0 }; - while matches!(self.state, State::Uninitialized | State::Inflating) { - let next_in_len = input.len() - in_buf.pos; - - // Call the decoder again with no input until it leaves the window short. - if next_in_len == 0 && !self.output_full { - if is_done { - if self.state == State::Inflating { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - self.state = State::End; - } - return Ok(in_buf.pos); - } - - if out.len() >= max_output { - return Ok(in_buf.pos); - } - - if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { - self.state = State::Error; - return Err(ZstdError::OutOfMemory); - } - // The decoder writes to initialized memory only. - let written_before = out.len(); - let window = (out.capacity() - written_before).min(max_output - written_before); - out.resize(written_before + window, 0); - let mut out_buf = OutBuffer { - dst: out.get_mut(written_before..).unwrap_or_default(), - pos: 0, - }; - let read_before = in_buf.pos; - - let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); - let bytes_written = out_buf.pos; - out.truncate(written_before + bytes_written); - let remaining = match result { - Ok(remaining) => remaining, - Err(code) => { - self.state = State::Error; - return Err(ZstdError::for_decompression( - code, - ZstdError::ZstdDecompressionError, - )); - } - }; - - self.output_full = bytes_written == window; - - if remaining == 0 { - // Frame complete, and fully flushed. - self.state = State::Uninitialized; - self.output_full = false; - if in_buf.pos >= input.len() { - if is_done { - self.state = State::End; - } - return Ok(in_buf.pos); - } - // More input available — reinitialize for the next frame. - self.stream.init_stream(); - continue; - } - - self.state = State::Inflating; - - if in_buf.pos - read_before == next_in_len { - if self.output_full { - continue; - } - if is_done { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - return Err(ZstdError::ShortRead); - } - } - Ok(in_buf.pos) - } -} - -// ── compressed embedded assets ──────────────────────────────────────────── - -/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes -/// Bun never executes or parses itself: the shell completion scripts and the -/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, -/// error overlay/page). Anything that runs inside Bun stays uncompressed. -/// -/// The exception is text that only `bun lint`, `bun fmt` and `bun check` -/// read. It may be compressed because it isn't on the path of running a program. -/// -/// Release (`bun_codegen_embed`) builds include the `.zst` twin that -/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to -/// `/compressed/.zst`; other builds evaluate `$fallback`. -#[macro_export] -macro_rules! embed_compressed { - // A codegen output (`/$sub`); debug builds read it at runtime. - (codegen $sub:literal) => { - $crate::embed_compressed!( - ("codegen/", $sub), - ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() - ) - }; - // Same, with a trailing NUL included in the slice (for C-string consumers). - (codegen_nul $sub:literal) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/codegen/", - $sub, - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); - &__COPY.get_or_init(|| { - let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); - let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); - copy.extend_from_slice(text); - copy.push(0); - copy.into_boxed_slice() - })[..] - } - }; - __bytes - }}; - // A file under `src/`; debug builds read it from the source tree at runtime. - (src $sub:literal) => { - $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) - }; - ($name:literal, $fallback:expr) => { - $crate::embed_compressed!(($name), $fallback) - }; - (($($name:literal),+), $fallback:expr) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/", - $($name,)+ - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - $fallback - } - }; - __bytes - }}; -} - -/// Cold, shared body of [`embed_compressed!`]'s release arm. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { - decompress_alloc(compressed) - .expect("embedded asset: invalid zstd frame") - .into_boxed_slice() -} - -/// [`inflate_embedded`] plus a trailing NUL byte. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { - let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); - inflated.reserve_exact(1); - inflated.push(0); - inflated.into_boxed_slice() -} -pub mod testing; diff --git a/src/zstd/wip/work/lazy/zstd/lib.rs b/src/zstd/wip/work/lazy/zstd/lib.rs deleted file mode 100644 index 3b2a2c6fd519..000000000000 --- a/src/zstd/wip/work/lazy/zstd/lib.rs +++ /dev/null @@ -1,386 +0,0 @@ -//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the -//! same bytes in, in both directions, and the same errors. -#![forbid(unsafe_code)] -#![feature(hint_prefetch)] -// Frames come from anywhere and a panic aborts the process. -#![deny( - clippy::indexing_slicing, - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::unreachable -)] -#![warn(unused_must_use)] - -mod common; -mod compress; -mod decompress; -mod error; -mod util; - -pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; -pub use decompress::{ - DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, - find_frame_compressed_size, get_frame_content_size, -}; -pub use error::ErrorCode; - -/// `ZSTD_inBuffer` -pub struct InBuffer<'a> { - pub src: &'a [u8], - /// How much of `src` is read. - pub pos: usize, -} - -/// `ZSTD_outBuffer` -pub struct OutBuffer<'a> { - pub dst: &'a mut [u8], - /// How much of `dst` is written. - pub pos: usize, -} - -/// `ZSTD_ResetDirective` -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum ResetDirective { - SessionOnly = 1, - Parameters = 2, - SessionAndParameters = 3, -} - -// ----------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] -pub enum ZstdError { - /// The output, or the decoder state the frame's window size dictates, could not be allocated. - OutOfMemory, - InvalidZstdData, - DecompressionFailed, - ZstdDecompressionError, - ShortRead, -} - -impl core::fmt::Display for ZstdError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.write_str(<&'static str>::from(self)) - } -} -impl core::error::Error for ZstdError {} - -impl ZstdError { - /// The error for a failed decompression call; `other` is the non-allocation failure. - fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { - if code == ErrorCode::MemoryAllocation { - ZstdError::OutOfMemory - } else { - other - } - } -} - -/// `len` zeros for a decoder to write to. -fn output_buffer(len: usize) -> Result, ZstdError> { - util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) -} - -/// Decompress data, automatically allocating the output buffer. -/// Handles both frames with known and unknown content sizes. -/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. -/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. -pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { - let size = find_decompressed_size(src); - - const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit - - if size == ZSTD_CONTENTSIZE_ERROR { - return Err(ZstdError::InvalidZstdData); - } - - // Use streaming decompression if: - // 1. Content size is unknown, OR - // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) - if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { - let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { - // A frame's output is rarely smaller than its input. - src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) - } else { - // The header size is untrusted: reserve no more than the fast path below would. - MAX_PREALLOCATE_SIZE - }; - let mut list: Vec = Vec::new(); - list.try_reserve_exact(initial_capacity) - .map_err(|_| ZstdError::OutOfMemory)?; - StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; - return Ok(list); - } - - // Fast path: size is known and within reasonable limits - let mut output = output_buffer(size as usize)?; - let written = decompress(&mut output, src) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is -/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. -pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { - const MAX_SIZE: u64 = 16 * 1024 * 1024; - let size = find_decompressed_size(src); - // Also if it is unknown or invalid, which are the two largest values. - if size > MAX_SIZE { - return Err(ZstdError::InvalidZstdData); - } - let mut output = output_buffer(size as usize)?; - let written = DDict::prefix(prefix) - .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum State { - Uninitialized, - Inflating, - End, - Error, -} - -/// Minimum spare output capacity offered to the decoder per call. -const STREAMING_OUTPUT_STEP: usize = 4096; - -// ────────────────────────────────────────────────────────────────────────── -// StreamingDecoder -// ────────────────────────────────────────────────────────────────────────── - -/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) -/// per call, so callers can hold the decoder across multiple body chunks. -pub struct StreamingDecoder { - stream: DCtx, - state: State, - /// The decoder filled its last window and may hold more. `max_output` can end a call there. - output_full: bool, -} - -impl Default for StreamingDecoder { - fn default() -> Self { - Self::new() - } -} - -impl StreamingDecoder { - pub fn new() -> Self { - Self { - stream: DCtx::new(), - state: State::Uninitialized, - output_full: false, - } - } - - #[inline] - pub fn is_inflating(&self) -> bool { - matches!(self.state, State::Inflating) - } - - /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is - /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. - /// Returns `ShortRead` when more input is required and `is_done` is false. - pub fn decompress( - &mut self, - input: &[u8], - out: &mut Vec, - max_output: usize, - is_done: bool, - ) -> Result { - if matches!(self.state, State::End | State::Error) { - return Ok(input.len()); - } - - let mut in_buf = InBuffer { src: input, pos: 0 }; - while matches!(self.state, State::Uninitialized | State::Inflating) { - let next_in_len = input.len() - in_buf.pos; - - // Call the decoder again with no input until it leaves the window short. - if next_in_len == 0 && !self.output_full { - if is_done { - if self.state == State::Inflating { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - self.state = State::End; - } - return Ok(in_buf.pos); - } - - if out.len() >= max_output { - return Ok(in_buf.pos); - } - - if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { - self.state = State::Error; - return Err(ZstdError::OutOfMemory); - } - // The decoder writes to initialized memory only. - let written_before = out.len(); - let window = (out.capacity() - written_before).min(max_output - written_before); - out.resize(written_before + window, 0); - let mut out_buf = OutBuffer { - dst: out.get_mut(written_before..).unwrap_or_default(), - pos: 0, - }; - let read_before = in_buf.pos; - - let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); - let bytes_written = out_buf.pos; - out.truncate(written_before + bytes_written); - let remaining = match result { - Ok(remaining) => remaining, - Err(code) => { - self.state = State::Error; - return Err(ZstdError::for_decompression( - code, - ZstdError::ZstdDecompressionError, - )); - } - }; - - self.output_full = bytes_written == window; - - if remaining == 0 { - // Frame complete, and fully flushed. - self.state = State::Uninitialized; - self.output_full = false; - if in_buf.pos >= input.len() { - if is_done { - self.state = State::End; - } - return Ok(in_buf.pos); - } - // More input available — reinitialize for the next frame. - self.stream.init_stream(); - continue; - } - - self.state = State::Inflating; - - if in_buf.pos - read_before == next_in_len { - if self.output_full { - continue; - } - if is_done { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - return Err(ZstdError::ShortRead); - } - } - Ok(in_buf.pos) - } -} - -// ── compressed embedded assets ──────────────────────────────────────────── - -/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes -/// Bun never executes or parses itself: the shell completion scripts and the -/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, -/// error overlay/page). Anything that runs inside Bun stays uncompressed. -/// -/// The exception is text that only `bun lint`, `bun fmt` and `bun check` -/// read. It may be compressed because it isn't on the path of running a program. -/// -/// Release (`bun_codegen_embed`) builds include the `.zst` twin that -/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to -/// `/compressed/.zst`; other builds evaluate `$fallback`. -#[macro_export] -macro_rules! embed_compressed { - // A codegen output (`/$sub`); debug builds read it at runtime. - (codegen $sub:literal) => { - $crate::embed_compressed!( - ("codegen/", $sub), - ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() - ) - }; - // Same, with a trailing NUL included in the slice (for C-string consumers). - (codegen_nul $sub:literal) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/codegen/", - $sub, - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); - &__COPY.get_or_init(|| { - let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); - let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); - copy.extend_from_slice(text); - copy.push(0); - copy.into_boxed_slice() - })[..] - } - }; - __bytes - }}; - // A file under `src/`; debug builds read it from the source tree at runtime. - (src $sub:literal) => { - $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) - }; - ($name:literal, $fallback:expr) => { - $crate::embed_compressed!(($name), $fallback) - }; - (($($name:literal),+), $fallback:expr) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/", - $($name,)+ - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - $fallback - } - }; - __bytes - }}; -} - -/// Cold, shared body of [`embed_compressed!`]'s release arm. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { - decompress_alloc(compressed) - .expect("embedded asset: invalid zstd frame") - .into_boxed_slice() -} - -/// [`inflate_embedded`] plus a trailing NUL byte. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { - let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); - inflated.reserve_exact(1); - inflated.push(0); - inflated.into_boxed_slice() -} -pub mod testing; diff --git a/src/zstd/wip/work/lazy/zstd/testing.rs b/src/zstd/wip/work/lazy/zstd/testing.rs deleted file mode 100644 index 2e64ee5e7a3e..000000000000 --- a/src/zstd/wip/work/lazy/zstd/testing.rs +++ /dev/null @@ -1,372 +0,0 @@ -//! Hooks for the differential tests. Only in scratch copies of the crate. -#![allow(clippy::all, dead_code, unreachable_pub)] - -use crate::ErrorCode; -use crate::compress::internal::*; -use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; - -fn bitmix(mut val: u64, len: u64) -> u64 { - val ^= val.rotate_right(49) ^ val.rotate_right(24); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^= (val >> 35).wrapping_add(len); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^ (val >> 28) -} - -#[derive(Clone, Copy)] -pub enum Dict<'a> { - None, - /// `ZSTD_CCtx_refPrefix`, raw content. - Prefix(&'a [u8]), - /// `ZSTD_CCtx_refCDict`, raw content, attached. With the parameters of the CDict. - CDict(&'a [u8], [u32; 7]), -} - -/// What is left in the match state after the frame. -#[derive(PartialEq, Eq, Debug, Default)] -pub struct State { - pub has_dms: bool, - pub next_to_update: u32, - pub loaded_dict_end: u32, - pub low_limit: u32, - pub dict_limit: u32, - pub hash_salt_entropy: u32, - pub lazy_skipping: bool, - pub hash_cache: [u32; 8], - pub rep: [u32; 3], - pub hash_table: Vec, - pub chain_table: Vec, - pub tag_table: Vec, -} - -fn params(cp: [u32; 7]) -> Result { - Ok(CompressionParameters { - window_log: cp[0], - chain_log: cp[1], - hash_log: cp[2], - search_log: cp[3], - min_match: cp[4], - target_length: cp[5], - strategy: Some(Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?), - }) -} - -/// `ZSTD_reset_matchState`, on a new one. -fn reset_match_state( - c_params: CompressionParameters, - use_row: bool, - for_cdict: bool, -) -> MatchState { - let strategy = c_params.strat(); - let row_used = row_match_finder_supported(strategy) && use_row; - let mut ms = MatchState::new(); - ms.hash_log3 = if !for_cdict && c_params.min_match == 3 { - c_params.window_log.min(17) - } else { - 0 - }; - ms.window.clear(); - ms.next_to_update = ms.window.dict_limit; - ms.hash_table = vec![0; 1 << c_params.hash_log]; - ms.chain_table = vec![ - 0; - if strategy != Strategy::Fast && !row_used { - 1 << c_params.chain_log - } else { - 0 - } - ]; - ms.hash_table3 = vec![ - 0; - if ms.hash_log3 != 0 { - 1 << ms.hash_log3 - } else { - 0 - } - ]; - if row_used { - ms.tag_table = vec![0; 1 << c_params.hash_log]; - ms.hash_salt = if for_cdict { - 0 - } else { - bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4) - }; - ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); - } - if !for_cdict && strategy >= Strategy::BtOpt { - ms.opt.lit_freq = vec![0; 256]; - ms.opt.lit_length_freq = vec![0; 36]; - ms.opt.match_length_freq = vec![0; 53]; - ms.opt.off_code_freq = vec![0; 32]; - ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; - ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; - } - ms.c_params = c_params; - ms -} - -/// `ZSTD_loadDictionaryContent` -fn load_dictionary_content( - ms: &mut MatchState, - use_row: bool, - dict: &[u8], - dtlm: DictTableLoadMethod, - tfp: TableFillPurpose, -) -> Result<(), ErrorCode> { - let c_params = ms.c_params; - let iend = dict.len(); - ms.window - .update(BufId::Dict.start(), BufId::Dict, dict.len(), false); - let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); - let ip = iend - dict.len().min(max_dict_size); - let prefix = Prefix { - buf: dict, - base: ms.window.base.wrapping_sub(BufId::Dict.start()), - }; - ms.next_to_update = prefix.index_of(ip); - ms.loaded_dict_end = prefix.index_of(iend); - if iend - ip > HASH_READ_SIZE { - match c_params.strat() { - Strategy::Fast => fast::fill_hash_table(ms, prefix, iend, dtlm, tfp)?, - Strategy::DFast => double_fast::fill_double_hash_table(ms, prefix, iend, dtlm, tfp)?, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { - if use_row { - ms.tag_table.fill(0); - lazy::row_update(ms, prefix, iend - HASH_READ_SIZE)?; - } else { - lazy::insert_and_find_first_index(ms, prefix, iend - HASH_READ_SIZE)?; - } - } - _ => opt::update_tree(ms, prefix, iend - HASH_READ_SIZE, iend)?, - } - ms.next_to_update = prefix.index_of(iend); - } - Ok(()) -} - -/// The sequences of all the blocks of a frame, as (literal length, match length, repeat code or 0, offset or 0). Each -/// block ends with the literals that are left as (length, 0, 0, 0). -/// -/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, as applied. -pub fn frame( - cp: [u32; 7], - use_row: bool, - dict: Dict<'_>, - src: &[u8], - streaming: bool, -) -> Result<(Vec<(u32, u32, u32, u32)>, State), ErrorCode> { - let c_params = params(cp)?; - let strategy = c_params.strat(); - let mut ms = reset_match_state(c_params, use_row, false); - - let block_size_max = (1usize << c_params.window_log).min(128 << 10); - let mut seq_store = SeqStore::new(); - seq_store.reserve( - block_size_max / if c_params.min_match == 3 { 3 } else { 4 }, - block_size_max, - )?; - - let dict_content = match dict { - Dict::None => &[][..], - Dict::Prefix(d) | Dict::CDict(d, _) => d, - }; - // As `ZSTD_compressStream2` does: blocks are copied to a buffer of the size of the window and a block. - let mut in_buff = vec![ - 0u8; - if streaming { - (1usize << c_params.window_log) + block_size_max - } else { - 0 - } - ]; - let mut in_buff_pos = 0usize; - - let mut cdict_ms = None; - match dict { - Dict::Prefix(d) if d.len() >= 8 => load_dictionary_content( - &mut ms, - use_row, - d, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - )?, - Dict::CDict(d, cdict_cp) if d.len() >= 8 => { - let mut c = reset_match_state(params(cdict_cp)?, use_row, true); - load_dictionary_content( - &mut c, - use_row, - d, - DictTableLoadMethod::Full, - TableFillPurpose::ForCDict, - )?; - // ZSTD_resetCCtx_byAttachingCDict - let cdict_end = c.window.index_of(c.window.next_src); - if ms.window.dict_limit < cdict_end { - ms.window.next_src = ms.window.base + cdict_end as usize; - ms.window.clear(); - } - ms.loaded_dict_end = ms.window.dict_limit; - cdict_ms = Some(c); - } - _ => {} - } - let mut dms = cdict_ms.as_ref().map(|c| DictMatchState { - ms: c, - buf: dict_content, - base: c.window.base.wrapping_sub(BufId::Dict.start()), - end: c.window.next_src - BufId::Dict.start(), - }); - - // ZSTD_compressContinue_internal - if !streaming - && !ms.window.update( - BufId::User.start(), - BufId::User, - src.len(), - ms.force_non_contiguous, - ) - { - ms.force_non_contiguous = false; - ms.next_to_update = ms.window.dict_limit; - } - // ZSTD_compress_frameChunk - let max_dist = 1u32 << c_params.window_log; - let prev = CompressedBlockState::new(); - let mut rep = prev.rep; - let mut out = Vec::new(); - let mut pos = 0; - while pos < src.len() { - let block_size = (src.len() - pos).min(block_size_max); - let (block_addr, block_start) = if streaming { - in_buff[in_buff_pos..in_buff_pos + block_size] - .copy_from_slice(&src[pos..pos + block_size]); - let addr = BufId::InBuff.start() + in_buff_pos; - if !ms - .window - .update(addr, BufId::InBuff, block_size, ms.force_non_contiguous) - { - ms.force_non_contiguous = false; - ms.next_to_update = ms.window.dict_limit; - } - (addr, in_buff_pos) - } else { - (BufId::User.start() + pos, pos) - }; - let bufs = Buffers { - in_buff: &in_buff, - user: src, - dict: dict_content, - }; - if ms - .window - .dict_is_invalid(block_addr + block_size, max_dist, ms.loaded_dict_end) - { - ms.loaded_dict_end = 0; - dms = None; - } - if ms - .window - .enforce_max_dist(block_addr, max_dist, ms.loaded_dict_end) - { - ms.loaded_dict_end = 0; - dms = None; - } - if ms.next_to_update < ms.window.low_limit { - ms.next_to_update = ms.window.low_limit; - } - // ZSTD_buildSeqStore - assert!(block_size >= 7); - seq_store.reset(); - let block = BlockSrc { - buf: bufs.get(ms.window.base_buf), - base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), - dict_buf: bufs.get(ms.window.dict_buf), - dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), - start: block_start, - end: block_start + block_size, - dms, - symbol_costs: &prev.entropy, - ldm_seq_store: None, - }; - let curr = block.index_of(block_start); - if curr > ms.next_to_update + 384 { - ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); - } - let dict_mode = ms.dict_mode(dms.as_ref()); - let mut reps = rep; - let last_ll = compress::compress_block( - &mut ms, - &mut seq_store, - &mut rep, - &block, - strategy, - use_row, - dict_mode, - )?; - - let mut total = last_ll; - for (i, seq) in seq_store.sequences.iter().enumerate() { - let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); - if seq_store.long_length_pos as usize == i { - match seq_store.long_length_type { - LongLengthType::LiteralLength => ll += 0x10000, - LongLengthType::MatchLength => ml += 0x10000, - LongLengthType::None => {} - } - } - update_rep(&mut reps, seq.off_base, ll == 0); - total += (ll + ml) as usize; - out.push(if seq.off_base <= 3 { - (ll, ml, seq.off_base, 0) - } else { - (ll, ml, 0, seq.off_base - 3) - }); - } - out.push((last_ll as u32, 0, 0, 0)); - assert_eq!(total, block_size, "the sequences cover the block"); - assert_eq!( - seq_store.lit_len - + last_ll - + seq_store - .sequences - .iter() - .enumerate() - .map(|(i, s)| s.ml_base as usize - + 3 - + if seq_store.long_length_pos as usize == i - && seq_store.long_length_type == LongLengthType::MatchLength - { - 0x10000 - } else { - 0 - }) - .sum::(), - block_size - ); - assert_eq!( - reps[..2], - rep[..2], - "the repeat offsets the block compressor leaves are those of its sequences" - ); - pos += block_size; - in_buff_pos += block_size; - if in_buff_pos + block_size_max > in_buff.len() { - in_buff_pos = 0; - } - } - let state = State { - has_dms: dms.is_some(), - next_to_update: ms.next_to_update, - loaded_dict_end: ms.loaded_dict_end, - low_limit: ms.window.low_limit, - dict_limit: ms.window.dict_limit, - hash_salt_entropy: ms.hash_salt_entropy, - lazy_skipping: ms.lazy_skipping, - hash_cache: ms.hash_cache, - rep, - hash_table: ms.hash_table, - chain_table: ms.chain_table, - tag_table: ms.tag_table, - }; - Ok((out, state)) -} diff --git a/src/zstd/wip/work/ldm/harness/Cargo.toml b/src/zstd/wip/work/ldm/harness/Cargo.toml deleted file mode 100644 index 2c22a568dc5f..000000000000 --- a/src/zstd/wip/work/ldm/harness/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "zstd_harness" -version = "0.0.0" -edition = "2024" - -[dependencies] -bun_zstd = { path = "/root/zstd-ref/work/ldm/zstd" } - -[profile.release] -debug = "line-tables-only" -overflow-checks = true -debug-assertions = true -panic = "abort" - -[profile.bench] -inherits = "release" -overflow-checks = false -debug-assertions = false - -[workspace] diff --git a/src/zstd/wip/work/ldm/harness/build.rs b/src/zstd/wip/work/ldm/harness/build.rs deleted file mode 100644 index 85ae4db83966..000000000000 --- a/src/zstd/wip/work/ldm/harness/build.rs +++ /dev/null @@ -1,19 +0,0 @@ -use std::process::Command; - -fn main() { - let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); - println!("cargo:rerun-if-env-changed=ZSTD_REF"); - println!("cargo:rerun-if-changed=shim.c"); - let out = std::env::var("OUT_DIR").unwrap(); - let ok = Command::new("clang-23") - .args(["-c", "-O2", "-g", "-fPIC", "-I/root/zstd-ref/zstd/lib", "-I/root/zstd-ref/zstd/lib/common", "-DZSTD_MULTITHREAD", - "-DZSTD_LEGACY_SUPPORT=0", "-DXXH_NAMESPACE=ZSTD_", "shim.c", "-o"]) - .arg(format!("{out}/shim.o")).status().unwrap().success(); - assert!(ok); - let _ = std::fs::remove_file(format!("{out}/libshim.a")); - assert!(Command::new("llvm-ar-23").args(["rcs", &format!("{out}/libshim.a"), &format!("{out}/shim.o")]).status().unwrap().success()); - println!("cargo:rustc-link-search=native={out}"); - println!("cargo:rustc-link-lib=static=shim"); - println!("cargo:rustc-link-search=native=/root/zstd-ref/build/{variant}"); - println!("cargo:rustc-link-lib=static=zstd"); -} diff --git a/src/zstd/wip/work/ldm/harness/rust-toolchain.toml b/src/zstd/wip/work/ldm/harness/rust-toolchain.toml deleted file mode 100644 index c71778e8ccd3..000000000000 --- a/src/zstd/wip/work/ldm/harness/rust-toolchain.toml +++ /dev/null @@ -1,35 +0,0 @@ -[toolchain] -channel = "nightly-2026-09-15" -# rust-src is needed for -Zbuild-std (Tier 3 targets like -# aarch64-unknown-freebsd have no prebuilt std). miri is for -# `bun run rust:miri`. targets ensures the -# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain -# (CI's images install this same channel with these components and -# targets, from scripts/build/ci-images/spec.ts; a source lint keeps -# the two files in step). Note: rustup ignores this whole file when -# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree -# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` -# self-heals that path. Keep this list as a fallback for direct -# `cargo build`/`cargo check`/rust-analyzer use. -# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd -# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). -# darwin/windows triples build natively on their own agents, so the host std -# comes with the toolchain install regardless; they're listed here so a dev -# running `bun run rust:check-all` or `cargo check --target` from any host -# still gets prebuilt std for every Tier 1/2 triple. -# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at -# what it emits. The build does not use them. -components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] -targets = [ - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "x86_64-unknown-linux-musl", - "aarch64-linux-android", - "x86_64-linux-android", - "x86_64-unknown-freebsd", - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-pc-windows-msvc", - "x86_64-pc-windows-msvc", -] diff --git a/src/zstd/wip/work/ldm/harness/shim.c b/src/zstd/wip/work/ldm/harness/shim.c deleted file mode 100644 index 97def6081ebb..000000000000 --- a/src/zstd/wip/work/ldm/harness/shim.c +++ /dev/null @@ -1,180 +0,0 @@ -/* What the tests need of the internals of the C library. */ -#include -#include -#include "compress/zstd_ldm.h" - -typedef struct { - ldmState_t st; - ldmParams_t p; - BYTE* bufs[3]; - RawSeqStore_t store; -} shim_ldm; - -shim_ldm* shim_ldm_new(const U32* p, const size_t* sizes, size_t capacity, U32 startIndex) -{ - shim_ldm* s = (shim_ldm*)calloc(1, sizeof(*s)); - int i; - s->p.enableLdm = (ZSTD_ParamSwitch_e)p[0]; - s->p.hashLog = p[1]; s->p.bucketSizeLog = p[2]; s->p.minMatchLength = p[3]; s->p.hashRateLog = p[4]; s->p.windowLog = p[5]; - ZSTD_window_init(&s->st.window); - s->st.hashTable = (ldmEntry_t*)calloc((size_t)1 << p[1], sizeof(ldmEntry_t)); - s->st.bucketOffsets = (BYTE*)calloc((size_t)1 << (p[1] - p[2]), 1); - if (startIndex) { - s->st.window.nextSrc = s->st.window.base + startIndex; - ZSTD_window_clear(&s->st.window); - } - for (i = 0; i < 3; i++) s->bufs[i] = (BYTE*)calloc(sizes[i] + 64, 1); - s->store.seq = (rawSeq*)calloc(capacity + 1, sizeof(rawSeq)); - s->store.capacity = capacity; - return s; -} - -void shim_ldm_free(shim_ldm* s) -{ - int i; - for (i = 0; i < 3; i++) free(s->bufs[i]); - free(s->st.hashTable); free(s->st.bucketOffsets); free(s->store.seq); free(s); -} - -void shim_ldm_update(shim_ldm* s, int buf, size_t pos, const void* data, size_t len) -{ - memcpy(s->bufs[buf] + pos, data, len); - ZSTD_window_update(&s->st.window, s->bufs[buf] + pos, len, 0); -} - -void shim_ldm_poke(shim_ldm* s, int buf, size_t pos, BYTE byte) { s->bufs[buf][pos] = byte; } - -void shim_ldm_load_dict(shim_ldm* s, int buf, size_t pos, const void* data, size_t len) -{ - const BYTE* ip = s->bufs[buf] + pos; - shim_ldm_update(s, buf, pos, data, len); - s->st.loadedDictEnd = (U32)(ip + len - s->st.window.base); - ZSTD_ldm_fillHashTable(&s->st, ip, ip + len, &s->p); -} - -size_t shim_ldm_generate(shim_ldm* s, int buf, size_t pos, size_t len, int reset) -{ - if (reset) { s->store.size = 0; s->store.pos = 0; } - return ZSTD_ldm_generateSequences(&s->st, &s->store, &s->p, s->bufs[buf] + pos, len); -} - -const rawSeq* shim_ldm_seqs(const shim_ldm* s, size_t* nb) { *nb = s->store.size; return s->store.seq; } - -void shim_ldm_window(const shim_ldm* s, U32* out) -{ - out[0] = s->st.window.dictLimit; out[1] = s->st.window.lowLimit; out[2] = s->st.loadedDictEnd; - out[3] = (U32)(s->st.window.nextSrc - s->st.window.base); out[4] = s->st.window.nbOverflowCorrections; -} - -const ldmEntry_t* shim_ldm_table(const shim_ldm* s) { return s->st.hashTable; } -const BYTE* shim_ldm_bucket_offsets(const shim_ldm* s) { return s->st.bucketOffsets; } - -/* ── ZSTD_ldm_blockCompress, on the match state of a real context ── */ - -typedef struct { - ZSTD_CCtx* cctx; - BYTE* dict; - BYTE* src; - RawSeqStore_t store; - U32 rep[ZSTD_REP_NUM]; -} shim_blk; - -/* cp: windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy. Rewritten with what is applied. */ -shim_blk* shim_blk_new(U32* cp, int useRow, const void* dict, size_t dictSize, const void* src, size_t srcSize, - const rawSeq* seqs, size_t nbSeqs) -{ - shim_blk* s = (shim_blk*)calloc(1, sizeof(*s)); - ZSTD_CCtx* c = ZSTD_createCCtx(); - char tmp[64]; - s->cctx = c; - s->dict = (BYTE*)malloc(dictSize + 64); memcpy(s->dict, dict, dictSize); - s->src = (BYTE*)malloc(srcSize + 64); memcpy(s->src, src, srcSize); - s->store.seq = (rawSeq*)malloc((nbSeqs + 1) * sizeof(rawSeq)); memcpy(s->store.seq, seqs, nbSeqs * sizeof(rawSeq)); - s->store.size = s->store.capacity = nbSeqs; - ZSTD_CCtx_setParameter(c, ZSTD_c_windowLog, (int)cp[0]); - ZSTD_CCtx_setParameter(c, ZSTD_c_chainLog, (int)cp[1]); - ZSTD_CCtx_setParameter(c, ZSTD_c_hashLog, (int)cp[2]); - ZSTD_CCtx_setParameter(c, ZSTD_c_searchLog, (int)cp[3]); - ZSTD_CCtx_setParameter(c, ZSTD_c_minMatch, (int)cp[4]); - ZSTD_CCtx_setParameter(c, ZSTD_c_targetLength, (int)cp[5]); - ZSTD_CCtx_setParameter(c, ZSTD_c_strategy, (int)cp[6]); - ZSTD_CCtx_setParameter(c, ZSTD_c_useRowMatchFinder, useRow ? ZSTD_ps_enable : ZSTD_ps_disable); - ZSTD_CCtx_setPledgedSrcSize(c, srcSize); - if (dictSize) ZSTD_CCtx_refPrefix(c, s->dict, dictSize); - { ZSTD_inBuffer in = { s->src, 0, 0 }; - ZSTD_outBuffer out = { tmp, sizeof(tmp), 0 }; - if (ZSTD_isError(ZSTD_compressStream2(c, &out, &in, ZSTD_e_continue))) abort(); - } - { ZSTD_compressionParameters const a = c->appliedParams.cParams; - cp[0] = a.windowLog; cp[1] = a.chainLog; cp[2] = a.hashLog; cp[3] = a.searchLog; cp[4] = a.minMatch; - cp[5] = a.targetLength; cp[6] = (U32)a.strategy; - } - memcpy(s->rep, c->blockState.prevCBlock->rep, sizeof(s->rep)); - { ZSTD_MatchState_t* const ms = &c->blockState.matchState; - if (!ZSTD_window_update(&ms->window, s->src, srcSize, ms->forceNonContiguous)) { - ms->forceNonContiguous = 0; - ms->nextToUpdate = ms->window.dictLimit; - } - } - return s; -} - -void shim_blk_free(shim_blk* s) -{ - ZSTD_freeCCtx(s->cctx); free(s->dict); free(s->src); free(s->store.seq); free(s); -} - -size_t shim_blk_max_block(const shim_blk* s) { return s->cctx->blockSizeMax; } - -/* What ZSTD_compress_frameChunk and ZSTD_buildSeqStore do for a block, with the sequences as external ones. */ -size_t shim_blk_run(shim_blk* s, size_t pos, size_t blockSize) -{ - ZSTD_CCtx* const zc = s->cctx; - ZSTD_MatchState_t* const ms = &zc->blockState.matchState; - const BYTE* const ip = s->src + pos; - U32 const maxDist = (U32)1 << zc->appliedParams.cParams.windowLog; - size_t lastLL; - ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); - ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); - if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; - - ZSTD_resetSeqStore(&zc->seqStore); - if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { - if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { - ZSTD_ldm_skipRawSeqStoreBytes(&s->store, blockSize); - } else { - ZSTD_ldm_skipSequences(&s->store, blockSize, zc->appliedParams.cParams.minMatch); - } - return 0; - } - ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; - ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; - { U32 const curr = (U32)(ip - ms->window.base); - if (curr > ms->nextToUpdate + 384) - ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); - } - lastLL = ZSTD_ldm_blockCompress(&s->store, ms, &zc->seqStore, s->rep, zc->appliedParams.useRowMatchFinder, ip, blockSize); - if (ZSTD_isError(lastLL)) return lastLL; - memcpy(zc->seqStore.lit, ip + blockSize - lastLL, lastLL); - zc->seqStore.lit += lastLL; - return lastLL; -} - -const SeqDef* shim_blk_seqs(const shim_blk* s, size_t* nb) -{ - *nb = (size_t)(s->cctx->seqStore.sequences - s->cctx->seqStore.sequencesStart); - return s->cctx->seqStore.sequencesStart; -} -const BYTE* shim_blk_literals(const shim_blk* s, size_t* nb) -{ - *nb = (size_t)(s->cctx->seqStore.lit - s->cctx->seqStore.litStart); - return s->cctx->seqStore.litStart; -} -/* longLengthType, longLengthPos, rep[3], nextToUpdate */ -void shim_blk_state(const shim_blk* s, U32* out) -{ - out[0] = (U32)s->cctx->seqStore.longLengthType; out[1] = s->cctx->seqStore.longLengthPos; - out[2] = s->rep[0]; out[3] = s->rep[1]; out[4] = s->rep[2]; - out[5] = s->cctx->blockState.matchState.nextToUpdate; -} -const RawSeqStore_t* shim_blk_store(const shim_blk* s) { return &s->store; } diff --git a/src/zstd/wip/work/ldm/harness/src/c.rs b/src/zstd/wip/work/ldm/harness/src/c.rs deleted file mode 100644 index d5a82e1f6b78..000000000000 --- a/src/zstd/wip/work/ldm/harness/src/c.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! The reference C library. -#![allow(non_camel_case_types, dead_code)] -use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; - -#[repr(C)] -pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } -#[repr(C)] -pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } -pub enum ZSTD_CCtx {} -pub enum ZSTD_DCtx {} -pub enum ZSTD_DDict {} - -unsafe extern "C" { - pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; - pub fn ZSTD_compressBound(size: usize) -> usize; - pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_isError(code: usize) -> c_uint; - pub fn ZSTD_getErrorCode(code: usize) -> c_uint; - pub fn ZSTD_getErrorName(code: usize) -> *const c_char; - pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; - pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; - pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; - pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; - pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; - pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; - pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; - pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; - pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; - pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; - pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; - pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; - pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; - pub fn ZSTD_defaultCLevel() -> c_int; - pub fn ZSTD_minCLevel() -> c_int; - pub fn ZSTD_maxCLevel() -> c_int; -} - -/// `Ok(value)` or `Err(error code)`. -pub fn res(code: usize) -> Result { - unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } -} - -pub struct CCtx(pub *mut ZSTD_CCtx); -impl CCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } - pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - /// Returns (result, out pos, in pos). - pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); - (r, o.pos, i.pos) - } -} -impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } - -pub struct DCtx(pub *mut ZSTD_DCtx); -impl DCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } - pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } - pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); - (r, o.pos, i.pos) - } -} -impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } - -pub fn compress(src: &[u8], level: i32) -> Vec { - let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; - let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); - dst.truncate(n); - dst -} -pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) -} diff --git a/src/zstd/wip/work/ldm/harness/src/data.rs b/src/zstd/wip/work/ldm/harness/src/data.rs deleted file mode 100644 index c48a645e1b5d..000000000000 --- a/src/zstd/wip/work/ldm/harness/src/data.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Deterministic inputs. -pub struct Rng(pub u64); -impl Rng { - pub fn next(&mut self) -> u64 { - // splitmix64 - self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); - z ^ (z >> 31) - } - pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } - pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } - pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } - pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } - /// Skewed towards small values. - pub fn size(&mut self, max: usize) -> usize { - let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); - (self.next() as usize & ((1usize << bits) - 1)).min(max) - } -} - -static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; - -/// One buffer of about `len` bytes, of a kind picked by `rng`. -pub fn data(rng: &mut Rng, len: usize) -> Vec { - let mut v = Vec::with_capacity(len + 64); - match rng.below(9) { - 0 => v.resize(len, rng.next() as u8), - 1 => while v.len() < len { v.push(rng.next() as u8) }, - 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, - 3 => { // few symbols, skewed - let n = rng.range(2, 40); - while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } - } - 4 => { // LZ-like: copies of earlier parts, at all distances - while v.len() < len { - if v.len() > 8 && rng.chance(3, 4) { - let dist = 1 + rng.size(v.len() - 1); - let n = 3 + rng.size(300); - for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } - } else { - for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } - } - } - } - 5 => { // periodic with noise - let period = rng.range(1, 300); - let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); - while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } - } - 6 => { // sections of different kinds - while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } - } - 7 => { // numbers as text - let mut x = rng.next() % 1000; - while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } - } - _ => { // long runs - while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } - } - } - v.truncate(len); - v -} diff --git a/src/zstd/wip/work/ldm/harness/src/ldm.rs b/src/zstd/wip/work/ldm/harness/src/ldm.rs deleted file mode 100644 index 03581042d108..000000000000 --- a/src/zstd/wip/work/ldm/harness/src/ldm.rs +++ /dev/null @@ -1,452 +0,0 @@ -//! Long distance matching and the block pre-splitter, against the C library. -use crate::c; -use crate::data::{Rng, data}; -use bun_zstd::testing as t; -use std::ffi::{c_int, c_void}; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct LdmParams { enable: u32, hash_log: u32, bucket_size_log: u32, min_match_length: u32, hash_rate_log: u32, window_log: u32 } -#[repr(C)] -#[derive(Clone, Copy, Debug)] -struct CParams { window_log: u32, chain_log: u32, hash_log: u32, search_log: u32, min_match: u32, target_length: u32, strategy: u32 } -#[repr(C)] -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct RawSeq { offset: u32, lit_length: u32, match_length: u32 } -#[repr(C)] -struct RawSeqStore { seq: *mut RawSeq, pos: usize, pos_in_sequence: usize, size: usize, capacity: usize } -#[repr(C)] -#[derive(Clone, Copy)] -struct LdmEntry { offset: u32, checksum: u32 } -enum Shim {} - -unsafe extern "C" { - fn ZSTD_splitBlock(block: *const c_void, size: usize, level: c_int, wksp: *mut c_void, wksp_size: usize) -> usize; - fn ZSTD_ldm_adjustParameters(p: *mut LdmParams, cp: *const CParams); - fn ZSTD_ldm_getTableSize(p: LdmParams) -> usize; - fn ZSTD_ldm_getMaxNbSeq(p: LdmParams, max_chunk_size: usize) -> usize; - fn ZSTD_ldm_skipSequences(s: *mut RawSeqStore, src_size: usize, min_match: u32); - fn ZSTD_ldm_skipRawSeqStoreBytes(s: *mut RawSeqStore, nb_bytes: usize); - fn shim_ldm_new(p: *const u32, sizes: *const usize, capacity: usize, start_index: u32) -> *mut Shim; - fn shim_ldm_free(s: *mut Shim); - fn shim_ldm_update(s: *mut Shim, buf: c_int, pos: usize, data: *const c_void, len: usize); - fn shim_ldm_poke(s: *mut Shim, buf: c_int, pos: usize, byte: u8); - fn shim_ldm_load_dict(s: *mut Shim, buf: c_int, pos: usize, data: *const c_void, len: usize); - fn shim_ldm_generate(s: *mut Shim, buf: c_int, pos: usize, len: usize, reset: c_int) -> usize; - fn shim_ldm_seqs(s: *const Shim, nb: *mut usize) -> *const RawSeq; - fn shim_ldm_window(s: *const Shim, out: *mut u32); - fn shim_ldm_table(s: *const Shim) -> *const LdmEntry; - fn shim_ldm_bucket_offsets(s: *const Shim) -> *const u8; -} - -// ── the pre-splitter ────────────────────────────────────────────────────── - -pub fn presplit(seed: u64, iters: u64) { - const SIZE: usize = 128 << 10; - let mut results = std::collections::BTreeMap::<(i32, usize), u64>::new(); - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - let mut block = Vec::with_capacity(SIZE + 64); - match rng.below(4) { - 0 => block = data(&mut rng, SIZE), - 1 => { // sections, cut anywhere - while block.len() < SIZE { let n = 1 + rng.below(SIZE); block.extend(data(&mut rng, n)); } - } - 2 => { // sections, cut at chunk boundaries - while block.len() < SIZE { let n = (8 << 10) * rng.range(1, 16); let mut d = data(&mut rng, n); d.resize(n, 0); block.extend(d); } - } - _ => { // a slow drift of the distribution - let (a, b) = (rng.range(1, 255), rng.range(1, 255)); - let base = rng.next() as u8; - for k in 0..SIZE { let n = (a * (SIZE - k) + b * k) / SIZE; block.push(base.wrapping_add(rng.below(n.max(1)) as u8)); } - } - } - block.resize(SIZE, 0); - for level in 0..=4 { - let mut wksp = vec![0xA5A5_A5A5_A5A5_A5A5u64; 8208 / 8]; - let expected = unsafe { ZSTD_splitBlock(block.as_ptr().cast(), SIZE, level, wksp.as_mut_ptr().cast(), 8208) }; - let got = t::split_block(&block, level).unwrap_or_else(|e| panic!("seed {s} level {level}: {e:?}")); - assert_eq!(got, expected, "seed {s} level {level}"); - *results.entry((level, got >> 10)).or_default() += 1; - } - } - println!("presplit ok: {iters} blocks from seed {seed}, at 5 levels. (level, KB) -> count: {results:?}"); -} - -// ── parameters ──────────────────────────────────────────────────────────── - -fn arr(p: LdmParams) -> [u32; 6] { [p.enable, p.hash_log, p.bucket_size_log, p.min_match_length, p.hash_rate_log, p.window_log] } - -pub fn params(seed: u64, iters: u64) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - let zero = |rng: &mut Rng, lo: usize, hi: usize| if rng.chance(1, 3) { 0 } else { rng.range(lo, hi) as u32 }; - let p = LdmParams { - enable: rng.below(3) as u32, hash_log: zero(&mut rng, 6, 30), bucket_size_log: zero(&mut rng, 1, 8), - min_match_length: zero(&mut rng, 4, 4096), hash_rate_log: zero(&mut rng, 0, 25), window_log: rng.below(32) as u32, - }; - let cp = CParams { - window_log: rng.range(10, 31) as u32, chain_log: rng.range(6, 30) as u32, hash_log: rng.range(6, 30) as u32, search_log: rng.range(1, 30) as u32, - min_match: rng.range(3, 7) as u32, target_length: rng.below(131073) as u32, strategy: rng.range(1, 9) as u32, - }; - let mut expected = p; - unsafe { ZSTD_ldm_adjustParameters(&mut expected, &cp) }; - let got = t::ldm_adjust_parameters(arr(p), [cp.window_log, cp.chain_log, cp.hash_log, cp.search_log, cp.min_match, cp.target_length, cp.strategy]); - assert_eq!(got, arr(expected), "seed {s}: {p:?} {cp:?}"); - for q in [expected, LdmParams { min_match_length: p.min_match_length.max(1), ..p }] { - assert_eq!(t::ldm_get_table_size(arr(q)), unsafe { ZSTD_ldm_getTableSize(q) }, "seed {s}: {q:?}"); - let n = rng.next() as usize >> rng.range(1, 63); - assert_eq!(t::ldm_get_max_nb_seq(arr(q), n), unsafe { ZSTD_ldm_getMaxNbSeq(q, n) }, "seed {s}: {q:?} {n}"); - } - } - println!("ldm params ok: {iters} iterations from seed {seed}"); -} - -// ── moving in a store of sequences ──────────────────────────────────────── - -pub fn skip(seed: u64, iters: u64) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - let capacity = rng.below(12); - let size = rng.below(capacity + 1); - let max = *rng.pick(&[3usize, 20, 300, 70_000, 5_000_000]); - let seq: Vec = (0..capacity).map(|_| (1 + rng.size(1 << 27) as u32, rng.size(max) as u32, rng.size(max) as u32)).collect(); - let mut store = t::RawStore { seq, pos: rng.below(size + 1), pos_in_sequence: 0, size }; - for step in 0..rng.range(1, 12) { - let mut c_seq: Vec = store.seq.iter().map(|q| RawSeq { offset: q.0, lit_length: q.1, match_length: q.2 }).collect(); - let mut c_store = RawSeqStore { seq: c_seq.as_mut_ptr(), pos: store.pos, pos_in_sequence: store.pos_in_sequence, size, capacity }; - let n = rng.size(max * 3); - let before = store.clone(); - let which = rng.chance(1, 2); - let min_match = rng.range(3, 7) as u32; - if which { - unsafe { ZSTD_ldm_skipSequences(&mut c_store, n, min_match) }; - t::ldm_skip_sequences(&mut store, n, min_match).unwrap(); - } else { - unsafe { ZSTD_ldm_skipRawSeqStoreBytes(&mut c_store, n) }; - t::ldm_skip_raw_seq_store_bytes(&mut store, n).unwrap(); - } - let expected = t::RawStore { - pos: c_store.pos, pos_in_sequence: c_store.pos_in_sequence, size: c_store.size, - seq: c_seq.iter().map(|q| (q.offset, q.lit_length, q.match_length)).collect(), - }; - assert_eq!(store, expected, "seed {s} step {step}: skip{} {n} min_match {min_match} from {before:?}", if which { "Sequences" } else { "RawSeqStoreBytes" }); - } - } - println!("ldm skip ok: {iters} iterations from seed {seed}"); -} - -// ── finding the sequences ───────────────────────────────────────────────── - -/// Content with repeats of all lengths at all distances. -pub fn long_repeats(rng: &mut Rng, len: usize) -> Vec { - let mut v: Vec = Vec::with_capacity(len + 64); - let max_copy = *rng.pick(&[200usize, 5000, 100_000, 2_000_000]); - let max_fresh = *rng.pick(&[100usize, 5000, 100_000, 1_000_000]); - let noise = rng.chance(1, 2); - let copies = rng.range(1, 7); - while v.len() < len { - if v.len() > 64 && rng.chance(copies, 8) { - let n = (1 + rng.size(max_copy)).min(v.len()); - let from = if rng.chance(1, 2) { rng.below(v.len() - n + 1) } else { v.len() - n - rng.size(v.len() - n) }; - v.extend_from_within(from..from + n); - if rng.chance(1, 3) { - for _ in 0..rng.range(1, 4) { let at = v.len() - 1 - rng.below(n); v[at] ^= 1 + (rng.next() as u8 & 0x7f); } - } - } else { - let n = 1 + rng.size(max_fresh); - if noise { for _ in 0..n.div_ceil(8) { v.extend_from_slice(&rng.next().to_le_bytes()); } } else { v.extend(data(rng, n)); } - } - } - v.truncate(len); - v -} - -#[derive(Default, Debug)] -pub struct Stats { scenarios: u64, bytes: u64, calls: u64, seqs: u64, calls_ext_dict: u64, seqs_ext_dict: u64, corrections: u64, errors: u64, dicts: u64, full: u64 } - -struct Pair { c: *mut Shim, r: t::Ldm, ctx: String } -impl Drop for Pair { fn drop(&mut self) { unsafe { shim_ldm_free(self.c) } } } -impl Pair { - fn c_window(&self) -> [u32; 5] { let mut w = [0u32; 5]; unsafe { shim_ldm_window(self.c, w.as_mut_ptr()) }; w } - fn check_window(&self, at: &str) { assert_eq!(self.r.window(), self.c_window(), "{}: window after {at}", self.ctx); } - fn update(&mut self, buf: usize, pos: usize, d: &[u8]) { - unsafe { shim_ldm_update(self.c, buf as c_int, pos, d.as_ptr().cast(), d.len()) }; - self.r.update(buf, pos, d); - self.check_window("update"); - } - fn poke(&mut self, buf: usize, pos: usize, byte: u8) { - unsafe { shim_ldm_poke(self.c, buf as c_int, pos, byte) }; - self.r.poke(buf, pos, byte); - } - fn load_dict(&mut self, buf: usize, pos: usize, d: &[u8]) { - unsafe { shim_ldm_load_dict(self.c, buf as c_int, pos, d.as_ptr().cast(), d.len()) }; - self.r.load_dict(buf, pos, d).unwrap_or_else(|e| panic!("{}: load_dict {e:?}", self.ctx)); - self.check_window("load_dict"); - } - /// Whether it went without an error. - fn generate(&mut self, buf: usize, pos: usize, len: usize, reset: bool, stats: &mut Stats) -> bool { - let before = self.c_window(); - let expected = c::res(unsafe { shim_ldm_generate(self.c, buf as c_int, pos, len, reset as c_int) }).map(|_| ()); - let got = self.r.generate(pos, len, reset).map_err(|e| e as u32); - let at = format!("generate(buf {buf}, pos {pos}, len {len}, reset {reset}), window before {before:?}"); - assert_eq!(got, expected, "{}: result of {at}", self.ctx); - let mut nb = 0usize; - let c_seqs = unsafe { let p = shim_ldm_seqs(self.c, &mut nb); std::slice::from_raw_parts(p, nb) }; - let r_seqs = self.r.seqs(); - let same = r_seqs.len() == nb && r_seqs.iter().zip(c_seqs).all(|(a, b)| *a == (b.offset, b.lit_length, b.match_length)); - if !same { - let k = r_seqs.iter().zip(c_seqs).position(|(a, b)| *a != (b.offset, b.lit_length, b.match_length)).unwrap_or(nb.min(r_seqs.len())); - panic!("{}: {at}: sequence {k} differs: got {:?}, expected {:?} ({} vs {nb} sequences)", self.ctx, r_seqs.get(k), c_seqs.get(k), r_seqs.len()); - } - self.check_window(&at); - let after = self.c_window(); - stats.calls += 1; - stats.bytes += len as u64; - if reset { stats.seqs += nb as u64; } - if after[1] < after[0] { stats.calls_ext_dict += 1; if reset { stats.seqs_ext_dict += nb as u64; } } - stats.corrections += u64::from(after[4] - before[4]); - if expected.is_err() { stats.errors += 1; } - expected.is_ok() - } - fn check_tables(&self, p: &LdmParams) { - let table = unsafe { std::slice::from_raw_parts(shim_ldm_table(self.c), 1 << p.hash_log) }; - for (k, (a, b)) in self.r.table().zip(table).enumerate() { - assert_eq!(a, (b.offset, b.checksum), "{}: entry {k} of the hash table", self.ctx); - } - let offsets = unsafe { std::slice::from_raw_parts(shim_ldm_bucket_offsets(self.c), 1 << (p.hash_log - p.bucket_size_log)) }; - assert!(self.r.bucket_offsets() == offsets, "{}: bucket offsets", self.ctx); - } -} - -fn scenario(s: u64, max_len: usize, stats: &mut Stats) { - let mut rng = Rng(s); - let mut p = LdmParams { - enable: 1, - hash_log: if rng.chance(1, 12) { rng.range(17, 24) } else { rng.range(6, 16) } as u32, - bucket_size_log: rng.range(1, 8) as u32, - min_match_length: match rng.below(4) { 0 => rng.range(4, 12), 1 => *rng.pick(&[16usize, 32, 63, 64, 65, 128]), 2 => rng.range(4, 300), _ => rng.range(4, 4096) } as u32, - hash_rate_log: if rng.chance(1, 8) { rng.range(0, 25) } else { rng.range(0, 10) } as u32, - window_log: 0, - }; - let cp = CParams { window_log: rng.range(10, 27) as u32, chain_log: 6, hash_log: 6, search_log: 1, min_match: 4, target_length: 0, strategy: rng.range(1, 9) as u32 }; - if rng.chance(1, 6) { p.hash_log = 0; p.hash_rate_log = 0; } - if rng.chance(1, 6) { p.hash_rate_log = 0; } - if rng.chance(1, 6) { p.bucket_size_log = 0; } - if rng.chance(1, 6) { p.min_match_length = 0; } - unsafe { ZSTD_ldm_adjustParameters(&mut p, &cp) }; - if p.hash_log > 24 { p.hash_log = 24; } - - let mut len = match rng.below(5) { 0 => rng.size(5000), 1 => rng.size(300_000), 2 => rng.size(max_len), _ => rng.range(max_len / 4, max_len) } + 1; - while (len as u64 * u64::from(p.min_match_length)) >> p.hash_rate_log > 100 << 20 { len /= 2; } - let content = long_repeats(&mut rng, len); - - let layout = rng.below(5); - let dict_len = if layout == 1 || layout == 2 || rng.chance(1, 8) { rng.size(len / 2) } else { 0 }; - let ring = if layout == 4 { (rng.size(len) + 1).max(16) } else { 0 }; - let room = len + 64; - let sizes = if layout == 4 { [ring, 0, dict_len] } else { [room, room, room] }; - let start_index = match rng.below(8) { - 0 => (3500u32 << 20).wrapping_sub(rng.below(len + (2 << 20)) as u32).wrapping_add(rng.below(1 << 20) as u32), - 1 => 2 + rng.size(3400 << 20) as u32, - _ => 0, - }; - let reset = rng.chance(3, 4); - let capacity = if rng.chance(1, 10) { rng.below(6) } else { len / p.min_match_length as usize + 1 }; - let max_update = *rng.pick(&[1000usize, 128 << 10, 1 << 20, 5 << 20, usize::MAX >> 1]); - let pokes = if rng.chance(1, 8) { rng.range(1, 200) } else { 0 }; - let piece = *rng.pick(&[0usize, 128 << 10, 128 << 10, 1]); - let ctx = format!("seed {s}: {p:?} len {len} layout {layout} dict {dict_len} ring {ring} start_index {start_index} reset {reset} capacity {capacity} pokes {pokes}"); - let pa = arr(p); - let mut pair = Pair { c: unsafe { shim_ldm_new(pa.as_ptr(), sizes.as_ptr(), capacity, start_index) }, r: t::Ldm::new(pa, sizes, capacity, start_index), ctx }; - stats.scenarios += 1; - - let (mut buf, mut pos) = (if layout == 4 { 0 } else { 1 }, 0usize); - let mut done = 0usize; - if dict_len > 0 { - stats.dicts += 1; - if layout == 2 { pair.load_dict(buf, 0, &content[..dict_len]); pos = dict_len; } else { pair.load_dict(2, 0, &content[..dict_len]); } - done = dict_len; - } - 'all: while done < len { - let mut n = (1 + rng.size(max_update.min(len - done) - 1)).min(len - done); - match layout { - 3 if rng.chance(1, 3) => { buf = rng.below(3); pos = rng.below(room - n + 1); } - 4 => { n = n.min(ring); if pos + n > ring { pos = 0; } } - _ => {} - } - if pos + n > sizes[buf] { pos = 0; } - pair.update(buf, pos, &content[done..done + n]); - let end = pos + n; - for _ in 0..pokes { - let b = rng.below(3); - if sizes[b] > 0 { pair.poke(b, rng.below(sizes[b]), rng.next() as u8); } - } - while pos < end { - let m = match piece { 0 => end - pos, 1 => 1 + rng.size(end - pos - 1), k => k.min(end - pos) }; - if !pair.generate(buf, pos, m, reset, stats) { break 'all; } - pos += m; - } - done += n; - } - pair.check_tables(&p); -} - -pub fn run(seed: u64, iters: u64, max_len: usize) { - let mut stats = Stats::default(); - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - if i % 200 == 0 { eprintln!("ldm iter {i} (seed {s})"); } - scenario(s, max_len, &mut stats); - } - println!("ldm ok: {iters} scenarios from seed {seed}, up to {max_len} bytes: {stats:?}"); -} - -pub fn bench(len: usize) { - let mut rng = Rng(42); - for (name, content) in [("repeats", long_repeats(&mut rng, len)), ("text", { let mut v = Vec::new(); while v.len() < len { v.extend(data(&mut Rng(2 + v.len() as u64 % 3), 300_000)); } v.truncate(len); v })] { - let mut p = LdmParams { enable: 1, hash_log: 0, bucket_size_log: 0, min_match_length: 0, hash_rate_log: 0, window_log: 0 }; - let cp = CParams { window_log: 27, chain_log: 6, hash_log: 6, search_log: 1, min_match: 4, target_length: 0, strategy: 2 }; - unsafe { ZSTD_ldm_adjustParameters(&mut p, &cp) }; - let pa = arr(p); - let sizes = [0, len, 0]; - let capacity = len / 64 + 1; - for _ in 0..3 { - let c = unsafe { shim_ldm_new(pa.as_ptr(), sizes.as_ptr(), capacity, 0) }; - let mut r = t::Ldm::new(pa, sizes, capacity, 0); - unsafe { shim_ldm_update(c, 1, 0, content.as_ptr().cast(), len) }; - r.update(1, 0, &content); - let t0 = std::time::Instant::now(); - unsafe { shim_ldm_generate(c, 1, 0, len, 1) }; - let t1 = std::time::Instant::now(); - r.generate(0, len, true).unwrap(); - let t2 = std::time::Instant::now(); - println!("{name}: {p:?}: C {:.1} MB/s, Rust {:.1} MB/s, {} sequences", len as f64 / 1e6 / (t1 - t0).as_secs_f64(), len as f64 / 1e6 / (t2 - t1).as_secs_f64(), r.seqs().len()); - unsafe { shim_ldm_free(c) }; - } - } - let block = data(&mut Rng(7), 128 << 10); - let block = &block[..128 << 10]; - for level in 0..=4 { - let mut wksp = vec![0u64; 8208 / 8]; - let n = 2000; - let t0 = std::time::Instant::now(); - let mut a = 0; - for _ in 0..n { a += unsafe { ZSTD_splitBlock(std::hint::black_box(block).as_ptr().cast(), block.len(), level, wksp.as_mut_ptr().cast(), 8208) }; } - let t1 = std::time::Instant::now(); - let mut b = 0; - for _ in 0..n { b += t::split_block(std::hint::black_box(block), level).unwrap(); } - let t2 = std::time::Instant::now(); - println!("split level {level}: C {:.2} us, Rust {:.2} us ({a} {b})", (t1 - t0).as_secs_f64() * 1e6 / n as f64, (t2 - t1).as_secs_f64() * 1e6 / n as f64); - } -} - -// ── ZSTD_ldm_blockCompress ──────────────────────────────────────────────── - -#[repr(C)] -#[derive(Clone, Copy)] -struct SeqDef { off_base: u32, lit_length: u16, ml_base: u16 } -enum ShimBlk {} -unsafe extern "C" { - fn shim_blk_new(cp: *mut u32, use_row: c_int, dict: *const c_void, dict_size: usize, src: *const c_void, src_size: usize, seqs: *const RawSeq, nb: usize) -> *mut ShimBlk; - fn shim_blk_free(s: *mut ShimBlk); - fn shim_blk_max_block(s: *const ShimBlk) -> usize; - fn shim_blk_run(s: *mut ShimBlk, pos: usize, block_size: usize) -> usize; - fn shim_blk_seqs(s: *const ShimBlk, nb: *mut usize) -> *const SeqDef; - fn shim_blk_literals(s: *const ShimBlk, nb: *mut usize) -> *const u8; - fn shim_blk_state(s: *const ShimBlk, out: *mut u32); - fn shim_blk_store(s: *const ShimBlk) -> *const RawSeqStore; -} - -unsafe fn slice_of<'a, T>(p: *const T, n: usize) -> &'a [T] { if n == 0 { &[] } else { unsafe { std::slice::from_raw_parts(p, n) } } } - -pub fn blocks(seed: u64, iters: u64, strategies: &[u32], max_len: usize) { - let (mut nb_blocks, mut nb_seqs, mut nb_raw, mut nb_dict) = (0u64, 0u64, 0u64, 0u64); - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if i % 200 == 0 { eprintln!("ldmblocks iter {i} (seed {s})"); } - let len = match rng.below(3) { 0 => rng.size(20_000), 1 => rng.size(max_len), _ => rng.range(max_len / 4, max_len) } + 1; - let dict_len = if rng.chance(1, 3) { 8 + rng.size(len) } else { 0 }; // less than 8 bytes are not loaded - let content = long_repeats(&mut rng, dict_len + len); - let (dict, src) = content.split_at(dict_len); - let mut cp = [ - rng.range(10, 24) as u32, rng.range(6, 18) as u32, rng.range(6, 18) as u32, rng.range(1, 7) as u32, rng.range(3, 7) as u32, - *rng.pick(&[0u32, 1, 2, 8, 32, 64, 256, 999]), *rng.pick(strategies), - ]; - let use_row = rng.chance(1, 2); - - // The long distance matches of the whole input, or made-up sequences. - let raw: Vec = if rng.chance(2, 3) { - let mut p = LdmParams { - enable: 1, hash_log: rng.range(6, 16) as u32, bucket_size_log: rng.range(1, 6) as u32, - min_match_length: *rng.pick(&[4u32, 8, 16, 32, 64, 200]), hash_rate_log: rng.range(0, 7) as u32, window_log: 0, - }; - let c = CParams { window_log: cp[0], chain_log: 6, hash_log: 6, search_log: 1, min_match: 4, target_length: 0, strategy: cp[6] }; - unsafe { ZSTD_ldm_adjustParameters(&mut p, &c) }; - let mut l = t::Ldm::new(arr(p), [0, len, dict_len], len / p.min_match_length as usize + 1, 0); - if dict_len > 0 { l.load_dict(2, 0, dict).unwrap(); } - l.update(1, 0, src); - l.generate(0, len, true).unwrap(); - l.seqs() - } else { - let (max_ll, max_ml) = (*rng.pick(&[0usize, 10, 1000, 70_000, 300_000]), *rng.pick(&[10usize, 1000, 70_000, 300_000])); - let mut v = Vec::new(); - let mut at = 0usize; - let stop = rng.below(len + len / 4 + 1); - while at < stop { - let (ll, ml) = (rng.size(max_ll), 4 + rng.size(max_ml)); - at += ll + ml; - let beyond = rng.below(2); - v.push((1 + rng.size((at - ml + dict_len).max(1).min(1 << cp[0]) - 1 + beyond) as u32, ll as u32, ml as u32)); - } - v - }; - let c_raw: Vec = raw.iter().map(|q| RawSeq { offset: q.0, lit_length: q.1, match_length: q.2 }).collect(); - - let c = unsafe { shim_blk_new(cp.as_mut_ptr(), use_row as c_int, dict.as_ptr().cast(), dict.len(), src.as_ptr().cast(), src.len(), c_raw.as_ptr(), c_raw.len()) }; - let ctx = format!("seed {s}: cp {cp:?} row {use_row} dict {dict_len} src {len} raw sequences {}", raw.len()); - let mut r = t::LdmBlocks::new(cp, use_row, dict, src, &raw).unwrap_or_else(|e| panic!("{ctx}: {e:?}")); - let max_block = unsafe { shim_blk_max_block(c) }; - let full_blocks = rng.chance(1, 2); - let mut pos = 0usize; - nb_raw += raw.len() as u64; - nb_dict += u64::from(dict_len > 0); - while pos < len { - let n = if full_blocks { max_block } else { 1 + rng.size(max_block - 1) }.min(len - pos); - if std::env::var_os("LDM_TRACE").is_some() { eprintln!("{ctx}: block at {pos} of {n} (max {max_block})"); } - let last_ll = c::res(unsafe { shim_blk_run(c, pos, n) }).unwrap_or_else(|e| panic!("{ctx}: C error {e}")); - let got = r.run(pos, n).unwrap_or_else(|e| panic!("{ctx}: block at {pos} of {n}: {e:?}")); - let expected = unsafe { - let mut nb = 0usize; - let seqs = slice_of(shim_blk_seqs(c, &mut nb), nb).iter().map(|q| (q.off_base, q.lit_length, q.ml_base)).collect(); - let literals = slice_of(shim_blk_literals(c, &mut nb), nb).to_vec(); - let mut st = [0u32; 6]; - shim_blk_state(c, st.as_mut_ptr()); - let store = &*shim_blk_store(c); - t::BlockOut { - seqs, literals, last_ll, long_length: if st[0] == 0 { None } else { Some((st[0], st[1])) }, rep: [st[2], st[3], st[4]], next_to_update: st[5], - store: t::RawStore { - seq: slice_of(store.seq, store.capacity).iter().map(|q| (q.offset, q.lit_length, q.match_length)).collect(), - pos: store.pos, pos_in_sequence: store.pos_in_sequence, size: store.size, - }, - } - }; - if got != expected { - let k = got.seqs.iter().zip(&expected.seqs).position(|(a, b)| a != b).unwrap_or(got.seqs.len().min(expected.seqs.len())); - panic!("{ctx}: block at {pos} of {n}: sequence {k}: got {:?}, expected {:?} ({} vs {}); last_ll {} vs {}; rep {:?} vs {:?}; long {:?} vs {:?}; next_to_update {} vs {}; literals same {}; store same {} (pos {} vs {}, in sequence {} vs {})", - got.seqs.get(k), expected.seqs.get(k), got.seqs.len(), expected.seqs.len(), got.last_ll, expected.last_ll, got.rep, expected.rep, - got.long_length, expected.long_length, got.next_to_update, expected.next_to_update, got.literals == expected.literals, got.store == expected.store, - got.store.pos, expected.store.pos, got.store.pos_in_sequence, expected.store.pos_in_sequence); - } - nb_blocks += 1; - nb_seqs += got.seqs.len() as u64; - pos += n; - } - unsafe { shim_blk_free(c) }; - } - println!("ldm blocks ok: {iters} inputs from seed {seed}, strategies {strategies:?}, up to {max_len} bytes: {nb_blocks} blocks, {nb_seqs} sequences, {nb_raw} raw sequences, {nb_dict} with a prefix"); -} diff --git a/src/zstd/wip/work/ldm/harness/src/main.rs b/src/zstd/wip/work/ldm/harness/src/main.rs deleted file mode 100644 index 8eaf738235fb..000000000000 --- a/src/zstd/wip/work/ldm/harness/src/main.rs +++ /dev/null @@ -1,30 +0,0 @@ -mod c; -mod data; -mod ldm; -mod seqs; - -fn main() { - let args: Vec = std::env::args().collect(); - let cmd = args.get(1).map(String::as_str).unwrap_or(""); - let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); - let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); - match cmd { - // seqs - "seqs" => { - let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); - seqs::run(seed, iters, &strategies, args.get(5).is_some_and(|s| s == "1")); - } - "presplit" => ldm::presplit(seed, iters), - "ldmparams" => ldm::params(seed, iters), - "ldmskip" => ldm::skip(seed, iters), - // ldm - "ldm" => ldm::run(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(4 << 20)), - // ldmblocks - "ldmblocks" => { - let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); - ldm::blocks(seed, iters, &strategies, args.get(5).and_then(|s| s.parse().ok()).unwrap_or(1 << 20)); - } - "ldmbench" => ldm::bench(seed as usize), - _ => eprintln!("unknown command"), - } -} diff --git a/src/zstd/wip/work/ldm/harness/src/seqs.rs b/src/zstd/wip/work/ldm/harness/src/seqs.rs deleted file mode 100644 index 7c03d3b0f952..000000000000 --- a/src/zstd/wip/work/ldm/harness/src/seqs.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! One block through a Rust block compressor and through the C library (`ZSTD_generateSequences`). -use crate::c; -use crate::data::{Rng, data}; -use std::ffi::{c_uint, c_void}; - -#[repr(C)] -#[derive(Clone, Copy, Default, Debug)] -struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } -#[repr(C)] -#[derive(Clone, Copy, Debug)] -struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } -unsafe extern "C" { - fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; - fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; - fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; -} - -fn c_sequences(cp: [u32; 7], use_row: bool, dict: &[u8], src: &[u8]) -> Vec<(u32, u32, u32)> { - let mut cctx = c::CCtx::new(); - for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } - cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder - cctx.set(1010, 2).unwrap(); // no block splitter - if !dict.is_empty() { cctx.ref_prefix(dict).unwrap(); } - let mut out = vec![Sequence::default(); src.len() / 3 + 8]; - let n = c::res(unsafe { ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()) }).unwrap(); - out[..n].iter().map(|s| (s.lit_length, s.match_length, s.offset)).collect() -} - -/// `strategies`: which of 1..=9 to test. `with_dict`: also test with a prefix, which is the extDict variants. -pub fn run(seed: u64, iters: u64, strategies: &[u32], with_dict: bool) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if i % 500 == 0 { eprintln!("seqs iter {i} (seed {s})"); } - let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071]); - let len = rng.range(7, max); - let mut src = data(&mut rng, len); - let dict = if with_dict && rng.chance(1, 2) { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; - if !dict.is_empty() && rng.chance(2, 3) { - for _ in 0..rng.range(1, 6) { - let n = rng.size(dict.len().min(src.len())); - let at = rng.below(dict.len() - n + 1); - let to = rng.below(src.len() - n + 1); - src[to..to + n].copy_from_slice(&dict[at..at + n]); - } - } - let strategy = *rng.pick(strategies); - let p = if rng.chance(1, 3) { - // the parameters of a level, with the strategy forced - let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; - p.strategy = strategy; - p - } else { - CParams { - window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, - search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, - target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, - } - }; - let p = unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) }; - let cp = [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy]; - let use_row = rng.chance(1, 2); - let expected = c_sequences(cp, use_row, &dict, &src); - let got = bun_zstd::testing::first_block(cp, use_row, &dict, &src); - let got = got.unwrap_or_else(|e| panic!("seed {s}: error {e:?} (cp {cp:?} row {use_row} dict {} src {})", dict.len(), src.len())); - if got != expected { - let at = got.iter().zip(&expected).position(|(a, b)| a != b).unwrap_or(got.len().min(expected.len())); - panic!("seed {s}: cp {cp:?} row {use_row} dict {} src {}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", - dict.len(), src.len(), got.get(at), expected.get(at), got.len(), expected.len()); - } - } - println!("seqs ok: {iters} iterations from seed {seed}, strategies {strategies:?}, dict {with_dict}"); -} diff --git a/src/zstd/wip/work/ldm/zstd/lib.rs b/src/zstd/wip/work/ldm/zstd/lib.rs deleted file mode 100644 index 31c98c72e5e4..000000000000 --- a/src/zstd/wip/work/ldm/zstd/lib.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the -//! same bytes in, in both directions, and the same errors. -#![forbid(unsafe_code)] -// Frames come from anywhere and a panic aborts the process. -#![deny( - clippy::indexing_slicing, - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::unreachable -)] -#![warn(unused_must_use)] - -mod common; -mod compress; -mod decompress; -mod error; -mod util; - -pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; -pub use decompress::{ - DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, - find_frame_compressed_size, get_frame_content_size, -}; -pub use error::ErrorCode; - -/// `ZSTD_inBuffer` -pub struct InBuffer<'a> { - pub src: &'a [u8], - /// How much of `src` is read. - pub pos: usize, -} - -/// `ZSTD_outBuffer` -pub struct OutBuffer<'a> { - pub dst: &'a mut [u8], - /// How much of `dst` is written. - pub pos: usize, -} - -/// `ZSTD_ResetDirective` -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum ResetDirective { - SessionOnly = 1, - Parameters = 2, - SessionAndParameters = 3, -} - -// ----------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] -pub enum ZstdError { - /// The output, or the decoder state the frame's window size dictates, could not be allocated. - OutOfMemory, - InvalidZstdData, - DecompressionFailed, - ZstdDecompressionError, - ShortRead, -} - -impl core::fmt::Display for ZstdError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.write_str(<&'static str>::from(self)) - } -} -impl core::error::Error for ZstdError {} - -impl ZstdError { - /// The error for a failed decompression call; `other` is the non-allocation failure. - fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { - if code == ErrorCode::MemoryAllocation { - ZstdError::OutOfMemory - } else { - other - } - } -} - -/// `len` zeros for a decoder to write to. -fn output_buffer(len: usize) -> Result, ZstdError> { - util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) -} - -/// Decompress data, automatically allocating the output buffer. -/// Handles both frames with known and unknown content sizes. -/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. -/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. -pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { - let size = find_decompressed_size(src); - - const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit - - if size == ZSTD_CONTENTSIZE_ERROR { - return Err(ZstdError::InvalidZstdData); - } - - // Use streaming decompression if: - // 1. Content size is unknown, OR - // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) - if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { - let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { - // A frame's output is rarely smaller than its input. - src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) - } else { - // The header size is untrusted: reserve no more than the fast path below would. - MAX_PREALLOCATE_SIZE - }; - let mut list: Vec = Vec::new(); - list.try_reserve_exact(initial_capacity) - .map_err(|_| ZstdError::OutOfMemory)?; - StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; - return Ok(list); - } - - // Fast path: size is known and within reasonable limits - let mut output = output_buffer(size as usize)?; - let written = decompress(&mut output, src) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is -/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. -pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { - const MAX_SIZE: u64 = 16 * 1024 * 1024; - let size = find_decompressed_size(src); - // Also if it is unknown or invalid, which are the two largest values. - if size > MAX_SIZE { - return Err(ZstdError::InvalidZstdData); - } - let mut output = output_buffer(size as usize)?; - let written = DDict::prefix(prefix) - .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum State { - Uninitialized, - Inflating, - End, - Error, -} - -/// Minimum spare output capacity offered to the decoder per call. -const STREAMING_OUTPUT_STEP: usize = 4096; - -// ────────────────────────────────────────────────────────────────────────── -// StreamingDecoder -// ────────────────────────────────────────────────────────────────────────── - -/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) -/// per call, so callers can hold the decoder across multiple body chunks. -pub struct StreamingDecoder { - stream: DCtx, - state: State, - /// The decoder filled its last window and may hold more. `max_output` can end a call there. - output_full: bool, -} - -impl Default for StreamingDecoder { - fn default() -> Self { - Self::new() - } -} - -impl StreamingDecoder { - pub fn new() -> Self { - Self { - stream: DCtx::new(), - state: State::Uninitialized, - output_full: false, - } - } - - #[inline] - pub fn is_inflating(&self) -> bool { - matches!(self.state, State::Inflating) - } - - /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is - /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. - /// Returns `ShortRead` when more input is required and `is_done` is false. - pub fn decompress( - &mut self, - input: &[u8], - out: &mut Vec, - max_output: usize, - is_done: bool, - ) -> Result { - if matches!(self.state, State::End | State::Error) { - return Ok(input.len()); - } - - let mut in_buf = InBuffer { src: input, pos: 0 }; - while matches!(self.state, State::Uninitialized | State::Inflating) { - let next_in_len = input.len() - in_buf.pos; - - // Call the decoder again with no input until it leaves the window short. - if next_in_len == 0 && !self.output_full { - if is_done { - if self.state == State::Inflating { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - self.state = State::End; - } - return Ok(in_buf.pos); - } - - if out.len() >= max_output { - return Ok(in_buf.pos); - } - - if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { - self.state = State::Error; - return Err(ZstdError::OutOfMemory); - } - // The decoder writes to initialized memory only. - let written_before = out.len(); - let window = (out.capacity() - written_before).min(max_output - written_before); - out.resize(written_before + window, 0); - let mut out_buf = OutBuffer { - dst: out.get_mut(written_before..).unwrap_or_default(), - pos: 0, - }; - let read_before = in_buf.pos; - - let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); - let bytes_written = out_buf.pos; - out.truncate(written_before + bytes_written); - let remaining = match result { - Ok(remaining) => remaining, - Err(code) => { - self.state = State::Error; - return Err(ZstdError::for_decompression( - code, - ZstdError::ZstdDecompressionError, - )); - } - }; - - self.output_full = bytes_written == window; - - if remaining == 0 { - // Frame complete, and fully flushed. - self.state = State::Uninitialized; - self.output_full = false; - if in_buf.pos >= input.len() { - if is_done { - self.state = State::End; - } - return Ok(in_buf.pos); - } - // More input available — reinitialize for the next frame. - self.stream.init_stream(); - continue; - } - - self.state = State::Inflating; - - if in_buf.pos - read_before == next_in_len { - if self.output_full { - continue; - } - if is_done { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - return Err(ZstdError::ShortRead); - } - } - Ok(in_buf.pos) - } -} - -// ── compressed embedded assets ──────────────────────────────────────────── - -/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes -/// Bun never executes or parses itself: the shell completion scripts and the -/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, -/// error overlay/page). Anything that runs inside Bun stays uncompressed. -/// -/// The exception is text that only `bun lint`, `bun fmt` and `bun check` -/// read. It may be compressed because it isn't on the path of running a program. -/// -/// Release (`bun_codegen_embed`) builds include the `.zst` twin that -/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to -/// `/compressed/.zst`; other builds evaluate `$fallback`. -#[macro_export] -macro_rules! embed_compressed { - // A codegen output (`/$sub`); debug builds read it at runtime. - (codegen $sub:literal) => { - $crate::embed_compressed!( - ("codegen/", $sub), - ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() - ) - }; - // Same, with a trailing NUL included in the slice (for C-string consumers). - (codegen_nul $sub:literal) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/codegen/", - $sub, - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); - &__COPY.get_or_init(|| { - let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); - let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); - copy.extend_from_slice(text); - copy.push(0); - copy.into_boxed_slice() - })[..] - } - }; - __bytes - }}; - // A file under `src/`; debug builds read it from the source tree at runtime. - (src $sub:literal) => { - $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) - }; - ($name:literal, $fallback:expr) => { - $crate::embed_compressed!(($name), $fallback) - }; - (($($name:literal),+), $fallback:expr) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/", - $($name,)+ - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - $fallback - } - }; - __bytes - }}; -} - -/// Cold, shared body of [`embed_compressed!`]'s release arm. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { - decompress_alloc(compressed) - .expect("embedded asset: invalid zstd frame") - .into_boxed_slice() -} - -/// [`inflate_embedded`] plus a trailing NUL byte. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { - let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); - inflated.reserve_exact(1); - inflated.push(0); - inflated.into_boxed_slice() -} -pub mod testing; diff --git a/src/zstd/wip/work/ldm/zstd/testing.rs b/src/zstd/wip/work/ldm/zstd/testing.rs deleted file mode 100644 index 6c3ab8e86702..000000000000 --- a/src/zstd/wip/work/ldm/zstd/testing.rs +++ /dev/null @@ -1,656 +0,0 @@ -//! Hooks for the differential tests. Only in scratch copies of the crate. -#![allow(clippy::all, dead_code, unreachable_pub)] - -use crate::ErrorCode; -use crate::compress::internal::*; -use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; - -fn bitmix(mut val: u64, len: u64) -> u64 { - val ^= val.rotate_right(49) ^ val.rotate_right(24); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^= (val >> 35).wrapping_add(len); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^ (val >> 28) -} - -/// The sequences of the first block of a frame, as (literal length, match length, offset), and then the literals -/// that are left as (length, 0, 0). -/// -/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, already adjusted to the sizes. -/// `dict` is a prefix (raw content, as `ZSTD_CCtx_refPrefix`), or empty. `src` is from 7 to 131071 bytes. -pub fn first_block( - cp: [u32; 7], - use_row: bool, - dict: &[u8], - src: &[u8], -) -> Result, ErrorCode> { - let strategy = Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?; - let c_params = CompressionParameters { - window_log: cp[0], - chain_log: cp[1], - hash_log: cp[2], - search_log: cp[3], - min_match: cp[4], - target_length: cp[5], - strategy: Some(strategy), - }; - let row_used = row_match_finder_supported(strategy) && use_row; - - // ZSTD_reset_matchState - let mut ms = MatchState::new(); - ms.hash_log3 = if c_params.min_match == 3 { - c_params.window_log.min(17) - } else { - 0 - }; - ms.window.clear(); - ms.next_to_update = ms.window.dict_limit; - ms.hash_table = vec![0; 1 << c_params.hash_log]; - ms.chain_table = vec![ - 0; - if strategy != Strategy::Fast && !row_used { - 1 << c_params.chain_log - } else { - 0 - } - ]; - ms.hash_table3 = vec![ - 0; - if ms.hash_log3 != 0 { - 1 << ms.hash_log3 - } else { - 0 - } - ]; - if row_used { - ms.tag_table = vec![0; 1 << c_params.hash_log]; - ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4); - ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); - } - if strategy >= Strategy::BtOpt { - ms.opt.lit_freq = vec![0; 256]; - ms.opt.lit_length_freq = vec![0; 36]; - ms.opt.match_length_freq = vec![0; 53]; - ms.opt.off_code_freq = vec![0; 32]; - ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; - ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; - } - ms.c_params = c_params; - - let block_size = (1usize << c_params.window_log).min(128 << 10); - let mut seq_store = SeqStore::new(); - seq_store.reserve( - block_size / if c_params.min_match == 3 { 3 } else { 4 }, - block_size, - )?; - - let bufs = Buffers { - in_buff: &[], - user: src, - dict, - }; - - // ZSTD_loadDictionaryContent(dtlm_fast, tfp_forCCtx) - if !dict.is_empty() { - let iend = dict.len(); - ms.window - .update(BufId::Dict.start(), BufId::Dict, dict.len(), false); - let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); - let ip = iend - dict.len().min(max_dict_size); - let prefix = Prefix { - buf: dict, - base: ms.window.base.wrapping_sub(BufId::Dict.start()), - }; - ms.next_to_update = prefix.index_of(ip); - ms.loaded_dict_end = prefix.index_of(iend); - if iend - ip > HASH_READ_SIZE { - match strategy { - Strategy::Fast => fast::fill_hash_table( - &mut ms, - prefix, - iend, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - )?, - Strategy::DFast => double_fast::fill_double_hash_table( - &mut ms, - prefix, - iend, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - )?, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { - if use_row { - ms.tag_table.fill(0); - lazy::row_update(&mut ms, prefix, iend - HASH_READ_SIZE)?; - } else { - lazy::insert_and_find_first_index(&mut ms, prefix, iend - HASH_READ_SIZE)?; - } - } - _ => opt::update_tree(&mut ms, prefix, iend - HASH_READ_SIZE, iend)?, - } - ms.next_to_update = prefix.index_of(iend); - } - } - - // ZSTD_compressContinue_internal - let src_addr = BufId::User.start(); - if !ms - .window - .update(src_addr, BufId::User, src.len(), ms.force_non_contiguous) - { - ms.force_non_contiguous = false; - ms.next_to_update = ms.window.dict_limit; - } - // ZSTD_compress_frameChunk - let max_dist = 1u32 << c_params.window_log; - if ms - .window - .dict_is_invalid(src_addr + src.len(), max_dist, ms.loaded_dict_end) - { - ms.loaded_dict_end = 0; - } - if ms - .window - .enforce_max_dist(src_addr, max_dist, ms.loaded_dict_end) - { - ms.loaded_dict_end = 0; - } - if ms.next_to_update < ms.window.low_limit { - ms.next_to_update = ms.window.low_limit; - } - // ZSTD_buildSeqStore - let prev = CompressedBlockState::new(); - let block = BlockSrc { - buf: bufs.get(ms.window.base_buf), - base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), - dict_buf: bufs.get(ms.window.dict_buf), - dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), - start: 0, - end: src.len(), - dms: None, - symbol_costs: &prev.entropy, - ldm_seq_store: None, - }; - let curr = block.index_of(0); - if curr > ms.next_to_update + 384 { - ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); - } - let dict_mode = ms.dict_mode(None); - let mut rep = prev.rep; - let last_ll = compress::compress_block( - &mut ms, - &mut seq_store, - &mut rep, - &block, - strategy, - use_row, - dict_mode, - )?; - - let mut out = Vec::new(); - let mut reps = prev.rep; - for (i, seq) in seq_store.sequences.iter().enumerate() { - let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); - if seq_store.long_length_pos as usize == i { - match seq_store.long_length_type { - LongLengthType::LiteralLength => ll += 0x10000, - LongLengthType::MatchLength => ml += 0x10000, - LongLengthType::None => {} - } - } - update_rep(&mut reps, seq.off_base, seq.lit_length == 0); - out.push((ll, ml, reps[0])); - } - out.push((last_ll as u32, 0, 0)); - assert_eq!( - reps, rep, - "the repeat offsets the block compressor leaves are those of its sequences" - ); - Ok(out) -} - -// ── long distance matching and the pre-splitter ─────────────────────────── - -use crate::compress::{ldm, presplit}; - -pub fn split_block(block: &[u8], level: i32) -> Result { - presplit::split_block(block, level) -} - -/// enableLdm, hashLog, bucketSizeLog, minMatchLength, hashRateLog, windowLog -fn ldm_params(p: [u32; 6]) -> LdmParams { - LdmParams { - enable_ldm: match p[0] { - 0 => ParamSwitch::Auto, - 1 => ParamSwitch::Enable, - _ => ParamSwitch::Disable, - }, - hash_log: p[1], - bucket_size_log: p[2], - min_match_length: p[3], - hash_rate_log: p[4], - window_log: p[5], - } -} - -pub fn ldm_adjust_parameters(p: [u32; 6], cp: [u32; 7]) -> [u32; 6] { - let mut params = ldm_params(p); - let c_params = CompressionParameters { - window_log: cp[0], - chain_log: cp[1], - hash_log: cp[2], - search_log: cp[3], - min_match: cp[4], - target_length: cp[5], - strategy: Strategy::from_int(cp[6] as i32), - }; - ldm::adjust_parameters(&mut params, &c_params); - [ - params.enable_ldm as u32, - params.hash_log, - params.bucket_size_log, - params.min_match_length, - params.hash_rate_log, - params.window_log, - ] -} -pub fn ldm_get_table_size(p: [u32; 6]) -> usize { - ldm::get_table_size(ldm_params(p)) -} -pub fn ldm_get_max_nb_seq(p: [u32; 6], max_chunk_size: usize) -> usize { - ldm::get_max_nb_seq(ldm_params(p), max_chunk_size) -} - -/// (offset, litLength, matchLength) -pub type Raw = (u32, u32, u32); -#[derive(Clone, PartialEq, Eq, Debug)] -pub struct RawStore { - pub seq: Vec, - pub pos: usize, - pub pos_in_sequence: usize, - pub size: usize, -} -impl RawStore { - fn to(&self) -> RawSeqStore { - RawSeqStore { - seq: self - .seq - .iter() - .map(|s| RawSeq { - offset: s.0, - lit_length: s.1, - match_length: s.2, - }) - .collect(), - pos: self.pos, - pos_in_sequence: self.pos_in_sequence, - size: self.size, - capacity: self.seq.len(), - } - } - fn from(s: &RawSeqStore) -> Self { - Self { - seq: s - .seq - .iter() - .map(|s| (s.offset, s.lit_length, s.match_length)) - .collect(), - pos: s.pos, - pos_in_sequence: s.pos_in_sequence, - size: s.size, - } - } -} -pub fn ldm_skip_sequences( - store: &mut RawStore, - src_size: usize, - min_match: u32, -) -> Result<(), ErrorCode> { - let mut s = store.to(); - let r = ldm::skip_sequences(&mut s, src_size, min_match); - *store = RawStore::from(&s); - r -} -pub fn ldm_skip_raw_seq_store_bytes( - store: &mut RawStore, - nb_bytes: usize, -) -> Result<(), ErrorCode> { - let mut s = store.to(); - let r = ldm::skip_raw_seq_store_bytes(&mut s, nb_bytes); - *store = RawStore::from(&s); - r -} - -const BUF_IDS: [BufId; 3] = [BufId::InBuff, BufId::User, BufId::Dict]; - -pub struct Ldm { - state: LdmState, - params: LdmParams, - bufs: [Vec; 3], - store: RawSeqStore, -} -impl Ldm { - /// `start_index`: if not 0, the window is as if that many bytes had gone through it and it had been cleared. - pub fn new(p: [u32; 6], buf_sizes: [usize; 3], capacity: usize, start_index: u32) -> Self { - let params = ldm_params(p); - let mut state = LdmState::new(); - state.hash_table = vec![LdmEntry::default(); 1 << params.hash_log]; - state.bucket_offsets = vec![0; 1 << (params.hash_log - params.bucket_size_log)]; - if start_index != 0 { - state.window.next_src = state.window.base + start_index as usize; - state.window.clear(); - } - let store = RawSeqStore { - seq: vec![RawSeq::default(); capacity], - pos: 0, - pos_in_sequence: 0, - size: 0, - capacity, - }; - Self { - state, - params, - bufs: buf_sizes.map(|n| vec![0; n]), - store, - } - } - pub fn update(&mut self, buf: usize, pos: usize, data: &[u8]) { - self.bufs[buf][pos..pos + data.len()].copy_from_slice(data); - self.state - .window - .update(BUF_IDS[buf].start() + pos, BUF_IDS[buf], data.len(), false); - } - /// Changes a byte without telling the window. - pub fn poke(&mut self, buf: usize, pos: usize, byte: u8) { - self.bufs[buf][pos] = byte; - } - pub fn load_dict(&mut self, buf: usize, pos: usize, data: &[u8]) -> Result<(), ErrorCode> { - self.update(buf, pos, data); - let prefix = Prefix { - buf: &self.bufs[buf], - base: self.state.window.base.wrapping_sub(BUF_IDS[buf].start()), - }; - self.state.loaded_dict_end = prefix.index_of(pos + data.len()); - ldm::fill_hash_table(&mut self.state, prefix, pos, pos + data.len(), &self.params) - } - pub fn generate(&mut self, pos: usize, len: usize, reset: bool) -> Result<(), ErrorCode> { - if reset { - self.store.size = 0; - self.store.pos = 0; - } - let bufs = Buffers { - in_buff: &self.bufs[0], - user: &self.bufs[1], - dict: &self.bufs[2], - }; - ldm::generate_sequences( - &mut self.state, - &mut self.store, - &self.params, - &bufs, - pos, - pos + len, - ) - } - pub fn seqs(&self) -> Vec { - self.store.seq[..self.store.size] - .iter() - .map(|s| (s.offset, s.lit_length, s.match_length)) - .collect() - } - /// dictLimit, lowLimit, loadedDictEnd, nextSrc - base, nbOverflowCorrections - pub fn window(&self) -> [u32; 5] { - let w = &self.state.window; - [ - w.dict_limit, - w.low_limit, - self.state.loaded_dict_end, - w.index_of(w.next_src), - w.nb_overflow_corrections, - ] - } - pub fn table(&self) -> impl Iterator + '_ { - self.state.hash_table.iter().map(|e| (e.offset, e.checksum)) - } - pub fn bucket_offsets(&self) -> &[u8] { - &self.state.bucket_offsets - } -} - -/// `ZSTD_ldm_blockCompress` over the blocks of one input, as the first input of a frame. -pub struct LdmBlocks { - ms: MatchState, - seq_store: SeqStore, - store: RawSeqStore, - rep: [u32; 3], - prev: CompressedBlockState, - dict: Vec, - src: Vec, - use_row: bool, -} -#[derive(PartialEq, Eq, Debug)] -pub struct BlockOut { - /// offBase, litLength, mlBase - pub seqs: Vec<(u32, u16, u16)>, - pub long_length: Option<(u32, u32)>, - pub literals: Vec, - pub last_ll: usize, - pub rep: [u32; 3], - pub next_to_update: u32, - pub store: RawStore, -} -impl LdmBlocks { - pub fn new( - cp: [u32; 7], - use_row: bool, - dict: &[u8], - src: &[u8], - seqs: &[Raw], - ) -> Result { - let strategy = Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?; - let c_params = CompressionParameters { - window_log: cp[0], - chain_log: cp[1], - hash_log: cp[2], - search_log: cp[3], - min_match: cp[4], - target_length: cp[5], - strategy: Some(strategy), - }; - let row_used = row_match_finder_supported(strategy) && use_row; - let mut ms = MatchState::new(); - ms.hash_log3 = if c_params.min_match == 3 { - c_params.window_log.min(17) - } else { - 0 - }; - ms.window.clear(); - ms.next_to_update = ms.window.dict_limit; - ms.hash_table = vec![0; 1 << c_params.hash_log]; - ms.chain_table = vec![ - 0; - if strategy != Strategy::Fast && !row_used { - 1 << c_params.chain_log - } else { - 0 - } - ]; - ms.hash_table3 = vec![ - 0; - if ms.hash_log3 != 0 { - 1 << ms.hash_log3 - } else { - 0 - } - ]; - if row_used { - ms.tag_table = vec![0; 1 << c_params.hash_log]; - ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4); - ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); - } - if strategy >= Strategy::BtOpt { - ms.opt.lit_freq = vec![0; 256]; - ms.opt.lit_length_freq = vec![0; 36]; - ms.opt.match_length_freq = vec![0; 53]; - ms.opt.off_code_freq = vec![0; 32]; - ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; - ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; - } - ms.c_params = c_params; - let mut seq_store = SeqStore::new(); - seq_store.reserve((128 << 10) / 3, 128 << 10)?; - if !dict.is_empty() { - let iend = dict.len(); - ms.window - .update(BufId::Dict.start(), BufId::Dict, dict.len(), false); - let max_dict_size = - 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); - let ip = iend - dict.len().min(max_dict_size); - let prefix = Prefix { - buf: dict, - base: ms.window.base.wrapping_sub(BufId::Dict.start()), - }; - ms.next_to_update = prefix.index_of(ip); - ms.loaded_dict_end = prefix.index_of(iend); - if iend - ip > HASH_READ_SIZE { - match strategy { - Strategy::Fast => fast::fill_hash_table( - &mut ms, - prefix, - iend, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - )?, - Strategy::DFast => double_fast::fill_double_hash_table( - &mut ms, - prefix, - iend, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - )?, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { - if use_row { - ms.tag_table.fill(0); - lazy::row_update(&mut ms, prefix, iend - HASH_READ_SIZE)?; - } else { - lazy::insert_and_find_first_index( - &mut ms, - prefix, - iend - HASH_READ_SIZE, - )?; - } - } - _ => opt::update_tree(&mut ms, prefix, iend - HASH_READ_SIZE, iend)?, - } - ms.next_to_update = prefix.index_of(iend); - } - } - if !ms.window.update( - BufId::User.start(), - BufId::User, - src.len(), - ms.force_non_contiguous, - ) { - ms.force_non_contiguous = false; - ms.next_to_update = ms.window.dict_limit; - } - let prev = CompressedBlockState::new(); - let store = RawStore { - seq: seqs.to_vec(), - pos: 0, - pos_in_sequence: 0, - size: seqs.len(), - } - .to(); - Ok(Self { - ms, - seq_store, - store, - rep: prev.rep, - prev, - dict: dict.to_vec(), - src: src.to_vec(), - use_row, - }) - } - - pub fn run(&mut self, pos: usize, block_size: usize) -> Result { - let ms = &mut self.ms; - let src_addr = BufId::User.start() + pos; - let max_dist = 1u32 << ms.c_params.window_log; - if ms - .window - .dict_is_invalid(src_addr + block_size, max_dist, ms.loaded_dict_end) - { - ms.loaded_dict_end = 0; - } - if ms - .window - .enforce_max_dist(src_addr, max_dist, ms.loaded_dict_end) - { - ms.loaded_dict_end = 0; - } - if ms.next_to_update < ms.window.low_limit { - ms.next_to_update = ms.window.low_limit; - } - self.seq_store.reset(); - let mut last_ll = 0; - if block_size < 7 { - if ms.c_params.strat() >= Strategy::BtOpt { - ldm::skip_raw_seq_store_bytes(&mut self.store, block_size)?; - } else { - ldm::skip_sequences(&mut self.store, block_size, ms.c_params.min_match)?; - } - } else { - let bufs = Buffers { - in_buff: &[], - user: &self.src, - dict: &self.dict, - }; - let block = BlockSrc { - buf: bufs.get(ms.window.base_buf), - base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), - dict_buf: bufs.get(ms.window.dict_buf), - dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), - start: pos, - end: pos + block_size, - dms: None, - symbol_costs: &self.prev.entropy, - ldm_seq_store: None, - }; - let curr = block.index_of(pos); - if curr > ms.next_to_update + 384 { - ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); - } - last_ll = ldm::block_compress( - &mut self.store, - ms, - &mut self.seq_store, - &mut self.rep, - self.use_row, - &block, - )?; - self.seq_store - .store_last_literals(&self.src[pos + block_size - last_ll..pos + block_size])?; - } - Ok(BlockOut { - seqs: self - .seq_store - .sequences - .iter() - .map(|s| (s.off_base, s.lit_length, s.ml_base)) - .collect(), - long_length: match self.seq_store.long_length_type { - LongLengthType::None => None, - LongLengthType::LiteralLength => Some((1, self.seq_store.long_length_pos)), - LongLengthType::MatchLength => Some((2, self.seq_store.long_length_pos)), - }, - literals: self.seq_store.literals().to_vec(), - last_ll, - rep: self.rep, - next_to_update: self.ms.next_to_update, - store: RawStore::from(&self.store), - }) - } -} diff --git a/src/zstd/wip/work/opt/STATUS.md b/src/zstd/wip/work/opt/STATUS.md deleted file mode 100644 index 8eec4032674e..000000000000 --- a/src/zstd/wip/work/opt/STATUS.md +++ /dev/null @@ -1,38 +0,0 @@ -# opt (zstd_opt.c -> compress/opt.rs): status at 12:30 UTC - -Deliverable: /root/zstd-ref/work/opt/zstd/compress/opt.rs. Complete port, public API of the stub unchanged. -No clippy finding in opt.rs, rustfmt clean (the fmt diffs of `cargo fmt --check` are all in the scratch-only testing.rs). - -## Verified against C (no difference found in any run of the final code) -Harness = zstd-lazy's fixed one, extended. Whole frames (several blocks, up to 400 KB), strategies 7,8,9. Compared: all -sequences, and after the frame hashTable, chainTable, hashTable3, window limits, nextToUpdate, rep, the 4 frequency -tables and the 8 sums / base prices of optState_t. -- Long runs, 100,000 iterations each, still running when this was written, no failure so far. Last progress lines: - seqs iter 84000 (seed 101084303);seqs iter 82000 (seed 102082306);seqs iter 84000 (seed 201084603);seqs iter 82000 (seed 202082606);seqs iter 88000 (seed 301088903); - (r0a, r0b: no dict; r1a, r1b: none/prefix = extDict; r2a: attached CDict = dictMatchState). Logs: runs/*.log -- Finished runs: modes 0, 1, 2: 3000 + 1000 + 800 + 1500 iterations each. -- "x" mode (shim_frame2 in harness/shim.c: block-level API, exact unadjusted cParams incl. targetLength 0, random LDM - rawSeqs through ZSTD_referenceExternalSequences, poked dictionary entropy tables with HUF_repeat_valid, - literalCompressionMode 0/1/2, input in two non-contiguous chunks, none/prefix/attached CDict): - 3000 + 3000 + 800 + 1500 iterations, about 2/3 with LDM sequences and 1/2 with entropy tables. -- Line + branch coverage of opt.rs under those runs: every line is hit but the ones listed below. - -## Not covered by any test -- zop_predef prices (3 early returns): cannot be reached, a first block of <= 8 bytes never enters the match loop. -- raw_literals_cost with 0 literals: never called so. -- lit_length_price(ZSTD_BLOCKSIZE_MAX): reachable (128 KB block without a sequence whose last 9+ bytes match). A data - kind for it was added to seqs.rs (noise + copied tail) but did not hit it in 6000 iterations. -- `inr < iend` false, `nb_matches < ZSTD_OPT_NUM` false, and btultra2's 2nd to 4th conditions false. -- Index overflow correction, windows > 22 bits. - -## How to run -cd harness && RUSTFLAGS="--cap-lints=warn" cargo build --release -./target/release/zstd_harness seqs 7,8,9 <0|1|2|3> [x] - 0 no dict, 1 none/prefix, 2 attached CDict, 3 all three; x = shim_frame2 extras -ZSTD_REF=asserts|trace picks a C lib built by cbuild/build.sh (asserts on / DEBUGLEVEL 8 in zstd_opt.c). - -## Notes for the integrator -- btultra2 (ZSTD_initStats_ultra) moves ms.window.base: BlockSrc.base must be derived from ms.window again for each block. -- The template seqs.rs sets cParams by 101+i, which swaps chainLog and hashLog (ids are windowLog, hashLog, chainLog). - The fixed harness is immune because it reads the applied cParams back from C. -- MAX_LIT (255) is missing from common/mod.rs: private const in opt.rs. diff --git a/src/zstd/wip/work/opt/harness/Cargo.toml b/src/zstd/wip/work/opt/harness/Cargo.toml deleted file mode 100644 index 131797af3edb..000000000000 --- a/src/zstd/wip/work/opt/harness/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "zstd_harness" -version = "0.0.0" -edition = "2024" - -[dependencies] -bun_zstd = { path = "/root/zstd-ref/work/opt/zstd" } - -[profile.release] -debug = "line-tables-only" -overflow-checks = true -debug-assertions = true -panic = "abort" - -[profile.bench] -inherits = "release" -overflow-checks = false -debug-assertions = false - -[workspace] diff --git a/src/zstd/wip/work/opt/harness/build.rs b/src/zstd/wip/work/opt/harness/build.rs deleted file mode 100644 index 91da5928b882..000000000000 --- a/src/zstd/wip/work/opt/harness/build.rs +++ /dev/null @@ -1,20 +0,0 @@ -use std::process::Command; -fn main() { - let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); - println!("cargo:rerun-if-env-changed=ZSTD_REF"); - println!("cargo:rerun-if-changed=shim.c"); - let out = std::env::var("OUT_DIR").unwrap(); - let ok = Command::new("clang-23") - .args(["-c", "-O2", "-fPIC", "-march=nehalem", "-DZSTD_MULTITHREAD", "-DZSTD_LEGACY_SUPPORT=0", "-DXXH_NAMESPACE=ZSTD_", - "-I/root/zstd-ref/zstd/lib", "-I/root/zstd-ref/zstd/lib/common", "shim.c", "-o"]) - .arg(format!("{out}/shim.o")).status().unwrap().success(); - assert!(ok); - let _ = std::fs::remove_file(format!("{out}/libshim.a")); - assert!(Command::new("llvm-ar-23").args(["rcs", &format!("{out}/libshim.a"), &format!("{out}/shim.o")]).status().unwrap().success()); - println!("cargo:rustc-link-search=native={out}"); - println!("cargo:rustc-link-lib=static=shim"); - let dir = if variant == "ref" || variant == "noasm" { format!("/root/zstd-ref/build/{variant}") } else { format!("/root/zstd-ref/work/opt/cbuild/{variant}") }; - println!("cargo:rerun-if-changed={dir}/libzstd.a"); - println!("cargo:rustc-link-search=native={dir}"); - println!("cargo:rustc-link-lib=static=zstd"); -} diff --git a/src/zstd/wip/work/opt/harness/rust-toolchain.toml b/src/zstd/wip/work/opt/harness/rust-toolchain.toml deleted file mode 100644 index c71778e8ccd3..000000000000 --- a/src/zstd/wip/work/opt/harness/rust-toolchain.toml +++ /dev/null @@ -1,35 +0,0 @@ -[toolchain] -channel = "nightly-2026-09-15" -# rust-src is needed for -Zbuild-std (Tier 3 targets like -# aarch64-unknown-freebsd have no prebuilt std). miri is for -# `bun run rust:miri`. targets ensures the -# Tier 2 cross-targets' prebuilt std is installed for THIS toolchain -# (CI's images install this same channel with these components and -# targets, from scripts/build/ci-images/spec.ts; a source lint keeps -# the two files in step). Note: rustup ignores this whole file when -# RUSTUP_TOOLCHAIN is set in the env (which CI does to pin across worktree -# symlinks) — the ninja `rust_build_cross` rule's `rustup target add` -# self-heals that path. Keep this list as a fallback for direct -# `cargo build`/`cargo check`/rust-analyzer use. -# Mirrors `allRustTargets` in scripts/build/rust.ts minus aarch64-unknown-freebsd -# (Tier 3 — no prebuilt rust-std, cargo `-Zbuild-std` builds it from rust-src). -# darwin/windows triples build natively on their own agents, so the host std -# comes with the toolchain install regardless; they're listed here so a dev -# running `bun run rust:check-all` or `cargo check --target` from any host -# still gets prebuilt std for every Tier 1/2 triple. -# llvm-tools: rustc's own llvm-nm, llvm-objdump, llvm-profdata and the rest, for looking at -# what it emits. The build does not use them. -components = ["rust-src", "rustfmt", "clippy", "miri", "llvm-tools"] -targets = [ - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "x86_64-unknown-linux-musl", - "aarch64-linux-android", - "x86_64-linux-android", - "x86_64-unknown-freebsd", - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-pc-windows-msvc", - "x86_64-pc-windows-msvc", -] diff --git a/src/zstd/wip/work/opt/harness/shim.c b/src/zstd/wip/work/opt/harness/shim.c deleted file mode 100644 index 95dacbec60fc..000000000000 --- a/src/zstd/wip/work/opt/harness/shim.c +++ /dev/null @@ -1,98 +0,0 @@ -// Exposes the internal state of the C library to the differential tests. -#define ZSTD_STATIC_LINKING_ONLY -#include "compress/zstd_compress_internal.h" -#include -#include - -typedef struct { - unsigned cp[7]; - unsigned useRow; - unsigned hasDms; - unsigned nextToUpdate, loadedDictEnd, lowLimit, dictLimit, rowHashLog, hashSaltEntropy, lazySkipping; - unsigned long long hashSalt; - unsigned hashCache[8]; - unsigned rep[3]; - const unsigned* hashTable; - const unsigned* chainTable; - const unsigned char* tagTable; - const unsigned* hashTable3; - unsigned hashLog3; - unsigned sums[8]; - const unsigned* freqs[4]; -} ShimState; - -void shim_state(const ZSTD_CCtx* c, ShimState* s) { - const ZSTD_MatchState_t* ms = &c->blockState.matchState; - const ZSTD_compressionParameters* p = &c->appliedParams.cParams; - int i; - s->cp[0] = p->windowLog; s->cp[1] = p->chainLog; s->cp[2] = p->hashLog; s->cp[3] = p->searchLog; - s->cp[4] = p->minMatch; s->cp[5] = p->targetLength; s->cp[6] = (unsigned)p->strategy; - s->useRow = c->appliedParams.useRowMatchFinder == ZSTD_ps_enable; - s->hasDms = ms->dictMatchState != NULL; - s->nextToUpdate = ms->nextToUpdate; s->loadedDictEnd = ms->loadedDictEnd; - s->lowLimit = ms->window.lowLimit; s->dictLimit = ms->window.dictLimit; - s->rowHashLog = ms->rowHashLog; s->hashSaltEntropy = ms->hashSaltEntropy; s->lazySkipping = (unsigned)ms->lazySkipping; - s->hashSalt = ms->hashSalt; - for (i = 0; i < 8; i++) s->hashCache[i] = ms->hashCache[i]; - for (i = 0; i < 3; i++) s->rep[i] = c->blockState.prevCBlock->rep[i]; - s->hashTable = ms->hashTable; s->chainTable = ms->chainTable; s->tagTable = ms->tagTable; - s->hashTable3 = ms->hashTable3; s->hashLog3 = ms->hashLog3; - s->sums[0] = ms->opt.litSum; s->sums[1] = ms->opt.litLengthSum; s->sums[2] = ms->opt.matchLengthSum; s->sums[3] = ms->opt.offCodeSum; - s->sums[4] = ms->opt.litSumBasePrice; s->sums[5] = ms->opt.litLengthSumBasePrice; s->sums[6] = ms->opt.matchLengthSumBasePrice; s->sums[7] = ms->opt.offCodeSumBasePrice; - s->freqs[0] = ms->opt.litFreq; s->freqs[1] = ms->opt.litLengthFreq; s->freqs[2] = ms->opt.matchLengthFreq; s->freqs[3] = ms->opt.offCodeFreq; -} - -typedef struct { - unsigned long long huf[257]; - unsigned ll[36], ml[53], of[32]; /* deltaNbBits */ -} ShimEntropy; - -static void shim_poke_fse(FSE_CTable* ct, unsigned tableLog, unsigned maxSymbol, const unsigned* deltaNbBits) { - FSE_symbolCompressionTransform* tt = (FSE_symbolCompressionTransform*)(ct + 1 + (1 << (tableLog - 1))); - unsigned s; - ((U16*)ct)[0] = (U16)tableLog; ((U16*)ct)[1] = (U16)maxSymbol; - for (s = 0; s <= maxSymbol; s++) { tt[s].deltaNbBits = deltaNbBits[s]; tt[s].deltaFindState = 0; } -} - -/* A frame by the block-level API, with what the public parameters cannot set: exact parameters, sequences that stand - * for those of long distance matching, the entropy tables of a dictionary. `cdict` or `dict` or none. */ -size_t shim_frame2(ZSTD_CCtx* c, const unsigned cp[7], const void* dict, size_t dictSize, const ZSTD_CDict* cdict, - int litMode, rawSeq* seqs, size_t nbSeqs, const ShimEntropy* e, - ZSTD_Sequence* out, size_t outCap, const void* first, size_t firstSize, const void* src, size_t srcSize) -{ - size_t const cap = ZSTD_compressBound(srcSize + firstSize); - void* const dst = malloc(cap); - size_t r; - if (cdict) { - ZSTD_frameParameters fp = { 0, 0, 0 }; - r = ZSTD_compressBegin_usingCDict_advanced(c, cdict, fp, ZSTD_CONTENTSIZE_UNKNOWN); - } else { - ZSTD_parameters p; - memset(&p, 0, sizeof(p)); - p.cParams.windowLog = cp[0]; p.cParams.chainLog = cp[1]; p.cParams.hashLog = cp[2]; p.cParams.searchLog = cp[3]; - p.cParams.minMatch = cp[4]; p.cParams.targetLength = cp[5]; p.cParams.strategy = (ZSTD_strategy)cp[6]; - r = ZSTD_compressBegin_advanced(c, dict, dictSize, p, srcSize + firstSize); - } - if (ZSTD_isError(r)) { free(dst); return r; } - c->appliedParams.postBlockSplitter = ZSTD_ps_disable; - c->appliedParams.literalCompressionMode = (ZSTD_ParamSwitch_e)litMode; - if (nbSeqs) ZSTD_referenceExternalSequences(c, seqs, nbSeqs); - if (e) { - ZSTD_entropyCTables_t* t = &c->blockState.prevCBlock->entropy; - memcpy(t->huf.CTable, e->huf, sizeof(e->huf)); - t->huf.repeatMode = HUF_repeat_valid; - shim_poke_fse(t->fse.litlengthCTable, LLFSELog, MaxLL, e->ll); - shim_poke_fse(t->fse.matchlengthCTable, MLFSELog, MaxML, e->ml); - shim_poke_fse(t->fse.offcodeCTable, OffFSELog, MaxOff, e->of); - } - c->seqCollector.collectSequences = 1; - c->seqCollector.seqStart = out; - c->seqCollector.seqIndex = 0; - c->seqCollector.maxSequences = outCap; - r = firstSize ? ZSTD_compressContinue_public(c, dst, cap, first, firstSize) : 0; - if (!ZSTD_isError(r)) r = ZSTD_compressEnd_public(c, dst, cap, src, srcSize); - free(dst); - c->seqCollector.collectSequences = 0; - if (ZSTD_isError(r)) return r; - return c->seqCollector.seqIndex; -} diff --git a/src/zstd/wip/work/opt/harness/src/c.rs b/src/zstd/wip/work/opt/harness/src/c.rs deleted file mode 100644 index d5a82e1f6b78..000000000000 --- a/src/zstd/wip/work/opt/harness/src/c.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! The reference C library. -#![allow(non_camel_case_types, dead_code)] -use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; - -#[repr(C)] -pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } -#[repr(C)] -pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } -pub enum ZSTD_CCtx {} -pub enum ZSTD_DCtx {} -pub enum ZSTD_DDict {} - -unsafe extern "C" { - pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; - pub fn ZSTD_compressBound(size: usize) -> usize; - pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_isError(code: usize) -> c_uint; - pub fn ZSTD_getErrorCode(code: usize) -> c_uint; - pub fn ZSTD_getErrorName(code: usize) -> *const c_char; - pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; - pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; - pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; - pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; - pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; - pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; - pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; - pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; - pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; - pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; - pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; - pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; - pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; - pub fn ZSTD_defaultCLevel() -> c_int; - pub fn ZSTD_minCLevel() -> c_int; - pub fn ZSTD_maxCLevel() -> c_int; -} - -/// `Ok(value)` or `Err(error code)`. -pub fn res(code: usize) -> Result { - unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } -} - -pub struct CCtx(pub *mut ZSTD_CCtx); -impl CCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } - pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - /// Returns (result, out pos, in pos). - pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); - (r, o.pos, i.pos) - } -} -impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } - -pub struct DCtx(pub *mut ZSTD_DCtx); -impl DCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } - pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } - pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); - (r, o.pos, i.pos) - } -} -impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } - -pub fn compress(src: &[u8], level: i32) -> Vec { - let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; - let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); - dst.truncate(n); - dst -} -pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) -} diff --git a/src/zstd/wip/work/opt/harness/src/data.rs b/src/zstd/wip/work/opt/harness/src/data.rs deleted file mode 100644 index c48a645e1b5d..000000000000 --- a/src/zstd/wip/work/opt/harness/src/data.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Deterministic inputs. -pub struct Rng(pub u64); -impl Rng { - pub fn next(&mut self) -> u64 { - // splitmix64 - self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); - z ^ (z >> 31) - } - pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } - pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } - pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } - pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } - /// Skewed towards small values. - pub fn size(&mut self, max: usize) -> usize { - let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); - (self.next() as usize & ((1usize << bits) - 1)).min(max) - } -} - -static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; - -/// One buffer of about `len` bytes, of a kind picked by `rng`. -pub fn data(rng: &mut Rng, len: usize) -> Vec { - let mut v = Vec::with_capacity(len + 64); - match rng.below(9) { - 0 => v.resize(len, rng.next() as u8), - 1 => while v.len() < len { v.push(rng.next() as u8) }, - 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, - 3 => { // few symbols, skewed - let n = rng.range(2, 40); - while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } - } - 4 => { // LZ-like: copies of earlier parts, at all distances - while v.len() < len { - if v.len() > 8 && rng.chance(3, 4) { - let dist = 1 + rng.size(v.len() - 1); - let n = 3 + rng.size(300); - for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } - } else { - for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } - } - } - } - 5 => { // periodic with noise - let period = rng.range(1, 300); - let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); - while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } - } - 6 => { // sections of different kinds - while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } - } - 7 => { // numbers as text - let mut x = rng.next() % 1000; - while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } - } - _ => { // long runs - while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } - } - } - v.truncate(len); - v -} diff --git a/src/zstd/wip/work/opt/harness/src/main.rs b/src/zstd/wip/work/opt/harness/src/main.rs deleted file mode 100644 index adce8cad3d7a..000000000000 --- a/src/zstd/wip/work/opt/harness/src/main.rs +++ /dev/null @@ -1,25 +0,0 @@ -mod c; -mod data; -mod seqs; - -fn main() { - let args: Vec = std::env::args().collect(); - let cmd = args.get(1).map(String::as_str).unwrap_or(""); - let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); - let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); - match cmd { - // seqs - "seqs" => { - let strategies: Vec = args[4].split(',').map(|s| s.parse().unwrap()).collect(); - // 0: no dictionary, 1: a prefix or none, 2: an attached CDict or none - let modes: &[seqs::Mode] = match args.get(5).map(String::as_str) { - Some("1") => &[seqs::Mode::NoDict, seqs::Mode::Prefix], - Some("3") => &[seqs::Mode::CDict, seqs::Mode::Prefix, seqs::Mode::NoDict], - Some("2") => &[seqs::Mode::CDict, seqs::Mode::CDict, seqs::Mode::CDict, seqs::Mode::NoDict], - _ => &[seqs::Mode::NoDict], - }; - seqs::run(seed, iters, &strategies, modes, args.get(6).is_some_and(|s| s == "x")); - } - _ => eprintln!("unknown command"), - } -} diff --git a/src/zstd/wip/work/opt/harness/src/seqs.rs b/src/zstd/wip/work/opt/harness/src/seqs.rs deleted file mode 100644 index 57a0a4752b18..000000000000 --- a/src/zstd/wip/work/opt/harness/src/seqs.rs +++ /dev/null @@ -1,235 +0,0 @@ -//! A frame through the Rust block compressors and through the C library (`ZSTD_generateSequences`). -use crate::c; -use crate::data::{Rng, data}; -use bun_zstd::testing::{Dict, Extra, State}; -use std::ffi::{c_uint, c_void}; - -#[repr(C)] -#[derive(Clone, Copy, Default, Debug)] -struct Sequence { offset: c_uint, lit_length: c_uint, match_length: c_uint, rep: c_uint } -#[repr(C)] -#[derive(Clone, Copy, Debug)] -struct CParams { window_log: c_uint, chain_log: c_uint, hash_log: c_uint, search_log: c_uint, min_match: c_uint, target_length: c_uint, strategy: c_uint } -#[repr(C)] -#[derive(Clone, Copy)] -struct CustomMem { a: *const c_void, b: *const c_void, c: *const c_void } -#[repr(C)] -struct ShimState { - cp: [c_uint; 7], use_row: c_uint, has_dms: c_uint, - next_to_update: c_uint, loaded_dict_end: c_uint, low_limit: c_uint, dict_limit: c_uint, row_hash_log: c_uint, hash_salt_entropy: c_uint, lazy_skipping: c_uint, - hash_salt: u64, hash_cache: [c_uint; 8], rep: [c_uint; 3], - hash_table: *const u32, chain_table: *const u32, tag_table: *const u8, - hash_table3: *const u32, hash_log3: c_uint, sums: [c_uint; 8], freqs: [*const u32; 4], -} -enum CDict {} -enum CCtxParams {} -unsafe extern "C" { - fn ZSTD_generateSequences(c: *mut c::ZSTD_CCtx, out: *mut Sequence, cap: usize, src: *const c_void, size: usize) -> usize; - fn ZSTD_adjustCParams(p: CParams, src_size: u64, dict_size: usize) -> CParams; - fn ZSTD_getCParams(level: i32, src_size: u64, dict_size: usize) -> CParams; - fn ZSTD_createCCtxParams() -> *mut CCtxParams; - fn ZSTD_freeCCtxParams(p: *mut CCtxParams) -> usize; - fn ZSTD_CCtxParams_setParameter(p: *mut CCtxParams, param: c_uint, v: i32) -> usize; - fn ZSTD_createCDict_advanced2(d: *const c_void, n: usize, load: c_uint, ty: c_uint, p: *const CCtxParams, mem: CustomMem) -> *mut CDict; - fn ZSTD_freeCDict(d: *mut CDict) -> usize; - fn ZSTD_CCtx_refCDict(c: *mut c::ZSTD_CCtx, d: *const CDict) -> usize; - fn ZSTD_getCParamsFromCDict(d: *const CDict) -> CParams; - fn shim_state(c: *const c::ZSTD_CCtx, s: *mut ShimState); - fn shim_frame2(c: *mut c::ZSTD_CCtx, cp: *const c_uint, dict: *const c_void, dict_size: usize, cdict: *const CDict, lit_mode: i32, - seqs: *mut [u32; 3], nb_seqs: usize, e: *const ShimEntropy, out: *mut Sequence, cap: usize, first: *const c_void, first_size: usize, src: *const c_void, size: usize) -> usize; -} - -#[repr(C)] -pub struct ShimEntropy { huf: [u64; 257], ll: [u32; 36], ml: [u32; 53], of: [u32; 32] } - -/// What `shim_frame2` sets beside the parameters. -pub struct CExtra { lit_mode: u32, ldm: Vec<[u32; 3]>, entropy: Option>, first_chunk: Vec } - -fn arr(p: CParams) -> [u32; 7] { [p.window_log, p.chain_log, p.hash_log, p.search_log, p.min_match, p.target_length, p.strategy] } - -#[derive(Clone, Copy, PartialEq, Debug)] -pub enum Mode { NoDict, Prefix, CDict } - -struct COut { seqs: Vec<(u32, u32, u32, u32)>, cp: [u32; 7], use_row: bool, cdict_cp: [u32; 7], state: State } - -/// `None` if the C library refuses (a last block that is too small). -fn c_frame(cp: [u32; 7], use_row: bool, mode: Mode, dict: &[u8], src: &[u8], extra: Option<&CExtra>) -> Option { - let mut cctx = c::CCtx::new(); - let mut cdict = std::ptr::null_mut(); - let mut cdict_cp = [0; 7]; - cctx.set(1010, 2).unwrap(); // no block splitter - if mode == Mode::CDict { - unsafe { - let p = ZSTD_createCCtxParams(); - for (i, v) in cp.iter().enumerate() { c::res(ZSTD_CCtxParams_setParameter(p, 101 + i as u32, *v as i32)).unwrap(); } - c::res(ZSTD_CCtxParams_setParameter(p, 1011, if use_row { 1 } else { 2 })).unwrap(); - let null = std::ptr::null(); - cdict = ZSTD_createCDict_advanced2(dict.as_ptr().cast(), dict.len(), 1, 1, p, CustomMem { a: null, b: null, c: null }); - assert!(!cdict.is_null()); - ZSTD_freeCCtxParams(p); - if extra.is_none() { c::res(ZSTD_CCtx_refCDict(cctx.0, cdict)).unwrap(); } - cdict_cp = arr(ZSTD_getCParamsFromCDict(cdict)); - } - cctx.set(1001, 1).unwrap(); // force attach - cctx.set(101, cp[0] as i32).unwrap(); - } else { - for (i, v) in cp.iter().enumerate() { cctx.set(101 + i as u32, *v as i32).unwrap(); } - cctx.set(1011, if use_row { 1 } else { 2 }).unwrap(); // useRowMatchFinder - if mode == Mode::Prefix { cctx.ref_prefix(dict).unwrap(); } - } - let total = src.len() + extra.map_or(0, |x| x.first_chunk.len()); - let mut out = vec![Sequence::default(); total / 3 + total / 1024 + 8]; - let n = c::res(unsafe { - match extra { - None => ZSTD_generateSequences(cctx.0, out.as_mut_ptr(), out.len(), src.as_ptr().cast(), src.len()), - Some(x) => { - let mut ldm = x.ldm.clone(); - let d = if mode == Mode::Prefix { dict } else { &[] }; - shim_frame2(cctx.0, cp.as_ptr(), d.as_ptr().cast(), d.len(), cdict, x.lit_mode as i32, ldm.as_mut_ptr(), ldm.len(), - x.entropy.as_deref().map_or(std::ptr::null(), |e| e), out.as_mut_ptr(), out.len(), x.first_chunk.as_ptr().cast(), x.first_chunk.len(), src.as_ptr().cast(), src.len()) - } - } - }); - let r = n.ok().map(|n| { - let mut s: ShimState = unsafe { std::mem::zeroed() }; - unsafe { shim_state(cctx.0, &mut s) }; - let row_used = s.use_row != 0 && (3..=5).contains(&s.cp[6]); - let table = |p: *const u32, n: usize| if n == 0 { Vec::new() } else { unsafe { std::slice::from_raw_parts(p, n) }.to_vec() }; - let state = State { - has_dms: s.has_dms != 0, next_to_update: s.next_to_update, loaded_dict_end: s.loaded_dict_end, low_limit: s.low_limit, dict_limit: s.dict_limit, - hash_salt_entropy: s.hash_salt_entropy, lazy_skipping: s.lazy_skipping != 0, hash_cache: s.hash_cache, rep: s.rep, - hash_table: table(s.hash_table, 1 << s.cp[2]), - chain_table: table(s.chain_table, if s.cp[6] != 1 && !row_used { 1 << s.cp[1] } else { 0 }), - hash_table3: table(s.hash_table3, if s.hash_log3 != 0 { 1 << s.hash_log3 } else { 0 }), - sums: if s.cp[6] >= 7 { s.sums } else { [0; 8] }, - freqs: if s.cp[6] >= 7 { [table(s.freqs[0], 256), table(s.freqs[1], 36), table(s.freqs[2], 53), table(s.freqs[3], 32)] } else { Default::default() }, - tag_table: if row_used { unsafe { std::slice::from_raw_parts(s.tag_table, 1 << s.cp[2]) }.to_vec() } else { Vec::new() }, - }; - COut { - seqs: out[..n].iter().map(|s| (s.lit_length, s.match_length, s.rep, if s.rep == 0 { s.offset } else { 0 })).collect(), - cp: s.cp, use_row: s.use_row != 0, cdict_cp, state, - } - }); - if !cdict.is_null() { unsafe { ZSTD_freeCDict(cdict) }; } - r -} - -fn first_diff(a: &[T], b: &[T]) -> Option { - if a.len() != b.len() { return Some(a.len().min(b.len())); } - a.iter().zip(b).position(|(a, b)| a != b) -} - -/// `strategies`: which of 1..=9 to test. `modes`: which kinds of dictionary. -pub fn run(seed: u64, iters: u64, strategies: &[u32], modes: &[Mode], with_extra: bool) { - let (mut with_ldm, mut with_entropy) = (0u64, 0u64); - let mut skipped = 0u64; - let mut dms_blocks = 0u64; - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if i % 2000 == 0 { eprintln!("seqs iter {i} (seed {s})"); } - let max = *rng.pick(&[40usize, 300, 3000, 30_000, 131_071, 400_000]); - let len = rng.range(7, max); - let mut src = data(&mut rng, len); - if rng.chance(1, 40) { - // Whole blocks of noise, of which only the end matches something. - src = (0..(128 << 10) * rng.range(1, 2)).map(|_| rng.next() as u8).collect(); - let n = rng.range(9, 40); - let (from, to) = (rng.below(src.len() - 2 * n), src.len() - n); - src.copy_within(from..from + n, to); - } - let mode = *rng.pick(modes); - let mut dict = if mode != Mode::NoDict { let m = *rng.pick(&[20usize, 1000, 100_000, 600_000]); let n = rng.size(m); data(&mut rng, n) } else { Vec::new() }; - if !dict.is_empty() && rng.chance(1, 4) { - // The input goes on where the dictionary ends. - let mut all = data(&mut rng, dict.len() + src.len()); - src = all.split_off(dict.len()); - dict = all; - } - let mode = if dict.is_empty() { Mode::NoDict } else { mode }; - if !dict.is_empty() && rng.chance(2, 3) { - for _ in 0..rng.range(1, 6) { - let n = rng.size(dict.len().min(src.len())); - let at = if rng.chance(1, 3) { dict.len() - n } else { rng.below(dict.len() - n + 1) }; - let to = if rng.chance(1, 4) { 0 } else { rng.below(src.len() - n + 1) }; - src[to..to + n].copy_from_slice(&dict[at..at + n]); - } - } - let strategy = *rng.pick(strategies); - let p = if rng.chance(1, 3) { - // the parameters of a level, with the strategy forced - let mut p = unsafe { ZSTD_getCParams(rng.range(1, 22) as i32, src.len() as u64, dict.len()) }; - p.strategy = strategy; - p - } else { - CParams { - window_log: rng.range(10, 22) as u32, chain_log: rng.range(6, 20) as u32, hash_log: rng.range(6, 20) as u32, - search_log: rng.range(1, 9) as u32, min_match: rng.range(3, 7) as u32, - target_length: *rng.pick(&[0u32, 1, 2, 4, 8, 16, 32, 64, 128, 256, 999, 4096]), strategy, - } - }; - let p = if with_extra && rng.chance(1, 2) { p } else { unsafe { ZSTD_adjustCParams(p, src.len() as u64, dict.len()) } }; - let use_row = rng.chance(1, 2); - let extra = with_extra.then(|| { - let mut ldm = Vec::new(); - if rng.chance(2, 3) { - let max_ll = *rng.pick(&[4usize, 100, 3000, 100_000]); - let max_ml = *rng.pick(&[8usize, 100, 3000, 5000, 200_000]); - let target = if rng.chance(1, 4) { rng.below(src.len()) } else { src.len() + rng.below(1000) }; - let mut total = 0; - while total < target { - let (ll, ml) = (rng.size(max_ll), rng.size(max_ml)); - // Most of them are matches, as far as the block goes. - let at = total + ll; - let offset = if rng.chance(1, 8) || at == 0 || at >= src.len() { 1 + rng.size(1 << p.window_log) } else { 1 + rng.size(at - 1) }; - ldm.push([offset as u32, ll as u32, ml as u32]); - total += ll + ml; - } - } - let entropy = rng.chance(1, 2).then(|| { - let mut e = Box::new(ShimEntropy { huf: [0; 257], ll: [0; 36], ml: [0; 53], of: [0; 32] }); - let max_symbol = if rng.chance(3, 4) { 255 } else { rng.below(256) as u64 }; - e.huf[0] = 11 | max_symbol << 8; - let lo = rng.below(2); - for v in &mut e.huf[1..] { *v = rng.range(lo, 11) as u64 | rng.next() << 8; } - let mut fill = |t: &mut [u32], max: usize| for v in t { let bits = rng.range(lo, max) as u32; *v = (bits << 16).wrapping_sub(rng.below(1 << 16) as u32 * (bits != 0) as u32); }; - fill(&mut e.ll, 10); fill(&mut e.ml, 10); fill(&mut e.of, 9); - e - }); - let first_chunk = if src.len() > 20 && rng.chance(1, 2) { - let rest = src.split_off(if rng.chance(1, 2) { rng.range(7, src.len() - 7) } else { src.len() - 7 - rng.size(src.len() - 14) }); - std::mem::replace(&mut src, rest) - } else { Vec::new() }; - CExtra { lit_mode: rng.below(3) as u32, ldm, entropy, first_chunk } - }); - let Some(mut c) = c_frame(arr(p), use_row, mode, &dict, &src, extra.as_ref()) else { skipped += 1; continue }; - let d = match mode { Mode::NoDict => Dict::None, Mode::Prefix => Dict::Prefix(&dict), Mode::CDict => Dict::CDict(&dict, c.cdict_cp) }; - let what = format!("seed {s}: {mode:?} cp {:?} cdict cp {:?} row {} dict {} src {}", c.cp, c.cdict_cp, c.use_row, dict.len(), src.len()); - let ldm: Vec<(u32, u32, u32)> = extra.iter().flat_map(|x| &x.ldm).map(|s| (s[0], s[1], s[2])).collect(); - let x = extra.as_ref().map_or(Extra::default(), |x| Extra { - lit_mode: x.lit_mode, ldm: &ldm, entropy: x.entropy.as_deref().map(|e| (&e.huf, &e.ll, &e.ml, &e.of)), - first_chunk: &x.first_chunk, - }); - with_ldm += !ldm.is_empty() as u64; - with_entropy += x.entropy.is_some() as u64; - let what = format!("{what} lit mode {} ldm {} entropy {} first chunk {}", x.lit_mode, ldm.len(), x.entropy.is_some(), x.first_chunk.len()); - let (got, mut state) = bun_zstd::testing::frame2(c.cp, c.use_row, d, &src, x).unwrap_or_else(|e| panic!("{what}: error {e:?}")); - if x.lit_mode == 2 { - // Nothing writes the statistics of literals. - state.freqs[0].clear(); c.state.freqs[0].clear(); - state.sums[0] = 0; c.state.sums[0] = 0; state.sums[4] = 0; c.state.sums[4] = 0; - } - if mode == Mode::CDict && dict.len() >= 8 { dms_blocks += 1; } - if let Some(at) = first_diff(&got, &c.seqs) { - panic!("{what}: sequence {at} differs: got {:?}, expected {:?} ({} vs {} sequences)", got.get(at), c.seqs.get(at), got.len(), c.seqs.len()); - } - if state != c.state { - let (a, b) = (&state, &c.state); - panic!("{what}: state differs: hash table 3 {:?} sums {:?} {:?} freqs {:?} hash table {:?} chain table {:?} tag table {:?}\n got {:?}\n exp {:?}", - first_diff(&a.hash_table3, &b.hash_table3), a.sums, b.sums, a.freqs == b.freqs, first_diff(&a.hash_table, &b.hash_table), first_diff(&a.chain_table, &b.chain_table), first_diff(&a.tag_table, &b.tag_table), - (a.has_dms, a.next_to_update, a.loaded_dict_end, a.low_limit, a.dict_limit, a.hash_salt_entropy, a.lazy_skipping, a.hash_cache, a.rep), - (b.has_dms, b.next_to_update, b.loaded_dict_end, b.low_limit, b.dict_limit, b.hash_salt_entropy, b.lazy_skipping, b.hash_cache, b.rep)); - } - } - println!("seqs ok: {iters} iterations from seed {seed} ({skipped} skipped, {dms_blocks} with an attached CDict, {with_ldm} with LDM sequences, {with_entropy} with entropy tables), strategies {strategies:?}, modes {modes:?}"); -} diff --git a/src/zstd/wip/work/opt/zstd/lib.rs b/src/zstd/wip/work/opt/zstd/lib.rs deleted file mode 100644 index 31c98c72e5e4..000000000000 --- a/src/zstd/wip/work/opt/zstd/lib.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the -//! same bytes in, in both directions, and the same errors. -#![forbid(unsafe_code)] -// Frames come from anywhere and a panic aborts the process. -#![deny( - clippy::indexing_slicing, - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::unreachable -)] -#![warn(unused_must_use)] - -mod common; -mod compress; -mod decompress; -mod error; -mod util; - -pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; -pub use decompress::{ - DCtx, DDict, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, - find_frame_compressed_size, get_frame_content_size, -}; -pub use error::ErrorCode; - -/// `ZSTD_inBuffer` -pub struct InBuffer<'a> { - pub src: &'a [u8], - /// How much of `src` is read. - pub pos: usize, -} - -/// `ZSTD_outBuffer` -pub struct OutBuffer<'a> { - pub dst: &'a mut [u8], - /// How much of `dst` is written. - pub pos: usize, -} - -/// `ZSTD_ResetDirective` -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum ResetDirective { - SessionOnly = 1, - Parameters = 2, - SessionAndParameters = 3, -} - -// ----------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] -pub enum ZstdError { - /// The output, or the decoder state the frame's window size dictates, could not be allocated. - OutOfMemory, - InvalidZstdData, - DecompressionFailed, - ZstdDecompressionError, - ShortRead, -} - -impl core::fmt::Display for ZstdError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.write_str(<&'static str>::from(self)) - } -} -impl core::error::Error for ZstdError {} - -impl ZstdError { - /// The error for a failed decompression call; `other` is the non-allocation failure. - fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { - if code == ErrorCode::MemoryAllocation { - ZstdError::OutOfMemory - } else { - other - } - } -} - -/// `len` zeros for a decoder to write to. -fn output_buffer(len: usize) -> Result, ZstdError> { - util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) -} - -/// Decompress data, automatically allocating the output buffer. -/// Handles both frames with known and unknown content sizes. -/// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. -/// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. -pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { - let size = find_decompressed_size(src); - - const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit - - if size == ZSTD_CONTENTSIZE_ERROR { - return Err(ZstdError::InvalidZstdData); - } - - // Use streaming decompression if: - // 1. Content size is unknown, OR - // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) - if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { - let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { - // A frame's output is rarely smaller than its input. - src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) - } else { - // The header size is untrusted: reserve no more than the fast path below would. - MAX_PREALLOCATE_SIZE - }; - let mut list: Vec = Vec::new(); - list.try_reserve_exact(initial_capacity) - .map_err(|_| ZstdError::OutOfMemory)?; - StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; - return Ok(list); - } - - // Fast path: size is known and within reasonable limits - let mut output = output_buffer(size as usize)?; - let written = decompress(&mut output, src) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -/// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is -/// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. -pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { - const MAX_SIZE: u64 = 16 * 1024 * 1024; - let size = find_decompressed_size(src); - // Also if it is unknown or invalid, which are the two largest values. - if size > MAX_SIZE { - return Err(ZstdError::InvalidZstdData); - } - let mut output = output_buffer(size as usize)?; - let written = DDict::prefix(prefix) - .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum State { - Uninitialized, - Inflating, - End, - Error, -} - -/// Minimum spare output capacity offered to the decoder per call. -const STREAMING_OUTPUT_STEP: usize = 4096; - -// ────────────────────────────────────────────────────────────────────────── -// StreamingDecoder -// ────────────────────────────────────────────────────────────────────────── - -/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) -/// per call, so callers can hold the decoder across multiple body chunks. -pub struct StreamingDecoder { - stream: DCtx, - state: State, - /// The decoder filled its last window and may hold more. `max_output` can end a call there. - output_full: bool, -} - -impl Default for StreamingDecoder { - fn default() -> Self { - Self::new() - } -} - -impl StreamingDecoder { - pub fn new() -> Self { - Self { - stream: DCtx::new(), - state: State::Uninitialized, - output_full: false, - } - } - - #[inline] - pub fn is_inflating(&self) -> bool { - matches!(self.state, State::Inflating) - } - - /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is - /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. - /// Returns `ShortRead` when more input is required and `is_done` is false. - pub fn decompress( - &mut self, - input: &[u8], - out: &mut Vec, - max_output: usize, - is_done: bool, - ) -> Result { - if matches!(self.state, State::End | State::Error) { - return Ok(input.len()); - } - - let mut in_buf = InBuffer { src: input, pos: 0 }; - while matches!(self.state, State::Uninitialized | State::Inflating) { - let next_in_len = input.len() - in_buf.pos; - - // Call the decoder again with no input until it leaves the window short. - if next_in_len == 0 && !self.output_full { - if is_done { - if self.state == State::Inflating { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - self.state = State::End; - } - return Ok(in_buf.pos); - } - - if out.len() >= max_output { - return Ok(in_buf.pos); - } - - if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { - self.state = State::Error; - return Err(ZstdError::OutOfMemory); - } - // The decoder writes to initialized memory only. - let written_before = out.len(); - let window = (out.capacity() - written_before).min(max_output - written_before); - out.resize(written_before + window, 0); - let mut out_buf = OutBuffer { - dst: out.get_mut(written_before..).unwrap_or_default(), - pos: 0, - }; - let read_before = in_buf.pos; - - let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); - let bytes_written = out_buf.pos; - out.truncate(written_before + bytes_written); - let remaining = match result { - Ok(remaining) => remaining, - Err(code) => { - self.state = State::Error; - return Err(ZstdError::for_decompression( - code, - ZstdError::ZstdDecompressionError, - )); - } - }; - - self.output_full = bytes_written == window; - - if remaining == 0 { - // Frame complete, and fully flushed. - self.state = State::Uninitialized; - self.output_full = false; - if in_buf.pos >= input.len() { - if is_done { - self.state = State::End; - } - return Ok(in_buf.pos); - } - // More input available — reinitialize for the next frame. - self.stream.init_stream(); - continue; - } - - self.state = State::Inflating; - - if in_buf.pos - read_before == next_in_len { - if self.output_full { - continue; - } - if is_done { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - return Err(ZstdError::ShortRead); - } - } - Ok(in_buf.pos) - } -} - -// ── compressed embedded assets ──────────────────────────────────────────── - -/// Embed an asset zstd-compressed and inflate it on first use. Only for bytes -/// Bun never executes or parses itself: the shell completion scripts and the -/// JS/CSS bundles that are shipped to a browser (dev-server client runtime, -/// error overlay/page). Anything that runs inside Bun stays uncompressed. -/// -/// The exception is text that only `bun lint`, `bun fmt` and `bun check` -/// read. It may be compressed because it isn't on the path of running a program. -/// -/// Release (`bun_codegen_embed`) builds include the `.zst` twin that -/// `scripts/build/codegen.ts` (`emitCompressedEmbeds`) writes to -/// `/compressed/.zst`; other builds evaluate `$fallback`. -#[macro_export] -macro_rules! embed_compressed { - // A codegen output (`/$sub`); debug builds read it at runtime. - (codegen $sub:literal) => { - $crate::embed_compressed!( - ("codegen/", $sub), - ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes() - ) - }; - // Same, with a trailing NUL included in the slice (for C-string consumers). - (codegen_nul $sub:literal) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded_nul(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/codegen/", - $sub, - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - static __COPY: ::std::sync::OnceLock<::std::boxed::Box<[u8]>> = ::std::sync::OnceLock::new(); - &__COPY.get_or_init(|| { - let text = ::bun_core::runtime_embed_file!(Codegen, $sub).as_bytes(); - let mut copy = ::std::vec::Vec::with_capacity(text.len() + 1); - copy.extend_from_slice(text); - copy.push(0); - copy.into_boxed_slice() - })[..] - } - }; - __bytes - }}; - // A file under `src/`; debug builds read it from the source tree at runtime. - (src $sub:literal) => { - $crate::embed_compressed!(("src/", $sub), ::bun_core::runtime_embed_file!(Src, $sub).as_bytes()) - }; - ($name:literal, $fallback:expr) => { - $crate::embed_compressed!(($name), $fallback) - }; - (($($name:literal),+), $fallback:expr) => {{ - #[allow(unexpected_cfgs)] - let __bytes: &'static [u8] = { - #[cfg(bun_codegen_embed)] - { - static __INFLATED: ::bun_core::Once<::std::boxed::Box<[u8]>> = ::bun_core::Once::new(); - __INFLATED - .get_or_init(|| { - $crate::inflate_embedded(::core::include_bytes!(::core::concat!( - ::core::env!("BUN_CODEGEN_DIR"), - "/compressed/", - $($name,)+ - ".zst" - ))) - }) - } - #[cfg(not(bun_codegen_embed))] - { - $fallback - } - }; - __bytes - }}; -} - -/// Cold, shared body of [`embed_compressed!`]'s release arm. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { - decompress_alloc(compressed) - .expect("embedded asset: invalid zstd frame") - .into_boxed_slice() -} - -/// [`inflate_embedded`] plus a trailing NUL byte. -#[cold] -#[inline(never)] -#[expect(clippy::expect_used, reason = "the build made the frame")] -pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { - let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); - inflated.reserve_exact(1); - inflated.push(0); - inflated.into_boxed_slice() -} -pub mod testing; diff --git a/src/zstd/wip/work/opt/zstd/testing.rs b/src/zstd/wip/work/opt/zstd/testing.rs deleted file mode 100644 index 36fe6bd042bb..000000000000 --- a/src/zstd/wip/work/opt/zstd/testing.rs +++ /dev/null @@ -1,288 +0,0 @@ -//! Hooks for the differential tests. Only in scratch copies of the crate. -#![allow(clippy::all, dead_code, unreachable_pub)] - -use crate::ErrorCode; -use crate::compress::internal::*; -use crate::compress::{self, double_fast, fast, lazy, opt, row_match_finder_supported}; - -fn bitmix(mut val: u64, len: u64) -> u64 { - val ^= val.rotate_right(49) ^ val.rotate_right(24); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^= (val >> 35).wrapping_add(len); - val = val.wrapping_mul(0x9FB21C651E98DF25); - val ^ (val >> 28) -} - -#[derive(Clone, Copy)] -pub enum Dict<'a> { - None, - /// `ZSTD_CCtx_refPrefix`, raw content. - Prefix(&'a [u8]), - /// `ZSTD_CCtx_refCDict`, raw content, attached. With the parameters of the CDict. - CDict(&'a [u8], [u32; 7]), -} - -/// What is left in the match state after the frame. -#[derive(PartialEq, Eq, Debug, Default)] -pub struct State { - pub has_dms: bool, - pub next_to_update: u32, - pub loaded_dict_end: u32, - pub low_limit: u32, - pub dict_limit: u32, - pub hash_salt_entropy: u32, - pub lazy_skipping: bool, - pub hash_cache: [u32; 8], - pub rep: [u32; 3], - pub hash_table: Vec, - pub chain_table: Vec, - pub tag_table: Vec, - pub hash_table3: Vec, - pub sums: [u32; 8], - pub freqs: [Vec; 4], -} - -fn params(cp: [u32; 7]) -> Result { - Ok(CompressionParameters { - window_log: cp[0], chain_log: cp[1], hash_log: cp[2], search_log: cp[3], min_match: cp[4], target_length: cp[5], - strategy: Some(Strategy::from_int(cp[6] as i32).ok_or(ErrorCode::Generic)?), - }) -} - -/// `ZSTD_reset_matchState`, on a new one. -fn reset_match_state(c_params: CompressionParameters, use_row: bool, for_cdict: bool) -> MatchState { - let strategy = c_params.strat(); - let row_used = row_match_finder_supported(strategy) && use_row; - let mut ms = MatchState::new(); - ms.hash_log3 = if !for_cdict && c_params.min_match == 3 { c_params.window_log.min(17) } else { 0 }; - ms.window.clear(); - ms.next_to_update = ms.window.dict_limit; - ms.hash_table = vec![0; 1 << c_params.hash_log]; - ms.chain_table = vec![0; if strategy != Strategy::Fast && !row_used { 1 << c_params.chain_log } else { 0 }]; - ms.hash_table3 = vec![0; if ms.hash_log3 != 0 { 1 << ms.hash_log3 } else { 0 }]; - if row_used { - ms.tag_table = vec![0; 1 << c_params.hash_log]; - ms.hash_salt = if for_cdict { 0 } else { bitmix(ms.hash_salt, 8) ^ bitmix(ms.hash_salt_entropy as u64, 4) }; - ms.row_hash_log = c_params.hash_log - c_params.search_log.clamp(4, 6); - } - if !for_cdict && strategy >= Strategy::BtOpt { - ms.opt.lit_freq = vec![0; 256]; - ms.opt.lit_length_freq = vec![0; 36]; - ms.opt.match_length_freq = vec![0; 53]; - ms.opt.off_code_freq = vec![0; 32]; - ms.opt.match_table = vec![Match::default(); ZSTD_OPT_SIZE]; - ms.opt.price_table = vec![Optimal::default(); ZSTD_OPT_SIZE]; - } - ms.c_params = c_params; - ms -} - -/// `ZSTD_loadDictionaryContent` -fn load_dictionary_content(ms: &mut MatchState, use_row: bool, dict: &[u8], dtlm: DictTableLoadMethod, tfp: TableFillPurpose) -> Result<(), ErrorCode> { - let c_params = ms.c_params; - let iend = dict.len(); - ms.window.update(BufId::Dict.start(), BufId::Dict, dict.len(), false); - let max_dict_size = 1usize << (c_params.hash_log + 3).max(c_params.chain_log + 1).min(31); - let ip = iend - dict.len().min(max_dict_size); - let prefix = Prefix { buf: dict, base: ms.window.base.wrapping_sub(BufId::Dict.start()) }; - ms.next_to_update = prefix.index_of(ip); - ms.loaded_dict_end = prefix.index_of(iend); - if iend - ip > HASH_READ_SIZE { - match c_params.strat() { - Strategy::Fast => fast::fill_hash_table(ms, prefix, iend, dtlm, tfp)?, - Strategy::DFast => double_fast::fill_double_hash_table(ms, prefix, iend, dtlm, tfp)?, - Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { - if use_row { - ms.tag_table.fill(0); - lazy::row_update(ms, prefix, iend - HASH_READ_SIZE)?; - } else { - lazy::insert_and_find_first_index(ms, prefix, iend - HASH_READ_SIZE)?; - } - } - _ => opt::update_tree(ms, prefix, iend - HASH_READ_SIZE, iend)?, - } - ms.next_to_update = prefix.index_of(iend); - } - Ok(()) -} - -/// What the public parameters of the C library cannot set. -#[derive(Clone, Copy, Default)] -pub struct Extra<'a> { - /// `literalCompressionMode`: 0 auto, 1 enable, 2 disable. - pub lit_mode: u32, - /// (offset, literal length, match length), as from long distance matching. - pub ldm: &'a [(u32, u32, u32)], - /// The Huffman table, and `deltaNbBits` for the codes of literal lengths, match lengths and offsets. - pub entropy: Option<(&'a [u64; 257], &'a [u32; 36], &'a [u32; 53], &'a [u32; 32])>, - /// Input that comes before `src`, from another buffer. - pub first_chunk: &'a [u8], -} - -/// `ZSTD_ldm_skipRawSeqStoreBytes` -fn skip_raw_seq_store_bytes(store: &mut RawSeqStore, nb_bytes: usize) { - let mut curr_pos = (store.pos_in_sequence + nb_bytes) as u32; - while curr_pos != 0 && store.pos < store.size { - let seq = store.seq[store.pos]; - if curr_pos >= seq.lit_length.wrapping_add(seq.match_length) { - curr_pos -= seq.lit_length.wrapping_add(seq.match_length); - store.pos += 1; - } else { - store.pos_in_sequence = curr_pos as usize; - break; - } - } - if curr_pos == 0 || store.pos == store.size { - store.pos_in_sequence = 0; - } -} - -pub fn frame(cp: [u32; 7], use_row: bool, dict: Dict<'_>, src: &[u8]) -> Result<(Vec<(u32, u32, u32, u32)>, State), ErrorCode> { - frame2(cp, use_row, dict, src, Extra::default()) -} - -/// The sequences of all the blocks of a frame, as (literal length, match length, repeat code or 0, offset or 0). Each -/// block ends with the literals that are left as (length, 0, 0, 0). -/// -/// `cp` is windowLog, chainLog, hashLog, searchLog, minMatch, targetLength, strategy, as applied. -pub fn frame2(cp: [u32; 7], use_row: bool, dict: Dict<'_>, src: &[u8], extra: Extra<'_>) -> Result<(Vec<(u32, u32, u32, u32)>, State), ErrorCode> { - let c_params = params(cp)?; - let strategy = c_params.strat(); - let mut ms = reset_match_state(c_params, use_row, false); - - let block_size_max = (1usize << c_params.window_log).min(128 << 10); - let mut seq_store = SeqStore::new(); - seq_store.reserve(block_size_max / if c_params.min_match == 3 { 3 } else { 4 }, block_size_max)?; - - let dict_content = match dict { Dict::None => &[][..], Dict::Prefix(d) | Dict::CDict(d, _) => d }; - let bufs = Buffers { in_buff: extra.first_chunk, user: src, dict: dict_content }; - - let mut cdict_ms = None; - match dict { - Dict::Prefix(d) if d.len() >= 8 => load_dictionary_content(&mut ms, use_row, d, DictTableLoadMethod::Fast, TableFillPurpose::ForCCtx)?, - Dict::CDict(d, cdict_cp) if d.len() >= 8 => { - let mut c = reset_match_state(params(cdict_cp)?, use_row, true); - load_dictionary_content(&mut c, use_row, d, DictTableLoadMethod::Full, TableFillPurpose::ForCDict)?; - // ZSTD_resetCCtx_byAttachingCDict - let cdict_end = c.window.index_of(c.window.next_src); - if ms.window.dict_limit < cdict_end { - ms.window.next_src = ms.window.base + cdict_end as usize; - ms.window.clear(); - } - ms.loaded_dict_end = ms.window.dict_limit; - cdict_ms = Some(c); - } - _ => {} - } - let mut dms = cdict_ms.as_ref().map(|c| DictMatchState { - ms: c, - buf: dict_content, - base: c.window.base.wrapping_sub(BufId::Dict.start()), - end: c.window.next_src - BufId::Dict.start(), - }); - - let max_dist = 1u32 << c_params.window_log; - let mut prev = CompressedBlockState::new(); - if let Some((huf, ll, ml, of)) = extra.entropy { - prev.entropy.huf.c_table.0 = *huf; - prev.entropy.huf.repeat_mode = crate::compress::huf::HufRepeat::Valid; - for (to, from) in prev.entropy.fse.litlength_c_table.symbol_tt.iter_mut().zip(ll) { to.delta_nb_bits = *from; } - for (to, from) in prev.entropy.fse.matchlength_c_table.symbol_tt.iter_mut().zip(ml) { to.delta_nb_bits = *from; } - for (to, from) in prev.entropy.fse.offcode_c_table.symbol_tt.iter_mut().zip(of) { to.delta_nb_bits = *from; } - } - ms.opt.literal_compression_mode = match extra.lit_mode { 0 => ParamSwitch::Auto, 1 => ParamSwitch::Enable, _ => ParamSwitch::Disable }; - let mut extern_seq_store = RawSeqStore { - seq: extra.ldm.iter().map(|&(offset, lit_length, match_length)| RawSeq { offset, lit_length, match_length }).collect(), - pos: 0, pos_in_sequence: 0, size: extra.ldm.len(), capacity: extra.ldm.len(), - }; - let mut rep = prev.rep; - let mut out = Vec::new(); - for (src, src_buf) in [(extra.first_chunk, BufId::InBuff), (src, BufId::User)] { - // ZSTD_compressContinue_internal - let src_addr = src_buf.start(); - if !ms.window.update(src_addr, src_buf, src.len(), ms.force_non_contiguous) { - ms.force_non_contiguous = false; - ms.next_to_update = ms.window.dict_limit; - } - // ZSTD_compress_frameChunk - let mut pos = 0; - while pos < src.len() { - let block_size = (src.len() - pos).min(block_size_max); - if ms.window.dict_is_invalid(src_addr + pos + block_size, max_dist, ms.loaded_dict_end) { - ms.loaded_dict_end = 0; - dms = None; - } - if ms.window.enforce_max_dist(src_addr + pos, max_dist, ms.loaded_dict_end) { - ms.loaded_dict_end = 0; - dms = None; - } - if ms.next_to_update < ms.window.low_limit { - ms.next_to_update = ms.window.low_limit; - } - // ZSTD_buildSeqStore - assert!(block_size >= 7); - seq_store.reset(); - let block = BlockSrc { - buf: bufs.get(ms.window.base_buf), - base: ms.window.base.wrapping_sub(ms.window.base_buf.start()), - dict_buf: bufs.get(ms.window.dict_buf), - dict_base: ms.window.dict_base.wrapping_sub(ms.window.dict_buf.start()), - start: pos, - end: pos + block_size, - dms, - symbol_costs: &prev.entropy, - ldm_seq_store: if extern_seq_store.pos < extern_seq_store.size { Some(&extern_seq_store) } else { None }, - }; - let curr = block.index_of(pos); - if curr > ms.next_to_update + 384 { - ms.next_to_update = curr - 192.min(curr - ms.next_to_update - 384); - } - let dict_mode = ms.dict_mode(dms.as_ref()); - let mut reps = rep; - let last_ll = compress::compress_block(&mut ms, &mut seq_store, &mut rep, &block, strategy, use_row, dict_mode)?; - if extern_seq_store.pos < extern_seq_store.size { - skip_raw_seq_store_bytes(&mut extern_seq_store, block_size); - } - - let mut total = last_ll; - for (i, seq) in seq_store.sequences.iter().enumerate() { - let (mut ll, mut ml) = (seq.lit_length as u32, seq.ml_base as u32 + 3); - if seq_store.long_length_pos as usize == i { - match seq_store.long_length_type { - LongLengthType::LiteralLength => ll += 0x10000, - LongLengthType::MatchLength => ml += 0x10000, - LongLengthType::None => {} - } - } - update_rep(&mut reps, seq.off_base, ll == 0); - total += (ll + ml) as usize; - out.push(if seq.off_base <= 3 { (ll, ml, seq.off_base, 0) } else { (ll, ml, 0, seq.off_base - 3) }); - } - out.push((last_ll as u32, 0, 0, 0)); - assert_eq!(total, block_size, "the sequences cover the block"); - assert_eq!(seq_store.lit_len + last_ll + seq_store.sequences.iter().enumerate().map(|(i, s)| s.ml_base as usize + 3 - + if seq_store.long_length_pos as usize == i && seq_store.long_length_type == LongLengthType::MatchLength { 0x10000 } else { 0 }).sum::(), block_size); - assert_eq!(reps[..2], rep[..2], "the repeat offsets the block compressor leaves are those of its sequences"); - pos += block_size; - } - } - let state = State { - has_dms: dms.is_some(), - next_to_update: ms.next_to_update, - loaded_dict_end: ms.loaded_dict_end, - low_limit: ms.window.low_limit, - dict_limit: ms.window.dict_limit, - hash_salt_entropy: ms.hash_salt_entropy, - lazy_skipping: ms.lazy_skipping, - hash_cache: ms.hash_cache, - rep, - hash_table: ms.hash_table, - chain_table: ms.chain_table, - tag_table: ms.tag_table, - hash_table3: ms.hash_table3, - sums: [ms.opt.lit_sum, ms.opt.lit_length_sum, ms.opt.match_length_sum, ms.opt.off_code_sum, ms.opt.lit_sum_base_price, - ms.opt.lit_length_sum_base_price, ms.opt.match_length_sum_base_price, ms.opt.off_code_sum_base_price], - freqs: [ms.opt.lit_freq, ms.opt.lit_length_freq, ms.opt.match_length_freq, ms.opt.off_code_freq], - }; - Ok((out, state)) -} From 45523f475a9086c495727f7d2e4a9573acfd1383 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 13:42:40 +0000 Subject: [PATCH 010/199] zstd: zero output once, feed workers all of a write, drop what nothing calls - VecOutput: output at the end of a Vec that zeroes about as much as gets written. StreamingDecoder and the CompressionStream coder zeroed the whole spare capacity before every call, which is quadratic in the number of frames. - node:zlib: with workers a call returns after any progress, so call again until the input or the output is used up. Streams with nbWorkers lost data. - The size that node:zlib reports to the GC comes from the types. - A job's input buffer grows with the input instead of being reserved whole. - Remove CCtx::reset, DCtx::reset, parameter updates in the middle of a frame and unused exports; share duplicated constants and helpers. - Tests: a trained dictionary, a stream with workers, one shared sample. --- src/runtime/node/zlib/NativeZstd.rs | 17 +- src/runtime/webcore/CompressionStreamCoder.rs | 50 +-- src/zstd/common/mod.rs | 5 +- src/zstd/compress/ldm.rs | 9 +- src/zstd/compress/mt.rs | 71 ++--- src/zstd/compress/params.rs | 74 ++--- src/zstd/compress/stream.rs | 71 ++--- src/zstd/decompress/mod.rs | 51 +--- src/zstd/lib.rs | 91 ++++-- test/js/bun/util/zstd.test.ts | 288 ++++++++++++------ .../node/zlib/zlib-estimated-size-gc.test.ts | 6 +- 11 files changed, 382 insertions(+), 351 deletions(-) diff --git a/src/runtime/node/zlib/NativeZstd.rs b/src/runtime/node/zlib/NativeZstd.rs index 24cb287685d0..2c69e3630e33 100644 --- a/src/runtime/node/zlib/NativeZstd.rs +++ b/src/runtime/node/zlib/NativeZstd.rs @@ -92,9 +92,8 @@ mod _impl { /// Per-mode external-allocation footprint, fixed at construction. fn external_size_for(mode: NodeMode) -> usize { match mode { - // About what a new context takes, with the tables it allocates at once. - NodeMode::ZSTD_COMPRESS => 14 * 1024, - NodeMode::ZSTD_DECOMPRESS => 28 * 1024, + NodeMode::ZSTD_COMPRESS => CCtx::SIZE, + NodeMode::ZSTD_DECOMPRESS => DCtx::SIZE, _ => 0, } } @@ -398,7 +397,17 @@ mod _impl { }; let result = match state { State::Compress(state) => { - state.compress_stream2(&mut output, &mut input, self.flush) + let mut ret = state.compress_stream2(&mut output, &mut input, self.flush); + // With workers, a call returns after any progress. `zlib.ts` takes room left in + // the output to mean that all the input is consumed. + while ret.is_ok() + && self.flush == EndDirective::Continue + && input.pos < input.src.len() + && output.pos < output.dst.len() + { + ret = state.compress_stream2(&mut output, &mut input, self.flush); + } + ret } State::Decompress(state) => { let mut ret = state.decompress_stream(&mut output, &mut input); diff --git a/src/runtime/webcore/CompressionStreamCoder.rs b/src/runtime/webcore/CompressionStreamCoder.rs index 560feb5e9af5..f64093107c0c 100644 --- a/src/runtime/webcore/CompressionStreamCoder.rs +++ b/src/runtime/webcore/CompressionStreamCoder.rs @@ -74,15 +74,6 @@ fn spare(out: &mut Vec, cap: usize) -> Result<&mut [core::mem::MaybeUninit, cap: usize) -> Result { - let start = out.len(); - let len = spare(out, cap)?.len(); - out.resize(start + len, 0); - Ok(start) -} - fn zstd_error(code: zstd::ErrorCode, message: &'static str) -> CodecError { if code == zstd::ErrorCode::MemoryAllocation { CodecError::OutOfMemory @@ -257,7 +248,7 @@ impl CompressionStreamCoder { (Format::Zstd, false) => { let mut cctx = Box::new(zstd::CCtx::new()); if let Some(lvl) = level - && (cctx.set_parameter(zstd::c_param::COMPRESSION_LEVEL, lvl)).is_err() + && (cctx.set_parameter(zstd::ZSTD_C_COMPRESSION_LEVEL, lvl)).is_err() { return Err(CodecError::Message("failed to set zstd level")); } @@ -594,24 +585,20 @@ impl CompressionStreamCoder { } else { zstd::EndDirective::Continue }; - let mut input_buf = zstd::InBuffer { - src: input, - pos: 0, - }; + let mut input_buf = zstd::InBuffer { src: input, pos: 0 }; + let mut out = zstd::VecOutput::new(out); loop { - if out.len() >= cap { + if out.filled() >= cap { return Ok(Progress::More { consumed: input_buf.pos, }); } - let start = zeroed_spare(out, cap)?; - let mut output_buf = zstd::OutBuffer { - dst: &mut out[start..], - pos: 0, - }; + let mut output_buf = out + .buffer(CHUNK, cap) + .map_err(|_| CodecError::OutOfMemory)?; let remaining = cctx.compress_stream2(&mut output_buf, &mut input_buf, end); let written = output_buf.pos; - out.truncate(start + written); + out.advance(written); let remaining = remaining.map_err(|code| zstd_error(code, "zstd encode failed"))?; if input_buf.pos == input.len() && (!finish || remaining == 0) { @@ -622,10 +609,8 @@ impl CompressionStreamCoder { Backend::ZstdDecode(dctx) => { // After a frame completes, whatever follows must be another frame // magic (see `zstd_head`); `step` has already re-attached a split one. - let mut input_buf = zstd::InBuffer { - src: input, - pos: 0, - }; + let mut input_buf = zstd::InBuffer { src: input, pos: 0 }; + let mut out = zstd::VecOutput::new(out); loop { if self.ended { let rest = &input[input_buf.pos..]; @@ -644,23 +629,20 @@ impl CompressionStreamCoder { self.zstd_head_len = head.len() as u8; return Ok(Progress::Done); } - // Between frames, this cannot fail. - let _ = dctx.reset(zstd::ResetDirective::SessionAndParameters); + dctx.init_stream(); self.ended = false; } - if out.len() >= cap { + if out.filled() >= cap { return Ok(Progress::More { consumed: input_buf.pos, }); } - let start = zeroed_spare(out, cap)?; - let mut output_buf = zstd::OutBuffer { - dst: &mut out[start..], - pos: 0, - }; + let mut output_buf = out + .buffer(CHUNK, cap) + .map_err(|_| CodecError::OutOfMemory)?; let remaining = dctx.decompress_stream(&mut output_buf, &mut input_buf); let (written, room) = (output_buf.pos, output_buf.dst.len()); - out.truncate(start + written); + out.advance(written); let remaining = remaining.map_err(|code| zstd_error(code, "zstd decode failed"))?; if remaining == 0 { diff --git a/src/zstd/common/mod.rs b/src/zstd/common/mod.rs index 7637ec7546b5..75a19054e2a4 100644 --- a/src/zstd/common/mod.rs +++ b/src/zstd/common/mod.rs @@ -18,7 +18,7 @@ pub(crate) const ZSTD_WINDOWLOG_MAX: u32 = 31; pub(crate) const ZSTD_WINDOWLOG_ABSOLUTEMIN: u32 = 10; pub(crate) const ZSTD_WINDOWLOG_LIMIT_DEFAULT: u32 = 27; -pub const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX; +pub(crate) const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX; pub const ZSTD_CONTENTSIZE_ERROR: u64 = u64::MAX - 1; pub(crate) const ZSTD_REP_NUM: usize = 3; @@ -31,6 +31,9 @@ pub(crate) const ZSTD_FRAMEIDSIZE: usize = 4; pub(crate) const ZSTD_BLOCKHEADERSIZE: usize = 3; pub(crate) const ZSTD_FRAMEHEADERSIZE_MAX: usize = 18; +pub(crate) const ZSTD_WORKSPACETOOLARGE_FACTOR: usize = 3; +pub(crate) const ZSTD_WORKSPACETOOLARGE_MAXDURATION: usize = 128; + /// `blockType_e` #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub(crate) enum BlockType { diff --git a/src/zstd/compress/ldm.rs b/src/zstd/compress/ldm.rs index 3e11201479e9..d4e253d22965 100644 --- a/src/zstd/compress/ldm.rs +++ b/src/zstd/compress/ldm.rs @@ -5,16 +5,13 @@ use super::internal::{ LdmParams, LdmState, MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, TableFillPurpose, count, count_2segments, offset_to_offbase, }; +use super::params::{ZSTD_HASHLOG_MAX, ZSTD_HASHLOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX}; use super::{compress_block, double_fast, fast}; use crate::common::ZSTD_REP_NUM; use crate::common::xxhash::Xxh64; use crate::error::{ErrorCode, Result}; use crate::util::{BUG, SliceExt}; -const ZSTD_HASHLOG_MIN: u32 = 6; -const ZSTD_HASHLOG_MAX: u32 = 30; -const ZSTD_LDM_BUCKETSIZELOG_MAX: u32 = 8; - const LDM_BUCKET_SIZE_LOG: u32 = 4; const LDM_MIN_MATCH_LENGTH: u32 = 64; const LDM_BATCH_SIZE: usize = 64; @@ -177,7 +174,7 @@ pub(crate) fn adjust_parameters(params: &mut LdmParams, c_params: &CompressionPa params.hash_log = params .window_log .wrapping_sub(params.hash_rate_log) - .clamp(ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + .clamp(ZSTD_HASHLOG_MIN as u32, ZSTD_HASHLOG_MAX as u32); } if params.min_match_length == 0 { params.min_match_length = LDM_MIN_MATCH_LENGTH; @@ -187,7 +184,7 @@ pub(crate) fn adjust_parameters(params: &mut LdmParams, c_params: &CompressionPa } if params.bucket_size_log == 0 { params.bucket_size_log = - (strategy as u32).clamp(LDM_BUCKET_SIZE_LOG, ZSTD_LDM_BUCKETSIZELOG_MAX); + (strategy as u32).clamp(LDM_BUCKET_SIZE_LOG, ZSTD_LDM_BUCKETSIZELOG_MAX as u32); } params.bucket_size_log = params.bucket_size_log.min(params.hash_log); } diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 66e96566dc27..352b25aaff99 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -22,14 +22,14 @@ use super::internal::{ }; use super::ldm; use super::params::{ - CCtxParams, CParamMode, ZSTDMT_JOBLOG_MAX, ZSTDMT_JOBSIZE_MAX, ZSTDMT_JOBSIZE_MIN, - ZSTDMT_NBWORKERS_MAX, compress_bound, cycle_log, get_c_params_from_cctx_params, + CCtxParams, NO_LDM_PARAMS, ZSTDMT_JOBLOG_MAX, ZSTDMT_JOBSIZE_MAX, ZSTDMT_JOBSIZE_MIN, + ZSTDMT_NBWORKERS_MAX, compress_bound, cycle_log, }; use super::stream::EndDirective; +use crate::common::ZSTD_BLOCKSIZE_MAX; use crate::common::xxhash::Xxh64; -use crate::common::{ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_UNKNOWN}; use crate::error::{ErrorCode, Result}; -use crate::util::SliceExt; +use crate::util::{SliceExt, try_copy}; use crate::{InBuffer, OutBuffer}; /// `SerialState`: what has to see the input in order. @@ -119,15 +119,6 @@ impl SerialState { } } -const NO_LDM_PARAMS: LdmParams = LdmParams { - enable_ldm: ParamSwitch::Auto, - hash_log: 0, - bucket_size_log: 0, - min_match_length: 0, - hash_rate_log: 0, - window_log: 0, -}; - /// What a job tells about itself. enum Progress { /// Some more output, and how much of the input is consumed by now. @@ -180,7 +171,7 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - capacity: job.max_nb_seq, }; let mut serial = job.serial_in.recv().map_err(|_| ErrorCode::Generic)?; - let generated = if cancelled.load(Ordering::Relaxed) { + let generated = if cancelled.load(Ordering::SeqCst) { Err(ErrorCode::Generic) } else { reset_struct_table(&mut raw_seq_store.seq, job.max_nb_seq, false) @@ -258,15 +249,12 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - let nb_chunks = src_size.div_ceil(CHUNK_SIZE); let mut ip = job.prefix_size; let send = |dst: &[u8], consumed: usize| -> Result<()> { - let mut data = Vec::new(); - data.try_reserve_exact(dst.len()) - .map_err(|_| ErrorCode::MemoryAllocation)?; - data.extend_from_slice(dst); + let data = try_copy(dst)?; let _ = job.progress.send(Progress::Output { data, consumed }); Ok(()) }; for chunk_nb in 1..nb_chunks { - if cancelled.load(Ordering::Relaxed) { + if cancelled.load(Ordering::SeqCst) { return Err(ErrorCode::Generic); } let chunk = Span { @@ -493,7 +481,7 @@ impl MtCtx { /// `ZSTDMT_waitForAllJobsCompleted` and `ZSTDMT_releaseAllJobResources`, for a frame that is given up. fn abandon_jobs(&mut self) { - self.cancelled.store(true, Ordering::Relaxed); + self.cancelled.store(true, Ordering::SeqCst); for job in &mut self.jobs { while !job.is_complete() { job.to_flush = 0; @@ -501,7 +489,7 @@ impl MtCtx { job.poll(true); } } - self.cancelled.store(false, Ordering::Relaxed); + self.cancelled.store(false, Ordering::SeqCst); self.jobs.clear(); self.in_buff = Vec::new(); self.in_buff_prefix_size = 0; @@ -516,21 +504,6 @@ impl MtCtx { .unwrap_or_else(|| Box::new(SerialState::new())) } - /// `ZSTDMT_updateCParams_whileCompressing`: for the jobs to come. - pub(crate) fn update_c_params_while_compressing(&mut self, cctx_params: &CCtxParams) { - // The window is in the frame header. - let saved_wlog = self.params.c_params.window_log; - self.params.compression_level = cctx_params.compression_level; - let mut c_params = get_c_params_from_cctx_params( - cctx_params, - ZSTD_CONTENTSIZE_UNKNOWN, - 0, - CParamMode::NoAttachDict, - ); - c_params.window_log = saved_wlog; - self.params.c_params = c_params; - } - /// `ZSTDMT_initCStream_internal` pub(crate) fn init_c_stream_internal( &mut self, @@ -614,11 +587,7 @@ impl MtCtx { self.in_buff_prefix_size = 0; if !end_frame { let new_prefix_size = src_size.min(self.target_prefix_size); - self.in_buff - .try_reserve_exact(new_prefix_size + self.target_section_size) - .map_err(|_| ErrorCode::MemoryAllocation)?; - self.in_buff - .extend_from_slice(data.sub(data.len() - new_prefix_size..)?); + self.in_buff = try_copy(data.sub(data.len() - new_prefix_size..)?)?; self.in_buff_prefix_size = new_prefix_size; } else { self.frame_ended = true; @@ -766,13 +735,24 @@ impl MtCtx { // The C code moves the prefix to the start of the round buffer. self.round_buff_pos = self.in_buff_prefix_size; } - self.in_buff - .try_reserve_exact(self.target_section_size) - .map_err(|_| ErrorCode::MemoryAllocation)?; self.in_buff_ready = true; Ok(true) } + /// Room for `more` input. A section that is flushed early is not to cost as much as a whole one. + fn reserve_input(&mut self, more: usize) -> Result<()> { + let needed = self.in_buff.len() + more; + if needed <= self.in_buff.capacity() { + return Ok(()); + } + let capacity = needed + .max(self.in_buff.capacity().saturating_mul(2)) + .min(self.in_buff_prefix_size + self.target_section_size); + self.in_buff + .try_reserve_exact(capacity - self.in_buff.len()) + .map_err(|_| ErrorCode::MemoryAllocation) + } + /// `ZSTDMT_compressStream_generic` pub(crate) fn compress_stream_generic( &mut self, @@ -790,6 +770,7 @@ impl MtCtx { if input.src.len() > input.pos && (self.in_buff_ready || self.try_get_input_range()?) { let filled = self.in_buff.len() - self.in_buff_prefix_size; let to_load = (input.src.len() - input.pos).min(self.target_section_size - filled); + self.reserve_input(to_load)?; self.in_buff .extend_from_slice(input.src.span(input.pos, to_load)?); input.pos += to_load; @@ -820,7 +801,7 @@ impl MtCtx { impl Drop for MtCtx { fn drop(&mut self) { - self.cancelled.store(true, Ordering::Relaxed); + self.cancelled.store(true, Ordering::SeqCst); for worker in self.workers.drain(..) { drop(worker.jobs); let _ = worker.thread.join(); diff --git a/src/zstd/compress/params.rs b/src/zstd/compress/params.rs index fd1d2b8d4af4..2be5372af0a0 100644 --- a/src/zstd/compress/params.rs +++ b/src/zstd/compress/params.rs @@ -9,18 +9,18 @@ use super::{ldm, row_match_finder_supported}; use crate::common::{ LITBITS, MAX_LL, MAX_ML, MAX_OFF, WILDCOPY_OVERLENGTH, ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WINDOWLOG_LIMIT_DEFAULT, - ZSTD_WINDOWLOG_MAX, + ZSTD_WINDOWLOG_MAX, ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, }; use crate::error::{ErrorCode, Result}; use crate::util::highbit32; pub const ZSTD_CLEVEL_DEFAULT: i32 = 3; -pub const ZSTD_MAX_CLEVEL: i32 = 22; -pub const ZSTD_MIN_CLEVEL: i32 = -(ZSTD_TARGETLENGTH_MAX); +const ZSTD_MAX_CLEVEL: i32 = 22; +const ZSTD_MIN_CLEVEL: i32 = -(ZSTD_TARGETLENGTH_MAX); const ZSTD_WINDOWLOG_MIN: i32 = 10; -const ZSTD_HASHLOG_MAX: i32 = 30; -const ZSTD_HASHLOG_MIN: i32 = 6; +pub(crate) const ZSTD_HASHLOG_MAX: i32 = 30; +pub(crate) const ZSTD_HASHLOG_MIN: i32 = 6; const ZSTD_CHAINLOG_MAX: i32 = 30; const ZSTD_CHAINLOG_MIN: i32 = ZSTD_HASHLOG_MIN; const ZSTD_SEARCHLOG_MAX: i32 = ZSTD_WINDOWLOG_MAX as i32 - 1; @@ -36,7 +36,7 @@ const ZSTD_LDM_HASHLOG_MAX: i32 = ZSTD_HASHLOG_MAX; const ZSTD_LDM_MINMATCH_MIN: i32 = 4; const ZSTD_LDM_MINMATCH_MAX: i32 = 4096; const ZSTD_LDM_BUCKETSIZELOG_MIN: i32 = 1; -const ZSTD_LDM_BUCKETSIZELOG_MAX: i32 = 8; +pub(crate) const ZSTD_LDM_BUCKETSIZELOG_MAX: i32 = 8; const ZSTD_LDM_HASHRATELOG_MIN: i32 = 0; const ZSTD_LDM_HASHRATELOG_MAX: i32 = ZSTD_WINDOWLOG_MAX as i32 - ZSTD_HASHLOG_MIN; const ZSTD_TARGETCBLOCKSIZE_MIN: i32 = 1340; @@ -50,28 +50,28 @@ pub(crate) const ZSTDMT_JOBLOG_MAX: u32 = 30; pub(crate) const ZSTDMT_JOBSIZE_MAX: i32 = 1024 << 20; /// `ZSTD_cParameter`, as far as `node:zlib` lets a parameter through. -pub mod c_param { - pub const FORMAT: i32 = 10; +pub(crate) mod c_param { + pub(crate) const FORMAT: i32 = 10; pub const COMPRESSION_LEVEL: i32 = 100; - pub const WINDOW_LOG: i32 = 101; - pub const HASH_LOG: i32 = 102; - pub const CHAIN_LOG: i32 = 103; - pub const SEARCH_LOG: i32 = 104; - pub const MIN_MATCH: i32 = 105; - pub const TARGET_LENGTH: i32 = 106; - pub const STRATEGY: i32 = 107; - pub const TARGET_C_BLOCK_SIZE: i32 = 130; - pub const ENABLE_LONG_DISTANCE_MATCHING: i32 = 160; - pub const LDM_HASH_LOG: i32 = 161; - pub const LDM_MIN_MATCH: i32 = 162; - pub const LDM_BUCKET_SIZE_LOG: i32 = 163; - pub const LDM_HASH_RATE_LOG: i32 = 164; - pub const CONTENT_SIZE_FLAG: i32 = 200; - pub const CHECKSUM_FLAG: i32 = 201; - pub const DICT_ID_FLAG: i32 = 202; - pub const NB_WORKERS: i32 = 400; - pub const JOB_SIZE: i32 = 401; - pub const OVERLAP_LOG: i32 = 402; + pub(crate) const WINDOW_LOG: i32 = 101; + pub(crate) const HASH_LOG: i32 = 102; + pub(crate) const CHAIN_LOG: i32 = 103; + pub(crate) const SEARCH_LOG: i32 = 104; + pub(crate) const MIN_MATCH: i32 = 105; + pub(crate) const TARGET_LENGTH: i32 = 106; + pub(crate) const STRATEGY: i32 = 107; + pub(crate) const TARGET_C_BLOCK_SIZE: i32 = 130; + pub(crate) const ENABLE_LONG_DISTANCE_MATCHING: i32 = 160; + pub(crate) const LDM_HASH_LOG: i32 = 161; + pub(crate) const LDM_MIN_MATCH: i32 = 162; + pub(crate) const LDM_BUCKET_SIZE_LOG: i32 = 163; + pub(crate) const LDM_HASH_RATE_LOG: i32 = 164; + pub(crate) const CONTENT_SIZE_FLAG: i32 = 200; + pub(crate) const CHECKSUM_FLAG: i32 = 201; + pub(crate) const DICT_ID_FLAG: i32 = 202; + pub(crate) const NB_WORKERS: i32 = 400; + pub(crate) const JOB_SIZE: i32 = 401; + pub(crate) const OVERLAP_LOG: i32 = 402; } /// `ZSTD_format_e` @@ -134,7 +134,7 @@ const NO_C_PARAMS: CompressionParameters = CompressionParameters { strategy: None, }; -const NO_LDM_PARAMS: LdmParams = LdmParams { +pub(crate) const NO_LDM_PARAMS: LdmParams = LdmParams { enable_ldm: ParamSwitch::Auto, hash_log: 0, bucket_size_log: 0, @@ -194,20 +194,6 @@ fn bound_check_unless_0(param: i32, value: i32) -> Result { Ok(value as u32) } -/// `ZSTD_isUpdateAuthorized`: whether the parameter can change while a frame is being written. -pub(crate) fn is_update_authorized(param: i32) -> bool { - matches!( - param, - c_param::COMPRESSION_LEVEL - | c_param::HASH_LOG - | c_param::CHAIN_LOG - | c_param::SEARCH_LOG - | c_param::MIN_MATCH - | c_param::TARGET_LENGTH - | c_param::STRATEGY - ) -} - impl CCtxParams { /// `ZSTD_CCtxParams_init` pub(crate) const fn new(compression_level: i32) -> Self { @@ -648,8 +634,6 @@ const SIZEOF_OPTIMAL: usize = 28; const SIZEOF_RAW_SEQ: usize = 12; const SIZEOF_LDM_ENTRY: usize = 8; const ZSTD_CWKSP_ALIGNMENT_BYTES: usize = 64; -const ZSTD_WORKSPACETOOLARGE_FACTOR: usize = 3; -const ZSTD_WORKSPACETOOLARGE_MAXDURATION: i32 = 128; /// `ZSTD_cwksp_aligned64_alloc_size` const fn aligned64(size: usize) -> usize { @@ -699,7 +683,7 @@ pub(crate) struct Workspace { object_end: usize, table_end: usize, alloc_start: usize, - oversized_duration: i32, + oversized_duration: usize, } /// What a context needs room for. diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs index f5bfecb8904d..5cfc01d8240d 100644 --- a/src/zstd/compress/stream.rs +++ b/src/zstd/compress/stream.rs @@ -3,16 +3,16 @@ use std::sync::Arc; use super::cctx::{CDict, DictContentType, FrameCompressor, Input, Span}; -use super::internal::{BufId, Buffers, DictTableLoadMethod}; +use super::internal::{BufId, Buffers, CompressedBlockState, DictTableLoadMethod}; use super::mt::MtCtx; use super::params::{ CCtxParams, CParamMode, ZSTD_CLEVEL_DEFAULT, ZSTDMT_JOBSIZE_MIN, compress_bound, - get_c_params_from_cctx_params, get_params_internal, is_update_authorized, - resolve_block_splitter_mode, resolve_enable_ldm, resolve_row_match_finder_mode, + get_c_params_from_cctx_params, get_params_internal, resolve_block_splitter_mode, + resolve_enable_ldm, resolve_row_match_finder_mode, }; use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, try_zeroed}; -use crate::{InBuffer, OutBuffer, ResetDirective}; +use crate::util::{SliceExt, try_copy, try_zeroed}; +use crate::{InBuffer, OutBuffer}; /// `ZSTD_EndDirective` #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -36,7 +36,6 @@ enum StreamStage { /// `ZSTD_CCtx` pub struct CCtx { frame: FrameCompressor, - c_params_changed: bool, requested_params: CCtxParams, in_buff: Vec, in_to_compress: usize, @@ -63,8 +62,8 @@ impl Default for CCtx { /// `ZSTD_limitCopy` fn limit_copy(dst: &mut [u8], src: &[u8]) -> usize { let length = dst.len().min(src.len()); - for (to, from) in dst.iter_mut().zip(src) { - *to = *from; + if let (Some(dst), Some(src)) = (dst.get_mut(..length), src.get(..length)) { + dst.copy_from_slice(src); } length } @@ -78,11 +77,13 @@ fn resize_buffer(buffer: &mut Vec, len: usize) -> Result<()> { } impl CCtx { + /// How much memory a new one takes. + pub const SIZE: usize = size_of::() + 2 * size_of::(); + /// `ZSTD_createCCtx` pub fn new() -> Self { Self { frame: FrameCompressor::new(), - c_params_changed: false, requested_params: CCtxParams::new(ZSTD_CLEVEL_DEFAULT), in_buff: Vec::new(), in_to_compress: 0, @@ -102,11 +103,7 @@ impl CCtx { /// `ZSTD_CCtx_setParameter`: `param` is one of [`super::params::c_param`]. pub fn set_parameter(&mut self, param: i32, value: i32) -> Result { if self.stream_stage != StreamStage::Init { - if is_update_authorized(param) { - self.c_params_changed = true; - } else { - return Err(ErrorCode::StageWrong); - } + return Err(ErrorCode::StageWrong); } self.requested_params.set_parameter(param, value) } @@ -120,52 +117,21 @@ impl CCtx { Ok(()) } - /// `ZSTD_clearAllDicts` - fn clear_all_dicts(&mut self) { - self.local_dict = None; - self.cdict = None; - } - /// `ZSTD_CCtx_loadDictionary`: every frame from now on starts with `dict`, of which a copy is kept. If it is empty, /// with no dictionary. pub fn load_dictionary(&mut self, dict: &[u8]) -> Result<()> { if self.stream_stage != StreamStage::Init { return Err(ErrorCode::StageWrong); } - self.clear_all_dicts(); - if dict.is_empty() { - return Ok(()); - } - let mut copy = Vec::new(); - copy.try_reserve_exact(dict.len()) - .map_err(|_| ErrorCode::MemoryAllocation)?; - copy.extend_from_slice(dict); - self.local_dict = Some(Arc::new(copy)); - Ok(()) - } - - /// `ZSTD_CCtx_reset` - pub fn reset(&mut self, reset: ResetDirective) -> Result<()> { - if matches!( - reset, - ResetDirective::SessionOnly | ResetDirective::SessionAndParameters - ) { - self.stream_stage = StreamStage::Init; - self.frame.pledged_src_size_plus_one = 0; - } - if matches!( - reset, - ResetDirective::Parameters | ResetDirective::SessionAndParameters - ) { - if self.stream_stage != StreamStage::Init { - return Err(ErrorCode::StageWrong); - } - self.clear_all_dicts(); - self.requested_params = CCtxParams::new(ZSTD_CLEVEL_DEFAULT); + self.local_dict = None; + self.cdict = None; + if !dict.is_empty() { + self.local_dict = Some(Arc::new(try_copy(dict)?)); } Ok(()) } + /// `ZSTD_CCtx_reset` with `ZSTD_reset_session_only` fn reset_session(&mut self) { self.stream_stage = StreamStage::Init; self.frame.pledged_src_size_plus_one = 0; @@ -440,10 +406,6 @@ impl CCtx { if self.frame.applied_params.nb_workers > 0 { let mtctx = self.mtctx.as_mut().ok_or(ErrorCode::Generic)?; - if self.c_params_changed { - mtctx.update_c_params_while_compressing(&self.requested_params); - self.c_params_changed = false; - } loop { let ipos = input.pos; let opos = output.pos; @@ -451,6 +413,7 @@ impl CCtx { self.frame.consumed_src_size += (input.pos - ipos) as u64; self.frame.produced_c_size += (output.pos - opos) as u64; if flush_min.is_err() || (end_op == EndDirective::End && flush_min == Ok(0)) { + // `reset_session`, but `mtctx` is borrowed. self.stream_stage = StreamStage::Init; self.frame.pledged_src_size_plus_one = 0; } diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index cfd310a9f876..e547eedcfbd4 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -17,26 +17,24 @@ use crate::common::{ BlockType, LL_BITS, LL_FSE_LOG, MAX_LL, MAX_ML, MAX_OFF, ML_BITS, ML_FSE_LOG, OFF_FSE_LOG, REP_START_VALUE, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_DID_FIELD_SIZE, ZSTD_FCS_FIELD_SIZE, - ZSTD_FRAMEIDSIZE, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGIC_SKIPPABLE_MASK, ZSTD_MAGIC_SKIPPABLE_START, - ZSTD_MAGICNUMBER, ZSTD_SKIPPABLEHEADERSIZE, ZSTD_WINDOWLOG_ABSOLUTEMIN, - ZSTD_WINDOWLOG_LIMIT_DEFAULT, ZSTD_WINDOWLOG_MAX, + ZSTD_FRAMEHEADERSIZE_MAX, ZSTD_FRAMEIDSIZE, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGIC_SKIPPABLE_MASK, + ZSTD_MAGIC_SKIPPABLE_START, ZSTD_MAGICNUMBER, ZSTD_SKIPPABLEHEADERSIZE, + ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WINDOWLOG_LIMIT_DEFAULT, ZSTD_WINDOWLOG_MAX, + ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, }; use crate::error::{ErrorCode, Result}; use crate::util::{SliceExt, read_le16, read_le32, read_le64, try_copy, try_zeroed}; -use crate::{InBuffer, OutBuffer, ResetDirective}; +use crate::{InBuffer, OutBuffer}; /// `ZSTD_FRAMEHEADERSIZE_PREFIX`: what it takes to know the size of a frame header. const FRAMEHEADERSIZE_PREFIX: usize = 5; const FRAMEHEADERSIZE_MIN: usize = 6; -const FRAMEHEADERSIZE_MAX: usize = 18; const ZSTD_MAXWINDOWSIZE_DEFAULT: usize = (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1; const ZSTD_NO_FORWARD_PROGRESS_MAX: u32 = 16; -const ZSTD_WORKSPACETOOLARGE_FACTOR: usize = 3; -const ZSTD_WORKSPACETOOLARGE_MAXDURATION: usize = 128; /// `ZSTD_d_windowLogMax`, the only parameter of a [`DCtx`] that `node:zlib` can name. -pub const ZSTD_D_WINDOW_LOG_MAX: i32 = 100; +const ZSTD_D_WINDOW_LOG_MAX: i32 = 100; // ── frame headers ───────────────────────────────────────────────────────── @@ -277,7 +275,7 @@ pub fn frame_extent(src: &[u8]) -> Result { } /// `ZSTD_findFrameCompressedSize`: how many bytes the frame at the start of `src` takes. -pub fn find_frame_compressed_size(src: &[u8]) -> Result { +fn find_frame_compressed_size(src: &[u8]) -> Result { match frame_extent(src)? { FrameExtent::Whole(size) => Ok(size), FrameExtent::AtLeast(_) => Err(ErrorCode::SrcSizeWrong), @@ -286,7 +284,7 @@ pub fn find_frame_compressed_size(src: &[u8]) -> Result { /// `ZSTD_findDecompressedSize`: the size of all the frames of `src` together, once decompressed. /// [`ZSTD_CONTENTSIZE_UNKNOWN`] if one of them does not say, [`ZSTD_CONTENTSIZE_ERROR`] if `src` is not whole frames. -pub fn find_decompressed_size(mut src: &[u8]) -> u64 { +pub(crate) fn find_decompressed_size(mut src: &[u8]) -> u64 { let mut total_dst_size = 0u64; while src.len() >= FRAMEHEADERSIZE_PREFIX { let frame_src_size = if read_le32(src, 0).is_ok_and(is_skippable_magic) { @@ -428,7 +426,7 @@ impl<'a> DDict<'a> { } /// What `ZSTD_DCtx_refPrefix` makes of `prefix`: content to refer to, whatever it starts with. - pub fn prefix(prefix: &'a [u8]) -> Result { + pub(crate) fn prefix(prefix: &'a [u8]) -> Result { Self::new(Cow::Borrowed(prefix), DictContentType::RawContent) } @@ -503,7 +501,7 @@ struct DState { /// The last byte of a frame's input is held back until its output is flushed. hostage_byte: bool, no_forward_progress: u32, - header_buffer: [u8; FRAMEHEADERSIZE_MAX], + header_buffer: [u8; ZSTD_FRAMEHEADERSIZE_MAX], oversized_duration: usize, } @@ -521,6 +519,9 @@ impl Default for DCtx { } impl DCtx { + /// How much memory a new one takes. + pub const SIZE: usize = size_of::() + size_of::(); + /// `ZSTD_createDCtx` pub fn new() -> Self { Self { @@ -549,7 +550,7 @@ impl DCtx { lh_size: 0, hostage_byte: false, no_forward_progress: 0, - header_buffer: [0; FRAMEHEADERSIZE_MAX], + header_buffer: [0; ZSTD_FRAMEHEADERSIZE_MAX], oversized_duration: 0, }, ddict: None, @@ -606,28 +607,6 @@ impl DCtx { Ok(()) } - /// `ZSTD_DCtx_reset` - pub fn reset(&mut self, reset: ResetDirective) -> Result<()> { - if matches!( - reset, - ResetDirective::SessionOnly | ResetDirective::SessionAndParameters - ) { - self.state.stream_stage = DStreamStage::Init; - self.state.no_forward_progress = 0; - } - if matches!( - reset, - ResetDirective::Parameters | ResetDirective::SessionAndParameters - ) { - if self.state.stream_stage != DStreamStage::Init { - return Err(ErrorCode::StageWrong); - } - self.ddict = None; - self.state.max_window_size = ZSTD_MAXWINDOWSIZE_DEFAULT; - } - Ok(()) - } - /// `ZSTD_initDStream`: forgets the frame in progress and the dictionary. pub fn init_stream(&mut self) { self.state.stream_stage = DStreamStage::Init; @@ -648,7 +627,7 @@ impl DCtx { } /// `ZSTD_decompress` -pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { +pub(crate) fn decompress(dst: &mut [u8], src: &[u8]) -> Result { DCtx::new().decompress(dst, src) } diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 03e72a9213f8..e21911b3742c 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -18,15 +18,13 @@ mod decompress; mod error; mod util; -pub use common::{ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN}; -pub use compress::params::{ - ZSTD_CLEVEL_DEFAULT, ZSTD_MAX_CLEVEL, ZSTD_MIN_CLEVEL, c_param, compress_bound, -}; +pub use common::ZSTD_CONTENTSIZE_ERROR; +use common::ZSTD_CONTENTSIZE_UNKNOWN; +pub use compress::params::c_param::COMPRESSION_LEVEL as ZSTD_C_COMPRESSION_LEVEL; +pub use compress::params::{ZSTD_CLEVEL_DEFAULT, compress_bound}; pub use compress::stream::{CCtx, EndDirective, compress}; -pub use decompress::{ - DCtx, DDict, FrameExtent, ZSTD_D_WINDOW_LOG_MAX, decompress, find_decompressed_size, - find_frame_compressed_size, frame_extent, get_frame_content_size, -}; +pub use decompress::{DCtx, DDict, FrameExtent, frame_extent, get_frame_content_size}; +use decompress::{decompress, find_decompressed_size}; pub use error::ErrorCode; /// `ZSTD_inBuffer` @@ -43,12 +41,58 @@ pub struct OutBuffer<'a> { pub pos: usize, } -/// `ZSTD_ResetDirective` -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum ResetDirective { - SessionOnly = 1, - Parameters = 2, - SessionAndParameters = 3, +/// Output at the end of a `Vec`. The codecs write to initialized memory only: this zeroes about as much as gets +/// written, however many calls that takes and however much capacity the `Vec` has to spare. +pub struct VecOutput<'a> { + vec: &'a mut Vec, + start: usize, + /// `vec[..filled]` is output, the rest of it zeros. The `Vec` is cut to it on drop. + filled: usize, +} + +impl<'a> VecOutput<'a> { + pub fn new(vec: &'a mut Vec) -> Self { + let filled = vec.len(); + Self { + vec, + start: filled, + filled, + } + } + + /// The length of the `Vec` up to the end of the output. + pub fn filled(&self) -> usize { + self.filled + } + + /// Room for one call of a codec: `step` bytes or more, but never past `limit`. [`advance`](Self::advance) by + /// what it wrote. + pub fn buffer(&mut self, step: usize, limit: usize) -> Result, ErrorCode> { + let room = self.vec.len() - self.filled; + let budget = limit.saturating_sub(self.filled); + if room < step.min(budget) { + let more = step.max(self.filled - self.start).min(budget) - room; + self.vec + .try_reserve(more) + .map_err(|_| ErrorCode::MemoryAllocation)?; + self.vec.resize(self.vec.len() + more, 0); + } + let end = self.vec.len().min(limit); + Ok(OutBuffer { + dst: self.vec.get_mut(self.filled..end).unwrap_or_default(), + pos: 0, + }) + } + + pub fn advance(&mut self, written: usize) { + self.filled = (self.filled + written).min(self.vec.len()); + } +} + +impl Drop for VecOutput<'_> { + fn drop(&mut self) { + self.vec.truncate(self.filled); + } } // ----------------------------------- @@ -205,7 +249,7 @@ impl StreamingDecoder { matches!(self.state, State::Inflating) } - /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is + /// Append decompressed bytes to `out` (growing in steps of 4096 bytes or more) until `input` is /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. /// Returns `ShortRead` when more input is required and `is_done` is false. pub fn decompress( @@ -220,6 +264,7 @@ impl StreamingDecoder { } let mut in_buf = InBuffer { src: input, pos: 0 }; + let mut out = VecOutput::new(out); while matches!(self.state, State::Uninitialized | State::Inflating) { let next_in_len = input.len() - in_buf.pos; @@ -235,27 +280,19 @@ impl StreamingDecoder { return Ok(in_buf.pos); } - if out.len() >= max_output { + if out.filled() >= max_output { return Ok(in_buf.pos); } - if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { + let Ok(mut out_buf) = out.buffer(STREAMING_OUTPUT_STEP, max_output) else { self.state = State::Error; return Err(ZstdError::OutOfMemory); - } - // The decoder writes to initialized memory only. - let written_before = out.len(); - let window = (out.capacity() - written_before).min(max_output - written_before); - out.resize(written_before + window, 0); - let mut out_buf = OutBuffer { - dst: out.get_mut(written_before..).unwrap_or_default(), - pos: 0, }; let read_before = in_buf.pos; let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); - let bytes_written = out_buf.pos; - out.truncate(written_before + bytes_written); + let (bytes_written, window) = (out_buf.pos, out_buf.dst.len()); + out.advance(bytes_written); let remaining = match result { Ok(remaining) => remaining, Err(code) => { diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index cd00ad10c97d..4646f452d4a1 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1066,72 +1066,97 @@ describe("the output is that of zstd 1.5.7", () => { state ^= state << 5; return (state >>>= 0); }; - const words = ["function ", "return ", "const ", "zstd", "\n ", "{", "}", "(", ");", "await ", "=> ", "length", "0"]; - const out = new Uint8Array(size); + const words = [ + "function ", + "return ", + "const ", + "zstd", + "\n ", + "{", + "}", + "(", + ");", + "await ", + "=> ", + "length", + "0", + ].map(word => Buffer.from(word)); + // Room for the last piece to overrun. + const out = new Uint8Array(size + 2048); let at = 0; while (at < size) { const kind = next() % 16; - if (kind < 8) { - for (const c of words[next() % words.length]) if (at < size) out[at++] = c.charCodeAt(0); - } else if (kind < 13 && at > 8) { + if (kind < 7) { + const word = words[next() % words.length]; + out.set(word, at); + at += word.length; + } else if (kind < 11 && at > 8) { const distance = 1 + (next() % Math.min(at, 1 << (next() % 20))); - for (let n = 3 + (next() % (1 << (next() % 10))); n > 0 && at < size; n--, at++) out[at] = out[at - distance]; + const length = 3 + (next() % (1 << (next() % 10))); + if (distance >= length) out.copyWithin(at, at - distance, at - distance + length); + else for (let i = 0; i < length; i++) out[at + i] = out[at + i - distance]; + at += length; } else if (kind < 15) { - for (let n = 1 + (next() % 40); n > 0 && at < size; n--) out[at++] = next(); + for (let n = 1 + (next() % 24); n > 0; n--) out[at++] = next() >>> (next() % 8 ? 27 : 24); } else { - const byte = next(); - for (let n = 1 + (next() % 2000); n > 0 && at < size; n--) out[at++] = byte; + const length = 1 + (next() % 300); + out.fill(next(), at, at + length); + at += length; } } - return out; + return out.subarray(0, size); } const digest = (bytes: Uint8Array) => `${bytes.length} ${Bun.hash.xxHash64(bytes).toString(16)}`; const c = zlib.constants; + // More than one job of the least size there is, for the workers. + const corpus = sample(700_000, 1); + const large = corpus.subarray(0, 300_000); + const small = corpus.subarray(300_000, 303_000); it("at every level", () => { - const large = sample(400_000, 1); - const small = sample(3_000, 2); const results: Record = {}; for (let level = 1; level <= 22; level++) { - results[level] = `${digest(zstdCompressSync(large, { level }))}, ${digest(zstdCompressSync(small, { level }))}`; + // The highest levels are slow. + const data = level < 19 ? large : large.subarray(0, 140_000); + results[level] = `${digest(zstdCompressSync(data, { level }))}, ${digest(zstdCompressSync(small, { level }))}`; } for (const level of [-1, -7, -100]) { const params = { [c.ZSTD_c_compressionLevel]: level }; - results[level] = `${digest(zlib.zstdCompressSync(large, { params }))}, ${digest(zlib.zstdCompressSync(small, { params }))}`; + results[level] = + `${digest(zlib.zstdCompressSync(large, { params }))}, ${digest(zlib.zstdCompressSync(small, { params }))}`; } expect(results).toMatchInlineSnapshot(` { - "-1": "23477 2ffb3d0f3737833b, 382 c2a4c99592694597", - "-100": "79898 7ffbd56b19ee394, 1100 ae3344236a56deb9", - "-7": "29376 651fda629ac959c4, 481 55bfd66062e7c9ad", - "1": "24480 542ab87e1aa5fc9d, 396 107a4b7a04d25d37", - "10": "22592 19e5d2e09fe1ccad, 363 808b964648b85372", - "11": "22525 e0091e2608de2f00, 365 d83d5f8947b95fc1", - "12": "22525 e0091e2608de2f00, 365 dd249e23858fad1", - "13": "22510 f89e05ebc8ca7534, 358 eb0fe5ebe5118657", - "14": "22503 56827c33f86de3ae, 358 2f482857ef5c63f0", - "15": "22494 cc870ca463c122df, 356 7ceb5d984fe44594", - "16": "22635 c4a94ba8858c12d5, 355 25870ac97fe974", - "17": "22416 9a7fd5afd172f89b, 355 eede2d4cbdcf2340", - "18": "22010 8c967ec9b15ed62c, 353 5fffa6db6ed0ebda", - "19": "21820 967a4e946bf30696, 353 5fffa6db6ed0ebda", - "2": "23951 df37d7c5becb22d8, 384 7f01926a9aa9a46b", - "20": "21820 967a4e946bf30696, 353 478b55fe951a5684", - "21": "21735 ca28a8b1ffaeb4d9, 353 478b55fe951a5684", - "22": "21651 e9a29d4a60784b2c, 354 50650b6ef5b2b80a", - "3": "23931 de5a0598c57be84b, 382 c2a4c99592694597", - "4": "23923 85d64ea007fb86d3, 369 312f9dca641ef386", - "5": "23732 be13ecafaf15c817, 369 f3efffcbd8524307", - "6": "22720 47e610d53a3c1546, 364 6175591ab882c5a6", - "7": "22749 8176e71ad03765e7, 361 c028bca7b43936ae", - "8": "22683 96b450c8ed9a4a06, 361 c028bca7b43936ae", - "9": "22683 96b450c8ed9a4a06, 363 808b964648b85372", + "-1": "44935 b2f48ca4f89477e9, 810 e35cec5c3500c856", + "-100": "167967 45693a3bc89d2838, 2112 a1f04bdd2c75d853", + "-7": "65641 25bb487794d2f45f, 1081 6841140ea417de55", + "1": "46828 2a61d18b13a9df88, 824 3a3b2b799ed70f7b", + "10": "42353 d560f70047c278d, 807 bc69857972f7737c", + "11": "42124 cc37ec9a45f8d164, 808 95559a23325a524d", + "12": "42124 cc37ec9a45f8d164, 806 5984ef401fd2141", + "13": "42364 fd1a09c03715fc5f, 807 6648f2fa58a397d4", + "14": "42287 87ee023408ff3e30, 807 6648f2fa58a397d4", + "15": "42233 109ef580b5685e85, 804 52073852c548e55e", + "16": "42522 3293e655be1b0330, 807 6648f2fa58a397d4", + "17": "41809 7cb274a8772cd4cc, 806 b609660f6b3d21d9", + "18": "41253 d616eadc7ae1a5de, 804 52073852c548e55e", + "19": "18189 44c06af9c0a7b97f, 804 52073852c548e55e", + "2": "45933 2795a6f87171faac, 811 bd7ae68cb7d95862", + "20": "18125 7c0ef33ba30541a2, 804 52073852c548e55e", + "21": "18125 7c0ef33ba30541a2, 804 52073852c548e55e", + "22": "18125 e1b7fc6756c67456, 804 52073852c548e55e", + "3": "45122 344d3c35e5ad0340, 810 e35cec5c3500c856", + "4": "45093 7634242fc968e2c3, 822 6b45b45a5d921bfc", + "5": "44424 daecc1abec496995, 810 4489026b2c29b8dc", + "6": "42948 a879387b2601910d, 809 b15c4cf338e1d77a", + "7": "42675 1f9ad80ad0656d09, 804 603c9d665714a2", + "8": "42562 26911127c1e54142, 804 603c9d665714a2", + "9": "42562 26911127c1e54142, 807 bc69857972f7737c", } `); }); it("with parameters", () => { - const data = sample(700_000, 3); const cases: Record> = { checksum: { [c.ZSTD_c_checksumFlag]: 1 }, noContentSize: { [c.ZSTD_c_contentSizeFlag]: 0 }, @@ -1154,7 +1179,12 @@ describe("the output is that of zstd 1.5.7", () => { // ZSTD_c_format: ZSTD_f_zstd1_magicless magicless: { 10: 1 }, workers: { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 1, [c.ZSTD_c_checksumFlag]: 1 }, - workersOverlap: { [c.ZSTD_c_nbWorkers]: 3, [c.ZSTD_c_jobSize]: 1, [c.ZSTD_c_overlapLog]: 9, [c.ZSTD_c_windowLog]: 17 }, + workersOverlap: { + [c.ZSTD_c_nbWorkers]: 3, + [c.ZSTD_c_jobSize]: 1, + [c.ZSTD_c_overlapLog]: 9, + [c.ZSTD_c_windowLog]: 17, + }, workersLongDistance: { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 1, @@ -1164,91 +1194,157 @@ describe("the output is that of zstd 1.5.7", () => { }; const results: Record = {}; for (const [name, params] of Object.entries(cases)) { + const data = name.startsWith("workers") ? corpus : large; const compressed = zlib.zstdCompressSync(data, { params }); results[name] = digest(compressed); if (name !== "magicless") expect(zlib.zstdDecompressSync(compressed).equals(data)).toBe(true); } expect(results).toMatchInlineSnapshot(` { - "binaryTree": "39948 1232d8597f5c6", - "checksum": "41241 3cd85e71c12cf382", - "hashChain": "43217 7356b4e36e4bf4d8", - "longDistance": "41055 eedc98c3e261bec7", - "longDistanceOptimal": "38970 b85121583248455b", - "magicless": "41233 5ee803ef2ad12e33", - "noContentSize": "41234 b8ffb3bc3aef5fff", - "optimal": "38610 6d0c3f5689a3abe9", - "rows": "40635 26480c2e116c84d5", - "smallBlocks": "41058 4ef9d31b35c0acf8", - "smallWindow": "47463 95833be523748c4a", - "workers": "42269 b08e46380ba74997", - "workersLongDistance": "42141 793c24a764fcda50", - "workersOverlap": "42697 e672bcac52118c4d", + "binaryTree": "43844 a5c9eabbae2dd756", + "checksum": "44939 b9750b95ad15111d", + "hashChain": "48609 ad3f314838cea64", + "longDistance": "44645 1be0f74f50e4c2f1", + "longDistanceOptimal": "42035 826bf710d9511423", + "magicless": "44931 ef5be3a06c229e22", + "noContentSize": "44932 69582ab759972bd0", + "optimal": "42193 af8904ce81d257c5", + "rows": "43948 1c7f2cd5acfbb9eb", + "smallBlocks": "44603 b857325b812bea9c", + "smallWindow": "55906 6786884ad22d6b2d", + "workers": "108770 1df628af1e7dc077", + "workersLongDistance": "109211 732c0b64dad8574b", + "workersOverlap": "111826 bfab26c4004f255d", } `); }); - it("with a dictionary", () => { - const dictionary = sample(20_000, 4); + // Short input searches the tables of the dictionary, long input starts from a copy of them. + function withDictionary(dictionary: Uint8Array) { const results: Record = {}; for (const level of [-3, 1, 3, 5, 8, 13, 16, 19]) { - // Short input searches the tables of the dictionary, long input starts from a copy of them. - for (const size of [5_000, 200_000]) { - const data = new Uint8Array(size); - data.set(sample(size, 5)); - data.set(dictionary.subarray(3_000, 6_000), 1_000); + for (const size of [5_000, 150_000]) { + const data = corpus.slice(400_000, 400_000 + size); + data.set(dictionary.subarray(-600, -100), 1_000); const options = { dictionary, params: { [c.ZSTD_c_compressionLevel]: level }, pledgedSrcSize: size }; const compressed = zlib.zstdCompressSync(data, options); results[`${level}, ${size}`] = digest(compressed); expect(zlib.zstdDecompressSync(compressed, { dictionary }).equals(data)).toBe(true); } } - expect(results).toMatchInlineSnapshot(` + return results; + } + + it("with a dictionary that is only content", () => { + expect(withDictionary(sample(20_000, 4))).toMatchInlineSnapshot(` { - "-3, 200000": "11919 4646c7adfd3c7bfc", - "-3, 5000": "99 bcf9a6b6959f244", - "1, 200000": "11393 8ca74189c4dd8590", - "1, 5000": "85 5d94a445fbc1debf", - "13, 200000": "10612 72de0a04d3ab1dcc", - "13, 5000": "81 cf05da8002ffcca1", - "16, 200000": "10283 7e7f3e5510d0205b", - "16, 5000": "76 83586779a3533541", - "19, 200000": "10258 ecf6c3f4322413b5", - "19, 5000": "76 83586779a3533541", - "3, 200000": "11130 17e414f5ae34a715", - "3, 5000": "86 9b1f540d55f03dd", - "5, 200000": "10837 6628de069ef68382", - "5, 5000": "81 e436c64b13b807ee", - "8, 200000": "10482 8c6f6c78776fc48e", - "8, 5000": "78 ba67e3347e65a4a8", + "-3, 150000": "31691 a6cba8ed83f78cbe", + "-3, 5000": "1280 99e5c402823fdcad", + "1, 150000": "25367 4eed614674d852f5", + "1, 5000": "1042 c36ace4ccb9d37ab", + "13, 150000": "23771 c93a9f67de8b8c82", + "13, 5000": "988 6a7990046fc8b0de", + "16, 150000": "22992 c47f0966b096f185", + "16, 5000": "969 3be3d777b3ae4229", + "19, 150000": "22935 28de114f08f4bd22", + "19, 5000": "969 552f7abe774fa6ca", + "3, 150000": "24931 2feb8f5d9e046e54", + "3, 5000": "1028 6aaf30fe0e09dd18", + "5, 150000": "24343 c26404163f4b0830", + "5, 5000": "1002 6cf280704008a5eb", + "8, 150000": "23534 8b65e660bd8911a6", + "8, 5000": "973 f42818736cafcdfb", } `); }); + it("with a dictionary that has tables", () => { + // zstd --train --maxdict=1200, on 400 of these samples. + const dictionary = Buffer.from( + "N6Qw7CRPAAA5EKgmaQOIpOamUZ6YaYFWNK4FBYVp7RRHeEwRH2jGgTCmatsY1bd2V0loSVJyU2hzaXNTUlJIUminQwFAaTAYEstI" + + "FScAAATAhwSXlAeZRmKhICQIhcKgMCAQBgPCgCAYDAaAQWEgGBgOCWRJpPcCAAB0bWcj0zgoEAaDwXAwHA4MszhRamoRAAAAAQAA" + + "AAQAAAAIAAAACg1jb24XFBxYHlgKYXdhaXQgYXdhaXN0IAMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMD" + + "AwMDAwNubm5ubm5ubm5ubm5ubmxlbmd0aAgCHgZmdW5jdGlvbiBDDgQJFgCsIAMDAwMDAwMDKCgoKCgoPT09PT09PT0MDJMND8YY" + + "EgAKDAUNt5sCBQoIKN/f39/f39/f39/f39/f3/4aenN0ZHpzdGTBwcHBwcHBwcHBwcEwGwcJCAsfDROdHA8V6gMHGg89PT0wYXdh" + + "aXQgc3RkCA8MCwcIFhvWenN0ZAgPDAsHCAogIN/f3wkNBBcXBQ4WFRUbDQYbBBACCwILAgsCCwILY29uc3QgcmV0dXJuIGZ1bmN0" + + "aW9uID09PT09PT09PT09PT09PWF3YWl0IHQgdB4GHg0IBwMpO2F3YWl0ID0+IGZ1bmN0aW9uIGZ1bmN0aW9uIG5ubm5ubm5ubm5u" + + "bm5ubm5ubm5ubm5ubm5ubm5ubm57BvoUBvoUBvoCAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICw8PDw8PDw8PD" + + "w8PDw8PDw8PDw8PDw8PDw8MkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQk" + + "JCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQk" + + "JCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQ9PiBsZW5ndGhoowEX" + + "ImNvbnN0IB8IHRwHHgIYChUJBAgECAMYEg8bCxwDuQ4dDgYOjY2NC4IWBgcwKKCgoKB7KTswBgccAg8aEwIWDx4dGBUMFA8fCugK" + + "FxsfCB0cBx4CGAoADgt9ZnVuY3Rpb24gGRgLCBUbHAYaBwgQCxwQBQofGBgAAgMKgbcSHAkIHhwPGRwOcRsoPT4gMGNvbnN0IGZ1" + + "bmN0aW9uIBQOAiQWEAgUDRwUBBQIGwbP8MQYChIOBBQeHgsNDxALHMUXCBEaBADwxBgKEg4EFB4eCw0PEAscxRcIERoEAPDEGAoS" + + "DgRhd2FpdCBsZW5ndGhjb25zdCBsZW5ndGhsZW5ndGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGho" + + "aCh7e3t7e3t7e3t7e3t7e3t7e3t7e3toaGhoaGhoaGhokJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQ", + "base64", + ); + expect(withDictionary(dictionary)).toMatchInlineSnapshot(` + { + "-3, 150000": "31674 a6f9cb1f5deb45f", + "-3, 5000": "1255 86e003673a214f0a", + "1, 150000": "25439 b187159ef67d798f", + "1, 5000": "967 7ab6c959cbc9173", + "13, 150000": "23377 807244f3412a1248", + "13, 5000": "918 51a5e4bbad4678c8", + "16, 150000": "23360 7e0e03d462f05b47", + "16, 5000": "918 51a5e4bbad4678c8", + "19, 150000": "23224 dbc52271327892b3", + "19, 5000": "916 a6838a535dce670d", + "3, 150000": "24745 9a4a4d7d66087c52", + "3, 5000": "942 2f652f811a065d94", + "5, 150000": "24209 a726791cfb67d565", + "5, 5000": "926 f64c220e68e0c73c", + "8, 150000": "24000 7f2abf062f75ff72", + "8, 5000": "913 b12d4bfdabcdaf2b", + } + `); + + const compressed = zlib.zstdCompressSync(small, { dictionary }); + expect(() => zlib.zstdDecompressSync(compressed)).toThrow( + expect.objectContaining({ code: "ZSTD_error_dictionary_wrong", errno: c.ZSTD_error_dictionary_wrong }), + ); + const damaged = Buffer.from(dictionary); + damaged.fill(0xff, 8, 40); + expect(() => zlib.zstdCompressSync(small, { dictionary: damaged })).toThrow( + expect.objectContaining({ code: "ZSTD_error_memory_allocation", errno: c.ZSTD_error_memory_allocation }), + ); + }); + + async function stream(params: Record, flushes: boolean) { + const stream = zlib.createZstdCompress({ params }); + const chunks: Buffer[] = []; + stream.on("data", chunk => chunks.push(chunk)); + const { promise, resolve, reject } = Promise.withResolvers(); + stream.on("end", resolve).on("error", reject); + for (let at = 0, n = 1; at < corpus.length; at += n, n = (n * 7) % 90_001) { + stream.write(corpus.subarray(at, at + n)); + if (flushes && n % 3 === 0) stream.flush(); + } + stream.end(); + await promise; + return Buffer.concat(chunks); + } + it("in a stream with flushes", async () => { - const data = sample(600_000, 6); const results: Record = {}; for (const level of [1, 6, 12, 17]) { - const stream = zlib.createZstdCompress({ params: { [c.ZSTD_c_compressionLevel]: level } }); - const chunks: Buffer[] = []; - stream.on("data", chunk => chunks.push(chunk)); - const { promise, resolve, reject } = Promise.withResolvers(); - stream.on("end", resolve).on("error", reject); - for (let at = 0, n = 1; at < data.length; at += n, n = (n * 7) % 90_001) { - stream.write(data.subarray(at, at + n)); - if (n % 3 === 0) stream.flush(); - } - stream.end(); - await promise; - results[level] = digest(Buffer.concat(chunks)); + results[level] = digest(await stream({ [c.ZSTD_c_compressionLevel]: level }, true)); } expect(results).toMatchInlineSnapshot(` { - "1": "34466 1bd006842da7fff7", - "12": "31194 38b6be8e301dd3fc", - "17": "31048 2bcbaf13b8ae7446", - "6": "31602 54d0ef9110ac7561", + "1": "111500 f8b357a7535f4d59", + "12": "99899 c7c64eb5fefd6b8c", + "17": "99325 43b1ca1d605f6a2a", + "6": "101943 73ac52aed8b6fec6", } `); }); + + it("in a stream with workers", async () => { + for (const flushes of [false, true]) { + const compressed = await stream({ [c.ZSTD_c_nbWorkers]: 1, [c.ZSTD_c_jobSize]: 1 }, flushes); + expect(zlib.zstdDecompressSync(compressed).equals(corpus)).toBe(true); + } + }); }); diff --git a/test/js/node/zlib/zlib-estimated-size-gc.test.ts b/test/js/node/zlib/zlib-estimated-size-gc.test.ts index 4970f74b872e..e10176b3849e 100644 --- a/test/js/node/zlib/zlib-estimated-size-gc.test.ts +++ b/test/js/node/zlib/zlib-estimated-size-gc.test.ts @@ -12,7 +12,7 @@ import zlib from "node:zlib"; // estimateShallowMemoryUsageOf(cell) == sizeof(cell) + estimated_size(). `min` // is a floor below the per-mode footprint the constructor caches (brotli encode -// 5143, brotli decode 855, zstd compress 5272, zstd decompress 95968). +// 5143, brotli decode 855, zstd compress about 14000, zstd decompress about 35000). function checkFootprintStableAcrossClose(create: () => any, min: number) { const engine = create(); engine.on("error", () => {}); @@ -34,11 +34,11 @@ test("brotli decompress: estimated size stays tied to construction mode across c }); test("zstd compress: estimated size stays tied to construction mode across close", () => { - checkFootprintStableAcrossClose(() => zlib.createZstdCompress(), 5000); + checkFootprintStableAcrossClose(() => zlib.createZstdCompress(), 12000); }); test("zstd decompress: estimated size stays tied to construction mode across close", () => { - checkFootprintStableAcrossClose(() => zlib.createZstdDecompress(), 90000); + checkFootprintStableAcrossClose(() => zlib.createZstdDecompress(), 30000); }); // GC-safety guard: drive a write so the JS thread enters with_mut, then force From de3e36bc8f43f358cc039d0b3d50a2f8dc6c36a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 13:44:07 +0000 Subject: [PATCH 011/199] zstd: the harness that compares the port with the C library --- src/zstd/wip/build-ref.sh | 15 + src/zstd/wip/harness/Cargo.toml | 21 ++ src/zstd/wip/harness/build.rs | 6 + src/zstd/wip/harness/src/c.rs | 104 ++++++ src/zstd/wip/harness/src/comp.rs | 285 +++++++++++++++ src/zstd/wip/harness/src/data.rs | 65 ++++ src/zstd/wip/harness/src/dec.rs | 183 ++++++++++ src/zstd/wip/harness/src/extra.rs | 556 ++++++++++++++++++++++++++++++ src/zstd/wip/harness/src/main.rs | 31 ++ src/zstd/wip/harness/src/sizes.rs | 1 + src/zstd/wip/harness/sync.sh | 20 ++ 11 files changed, 1287 insertions(+) create mode 100755 src/zstd/wip/build-ref.sh create mode 100644 src/zstd/wip/harness/Cargo.toml create mode 100644 src/zstd/wip/harness/build.rs create mode 100644 src/zstd/wip/harness/src/c.rs create mode 100644 src/zstd/wip/harness/src/comp.rs create mode 100644 src/zstd/wip/harness/src/data.rs create mode 100644 src/zstd/wip/harness/src/dec.rs create mode 100644 src/zstd/wip/harness/src/extra.rs create mode 100644 src/zstd/wip/harness/src/main.rs create mode 100644 src/zstd/wip/harness/src/sizes.rs create mode 100755 src/zstd/wip/harness/sync.sh diff --git a/src/zstd/wip/build-ref.sh b/src/zstd/wip/build-ref.sh new file mode 100755 index 000000000000..209061b63378 --- /dev/null +++ b/src/zstd/wip/build-ref.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Builds the reference C zstd exactly as Bun's scripts/build/deps/zstd.ts did (pin f8745da6 + Bun's two patches). +set -e +cd /root/zstd-ref +SRC="common/debug common/entropy_common common/error_private common/fse_decompress common/pool common/threading common/xxhash common/zstd_common compress/fse_compress compress/hist compress/huf_compress compress/zstd_compress compress/zstd_compress_literals compress/zstd_compress_sequences compress/zstd_compress_superblock compress/zstd_double_fast compress/zstd_fast compress/zstd_lazy compress/zstd_ldm compress/zstd_opt compress/zstd_preSplit compress/zstdmt_compress decompress/huf_decompress decompress/zstd_ddict decompress/zstd_decompress decompress/zstd_decompress_block" +build() { # name, extra flags, asm? + local name=$1; shift; local asm=$1; shift + mkdir -p build/$name; local objs="" + for s in $SRC; do o=build/$name/$(basename $s).o; clang-23 -c -O2 -g -fPIC -march=nehalem -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=0 -DXXH_NAMESPACE=ZSTD_ -Izstd/lib -Izstd/lib/common "$@" zstd/lib/$s.c -o $o & objs="$objs $o"; done + if [ $asm = 1 ]; then clang-23 -c -fPIC zstd/lib/decompress/huf_decompress_amd64.S -o build/$name/huf_asm.o; objs="$objs build/$name/huf_asm.o"; fi + wait + rm -f build/$name/libzstd.a; llvm-ar-23 rcs build/$name/libzstd.a $objs +} +build ref 1 +build noasm 0 -DZSTD_DISABLE_ASM=1 -DDYNAMIC_BMI2=0 diff --git a/src/zstd/wip/harness/Cargo.toml b/src/zstd/wip/harness/Cargo.toml new file mode 100644 index 000000000000..6be9d34131e2 --- /dev/null +++ b/src/zstd/wip/harness/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "zstd_harness" +version = "0.0.0" +edition = "2024" + +[dependencies] +bun_zstd = { path = "zstd-pub" } + +[profile.release] +debug = "line-tables-only" +overflow-checks = true +debug-assertions = true +panic = "abort" + +[profile.bench] +codegen-units = 1 +inherits = "release" +overflow-checks = false +debug-assertions = false + +[workspace] diff --git a/src/zstd/wip/harness/build.rs b/src/zstd/wip/harness/build.rs new file mode 100644 index 000000000000..e0b8fe9e7a87 --- /dev/null +++ b/src/zstd/wip/harness/build.rs @@ -0,0 +1,6 @@ +fn main() { + let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); + println!("cargo:rerun-if-env-changed=ZSTD_REF"); + println!("cargo:rustc-link-search=native=/root/zstd-ref/build/{variant}"); + println!("cargo:rustc-link-lib=static=zstd"); +} diff --git a/src/zstd/wip/harness/src/c.rs b/src/zstd/wip/harness/src/c.rs new file mode 100644 index 000000000000..d5a82e1f6b78 --- /dev/null +++ b/src/zstd/wip/harness/src/c.rs @@ -0,0 +1,104 @@ +//! The reference C library. +#![allow(non_camel_case_types, dead_code)] +use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; + +#[repr(C)] +pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } +#[repr(C)] +pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } +pub enum ZSTD_CCtx {} +pub enum ZSTD_DCtx {} +pub enum ZSTD_DDict {} + +unsafe extern "C" { + pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; + pub fn ZSTD_compressBound(size: usize) -> usize; + pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_isError(code: usize) -> c_uint; + pub fn ZSTD_getErrorCode(code: usize) -> c_uint; + pub fn ZSTD_getErrorName(code: usize) -> *const c_char; + pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; + pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; + pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; + pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; + pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; + pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; + pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; + pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; + pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; + pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; + pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; + pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; + pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; + pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; + pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; + pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; + pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; + pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; + pub fn ZSTD_defaultCLevel() -> c_int; + pub fn ZSTD_minCLevel() -> c_int; + pub fn ZSTD_maxCLevel() -> c_int; +} + +/// `Ok(value)` or `Err(error code)`. +pub fn res(code: usize) -> Result { + unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } +} + +pub struct CCtx(pub *mut ZSTD_CCtx); +impl CCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } + pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + /// Returns (result, out pos, in pos). + pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); + (r, o.pos, i.pos) + } +} +impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } + +pub struct DCtx(pub *mut ZSTD_DCtx); +impl DCtx { + pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } + pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } + pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } + pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } + pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } + pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) + } + pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { + let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; + let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; + let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); + (r, o.pos, i.pos) + } +} +impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } + +pub fn compress(src: &[u8], level: i32) -> Vec { + let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; + let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); + dst.truncate(n); + dst +} +pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { + res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) +} diff --git a/src/zstd/wip/harness/src/comp.rs b/src/zstd/wip/harness/src/comp.rs new file mode 100644 index 000000000000..586336aa7dde --- /dev/null +++ b/src/zstd/wip/harness/src/comp.rs @@ -0,0 +1,285 @@ +//! The compressor against the C one: same calls, same results, same bytes, same positions. +use crate::c; +use crate::data::{Rng, data}; +use bun_zstd as z; + +fn r2(r: Result) -> Result { r.map_err(|e| e as u32) } +fn r0(r: Result<(), z::ErrorCode>) -> Result { r.map(|()| 0).map_err(|e| e as u32) } + +/// A dictionary: raw content, or in zstd's format (made from a real one by the C trainer is not available, so entropy +/// tables are taken from a frame... here: raw, or magic + garbage to exercise the error path). +fn dictionary(rng: &mut Rng, base: &[u8]) -> Vec { + let len = rng.size(1 << 17); + let mut d = if rng.chance(1, 2) && !base.is_empty() { + let start = rng.below(base.len()); + let end = (start + len).min(base.len()); + base[start..end].to_vec() + } else { + data(rng, len) + }; + if rng.chance(1, 12) && d.len() >= 8 { + d[..4].copy_from_slice(&0xEC30A437u32.to_le_bytes()); + } + d +} + +fn level(rng: &mut Rng) -> i32 { + match rng.below(10) { + 0 => -(rng.range(1, 50) as i32), + 1 => rng.range(0, 30) as i32 - 4, + 2 => -(rng.size(200000) as i32), + _ => rng.range(1, 22) as i32, + } +} + +pub fn one_shot(rng: &mut Rng, max: usize) { + let len = rng.size(max); + let src = data(rng, len); + let lvl = level(rng); + let bound = unsafe { c::ZSTD_compressBound(len) }; + assert_eq!(bound, z::compress_bound(len)); + let cap = match rng.below(8) { 0 => rng.below(bound + 1), 1 => bound.saturating_sub(rng.below(20)), _ => bound + rng.below(50) }; + let mut a = vec![0u8; cap]; + let mut b = vec![0u8; cap]; + let ra = c::res(unsafe { c::ZSTD_compress(a.as_mut_ptr().cast(), cap, src.as_ptr().cast(), len, lvl) }); + let rb = r2(z::compress(&mut b, &src, lvl)); + assert_eq!(ra, rb, "one shot: len {len} level {lvl} cap {cap}"); + if let Ok(n) = ra { + if a[..n] != b[..n] { + let at = a[..n].iter().zip(&b[..n]).position(|(x, y)| x != y); + panic!("one shot bytes differ at {at:?}/{n}: len {len} level {lvl}"); + } + } +} + +const PARAMS: &[(i32, i32, i32)] = &[ + (10, 0, 1), (100, -50, 22), (101, 10, 27), (102, 6, 26), (103, 6, 26), (104, 1, 12), (105, 3, 7), (106, 0, 4096), + (107, 1, 9), (130, 0, 20000), (160, 0, 2), (161, 6, 24), (162, 4, 4096), (163, 1, 8), (164, 0, 12), + (200, 0, 1), (201, 0, 1), (202, 0, 1), +]; + +fn set_params(rng: &mut Rng, a: &mut c::CCtx, b: &mut z::CCtx, log: &mut Vec, mt: bool) { + let n = match rng.below(4) { 0 => 0, 1 => 1, 2 => rng.below(4), _ => rng.below(9) }; + for _ in 0..n { + let (p, v) = if rng.chance(1, 25) { + (rng.below(403) as i32, rng.range(0, 40) as i32 - 5) + } else { + let &(p, lo, hi) = rng.pick(PARAMS); + let v = if rng.chance(1, 20) { rng.range(0, 3000000) as i32 - 200000 } else { lo + rng.below((hi - lo + 1) as usize) as i32 }; + (p, v) + }; + if !mt && (400..=402).contains(&p) { continue; } + let ra = a.set(p as u32, v); + let rb = r2(b.set_parameter(p, v)); + log.push(format!("set {p} {v} -> {ra:?}")); + assert_eq!(ra, rb, "{log:?}"); + } +} + +/// One context, several frames, random calls. +pub fn stream(rng: &mut Rng, max: usize) { + let mut a = c::CCtx::new(); + let mut b = z::CCtx::new(); + let mut log: Vec = Vec::new(); + let frames = if rng.chance(1, 3) { rng.range(2, 5) } else { 1 }; + for _ in 0..frames { + if rng.chance(1, 8) { a.reset(1).unwrap(); b.reset_session(); log.push("reset".into()); } + set_params(rng, &mut a, &mut b, &mut log, false); + let len = rng.size(max); + let src = data(rng, len); + if rng.chance(1, 4) && std::env::var_os("NODICT").is_none() { + let d = if rng.chance(1, 10) { Vec::new() } else { dictionary(rng, &src) }; + let ra = a.load_dict(&d); + let rb = r0(b.load_dictionary(&d)); + log.push(format!("dict {} -> {ra:?}", d.len())); + assert_eq!(ra, rb, "{log:?}"); + } + if rng.chance(1, 4) { + let n = if rng.chance(1, 8) { rng.size(max) as u64 } else { len as u64 }; + let ra = a.pledge(n); + let rb = r0(b.set_pledged_src_size(n)); + log.push(format!("pledge {n} (len {len}) -> {ra:?}")); + assert_eq!(ra, rb, "{log:?}"); + } + log.push(format!("frame len {len}")); + // how the input and the output are cut + let in_mode = rng.below(4); + let out_mode = rng.below(4); + let small = len < 5000; + let mut spos_total = 0usize; + let mut calls = 0usize; + let mut failed = false; + loop { + calls += 1; + let left = len - spos_total; + let chunk = match in_mode { + 0 => left, + 1 => rng.range(0, if small { 64 } else { 100_000 }).min(left), + 2 => rng.size(1 << 19).min(left), + _ => if rng.chance(1, 2) { left } else { rng.below(left + 1) }, + }; + let cap = match out_mode { + 0 => z::compress_bound(chunk) + rng.below(100), + 1 => rng.range(0, if small { 40 } else { 50_000 }), + 2 => rng.size(1 << 18), + _ => 1 << 17, + }; + let last = spos_total + chunk == len; + let end = if last && rng.chance(3, 4) { 2 } else { *rng.pick(&[0u32, 0, 0, 1, 1, 2]) }; + let end = if calls > 4000 { 2 } else { end }; + let input = &src[spos_total..spos_total + chunk]; + let mut oa = vec![0u8; cap]; + let mut ob = vec![0u8; cap]; + let (ra, opa, ipa) = a.stream2(&mut oa, 0, input, 0, end); + let (rb, opb, ipb) = { + let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; + let mut i = z::InBuffer { src: input, pos: 0 }; + let r = r2(b.compress_stream2(&mut o, &mut i, match end { 0 => z::EndDirective::Continue, 1 => z::EndDirective::Flush, _ => z::EndDirective::End })); + (r, o.pos, i.pos) + }; + if (ra, opa, ipa) != (rb, opb, ipb) || oa[..opa] != ob[..opb] { + let at = oa.iter().zip(&ob).position(|(x, y)| x != y); + panic!("stream: call {calls} chunk {chunk} cap {cap} end {end}: C {:?} Rust {:?} first diff {at:?}\n{log:?}", (ra, opa, ipa), (rb, opb, ipb)); + } + spos_total += ipa; + if ra.is_err() { failed = true; break; } + if end == 2 && ra == Ok(0) { + if spos_total == len { break; } + log.push(format!("frame ended at {spos_total}")); + } + if calls > 6000 { panic!("no end {log:?}"); } + } + if failed { a.reset(1).unwrap(); b.reset_session(); } + } +} + +pub fn run(seed: u64, iters: u64, max: usize) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if std::env::var_os("TRACE").is_some() { eprintln!("iter {i} seed {s}"); } + if rng.chance(1, 3) { one_shot(&mut rng, max) } else { stream(&mut rng, max) } + } + println!("comp ok: seed {seed}, {iters} iterations"); +} + +/// Multithreaded: when output comes out depends on timing, so only the whole output is compared. +pub fn mt(rng: &mut Rng, max: usize) { + let mut a = c::CCtx::new(); + let mut b = z::CCtx::new(); + let mut log: Vec = Vec::new(); + let frames = if rng.chance(1, 3) { rng.range(2, 3) } else { 1 }; + for _ in 0..frames { + let mut set = |p: i32, v: i32, log: &mut Vec| { + let ra = a.set(p as u32, v); + let rb = r2(b.set_parameter(p, v)); + log.push(format!("set {p} {v} -> {ra:?}")); + assert_eq!(ra, rb, "{log:?}"); + }; + set(400, rng.range(1, 4) as i32, &mut log); + if rng.chance(1, 2) { set(401, rng.range(0, 3 << 20) as i32, &mut log); } + if rng.chance(1, 2) { set(402, rng.range(0, 9) as i32, &mut log); } + set(100, if rng.chance(1, 6) { rng.range(13, 19) as i32 } else { rng.range(1, 12) as i32 - 3 }, &mut log); + if rng.chance(1, 2) { set(101, rng.range(10, 22) as i32, &mut log); } + if rng.chance(1, 3) { set(160, 1, &mut log); if rng.chance(1, 2) { set(162, rng.range(4, 200) as i32, &mut log); } if rng.chance(1,2) { set(161, rng.range(6, 20) as i32, &mut log); } } + if rng.chance(1, 2) { set(201, 1, &mut log); } + if rng.chance(1, 8) { set(130, rng.range(1340, 60000) as i32, &mut log); } + let len = if rng.chance(1, 4) { rng.size(max) } else { rng.range(max / 4, max) }; + let src = data(rng, len); + if rng.chance(1, 5) { + let d = dictionary(rng, &src); + let ra = a.load_dict(&d); + let rb = r0(b.load_dictionary(&d)); + log.push(format!("dict {} -> {ra:?}", d.len())); + assert_eq!(ra, rb, "{log:?}"); + } + if rng.chance(1, 4) { + let ra = a.pledge(len as u64); + let rb = r0(b.set_pledged_src_size(len as u64)); + log.push(format!("pledge {len}")); + assert_eq!(ra, rb, "{log:?}"); + } + log.push(format!("frame len {len}")); + let mut out_a: Vec = Vec::new(); + let mut out_b: Vec = Vec::new(); + let mut pos = 0usize; + let in_mode = rng.below(3); + loop { + let left = len - pos; + let chunk = match in_mode { 0 => left, 1 => rng.size(1 << 21).min(left), _ => rng.range(0, 300_000).min(left) }; + let last = pos + chunk == len; + let end = if last { 2u32 } else { *rng.pick(&[0u32, 0, 0, 0, 1]) }; + let input = &src[pos..pos + chunk]; + let cap = z::compress_bound(len.max(1 << 20)) + 1024; + let mut buf = vec![0u8; cap]; + let mut err_a = None; + let mut err_b = None; + let mut ip = 0usize; + loop { + let (r, op, np) = a.stream2(&mut buf, 0, input, ip, end); + out_a.extend_from_slice(&buf[..op]); + ip = np; + match r { Err(e) => { err_a = Some(e); break; } Ok(left) => if (end == 0 && ip == chunk) || (end != 0 && left == 0) { break; } } + } + let mut ip = 0usize; + loop { + let mut o = z::OutBuffer { dst: &mut buf, pos: 0 }; + let mut i = z::InBuffer { src: input, pos: ip }; + let r = r2(b.compress_stream2(&mut o, &mut i, match end { 0 => z::EndDirective::Continue, 1 => z::EndDirective::Flush, _ => z::EndDirective::End })); + let op = o.pos; + ip = i.pos; + out_b.extend_from_slice(&buf[..op]); + match r { Err(e) => { err_b = Some(e); break; } Ok(left) => if (end == 0 && ip == chunk) || (end != 0 && left == 0) { break; } } + } + assert_eq!(err_a, err_b, "{log:?}"); + log.push(format!("chunk {chunk} end {end}")); + if err_a.is_some() { a.reset(1).unwrap(); b.reset_session(); break; } + pos += chunk; + if last { break; } + } + if out_a != out_b { + let at = out_a.iter().zip(&out_b).position(|(x, y)| x != y); + panic!("mt: {} vs {} bytes, first diff {at:?}\n{log:?}", out_a.len(), out_b.len()); + } + } +} + +pub fn run_mt(seed: u64, iters: u64, max: usize) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if std::env::var_os("TRACE").is_some() { eprintln!("iter {i} seed {s}"); } + mt(&mut rng, max); + } + println!("mt ok: seed {seed}, {iters} iterations"); +} + +/// Dictionaries, by strategy. +pub fn run_dict(seed: u64, iters: u64, max: usize) { + let mut fails = std::collections::BTreeMap::new(); + for i in 0..iters { + let mut rng = Rng(seed.wrapping_mul(1_000_003).wrapping_add(i)); + let strat = rng.range(1, 9) as i32; + let len = rng.size(max); + let src = data(&mut rng, len); + let d = dictionary(&mut rng, &src); + let pledge = rng.chance(1, 2); + let mut a = c::CCtx::new(); + let mut b = z::CCtx::new(); + a.set(107, strat).unwrap(); b.set_parameter(107, strat).unwrap(); + a.load_dict(&d).unwrap(); b.load_dictionary(&d).unwrap(); + let cap = z::compress_bound(len) + 64; + let (mut oa, mut ob) = (vec![0u8; cap], vec![0u8; cap]); + let end = if pledge { 2 } else { 1 }; + let (ra, pa, _) = a.stream2(&mut oa, 0, &src, 0, end); + let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; + let mut inp = z::InBuffer { src: &src, pos: 0 }; + let rb = r2(b.compress_stream2(&mut o, &mut inp, if pledge { z::EndDirective::End } else { z::EndDirective::Flush })); + let pb = o.pos; + let ok = ra == rb && pa == pb && oa[..pa] == ob[..pb]; + let e = fails.entry((strat, pledge)).or_insert((0u32, 0u32)); + e.0 += 1; + if !ok { e.1 += 1; if e.1 == 1 { eprintln!("first fail strat {strat} pledge {pledge} len {len} dict {} iter {i}: {ra:?} {rb:?} {pa} {pb}", d.len()); } } + } + for (k, v) in fails { println!("{k:?}: {} of {} differ", v.1, v.0); } +} diff --git a/src/zstd/wip/harness/src/data.rs b/src/zstd/wip/harness/src/data.rs new file mode 100644 index 000000000000..c48a645e1b5d --- /dev/null +++ b/src/zstd/wip/harness/src/data.rs @@ -0,0 +1,65 @@ +//! Deterministic inputs. +pub struct Rng(pub u64); +impl Rng { + pub fn next(&mut self) -> u64 { + // splitmix64 + self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + } + pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } + pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } + pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } + pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } + /// Skewed towards small values. + pub fn size(&mut self, max: usize) -> usize { + let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); + (self.next() as usize & ((1usize << bits) - 1)).min(max) + } +} + +static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; + +/// One buffer of about `len` bytes, of a kind picked by `rng`. +pub fn data(rng: &mut Rng, len: usize) -> Vec { + let mut v = Vec::with_capacity(len + 64); + match rng.below(9) { + 0 => v.resize(len, rng.next() as u8), + 1 => while v.len() < len { v.push(rng.next() as u8) }, + 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, + 3 => { // few symbols, skewed + let n = rng.range(2, 40); + while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } + } + 4 => { // LZ-like: copies of earlier parts, at all distances + while v.len() < len { + if v.len() > 8 && rng.chance(3, 4) { + let dist = 1 + rng.size(v.len() - 1); + let n = 3 + rng.size(300); + for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } + } else { + for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } + } + } + } + 5 => { // periodic with noise + let period = rng.range(1, 300); + let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); + while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } + } + 6 => { // sections of different kinds + while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } + } + 7 => { // numbers as text + let mut x = rng.next() % 1000; + while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } + } + _ => { // long runs + while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } + } + } + v.truncate(len); + v +} diff --git a/src/zstd/wip/harness/src/dec.rs b/src/zstd/wip/harness/src/dec.rs new file mode 100644 index 000000000000..418ae256c1ae --- /dev/null +++ b/src/zstd/wip/harness/src/dec.rs @@ -0,0 +1,183 @@ +//! The decoder against the C decoder: valid frames, damaged frames, one call and streaming. +use crate::c; +use crate::data::{Rng, data}; +use bun_zstd as z; + +fn rs(r: Result) -> Result { r.map_err(|e| e as u32) } + +/// A frame (or several) made by the C compressor with parameters picked by `rng`. Returns (frames, content, dictionary). +pub fn make_frames(rng: &mut Rng) -> (Vec, Vec, Vec) { + let mut out = Vec::new(); + let mut content = Vec::new(); + let dict = if rng.chance(1, 4) { let n = rng.size(200_000); data(rng, n) } else { Vec::new() }; + let nframes = if rng.chance(1, 5) { rng.range(2, 4) } else { 1 }; + for _ in 0..nframes { + if rng.chance(1, 12) { + // skippable frame + let n = rng.size(300); + out.extend_from_slice(&(0x184D2A50u32 + rng.below(16) as u32).to_le_bytes()); + out.extend_from_slice(&(n as u32).to_le_bytes()); + out.extend_from_slice(&data(rng, n)); + continue; + } + let max = *rng.pick(&[0usize, 10, 300, 5_000, 70_000, 300_000, 1_500_000]); + let len = rng.size(max); + let mut src = data(rng, len); + if !dict.is_empty() && rng.chance(1, 2) && !src.is_empty() { + // make the dictionary useful + let n = rng.size(dict.len().min(src.len())); + let at = rng.below(dict.len() - n + 1); + let to = rng.below(src.len() - n + 1); + src[to..to + n].copy_from_slice(&dict[at..at + n]); + } + let mut cctx = c::CCtx::new(); + let level = *rng.pick(&[-50, -5, -1, 1, 1, 2, 3, 3, 3, 4, 5, 6, 7, 9, 12, 13, 16, 19, 22]); + cctx.set(100, if src.len() > 400_000 { level.min(12) } else { level }).unwrap(); + if rng.chance(1, 3) { cctx.set(201, 1).unwrap(); } // checksum + if rng.chance(1, 4) { cctx.set(200, 0).unwrap(); } // no content size + if rng.chance(1, 4) { cctx.set(101, rng.range(10, 22) as i32).unwrap(); } // windowLog + if rng.chance(1, 10) { cctx.set(130, rng.range(1340, 20_000) as i32).unwrap(); } // targetCBlockSize + if rng.chance(1, 10) { cctx.set(160, 1).unwrap(); } // LDM + if rng.chance(1, 8) { cctx.set(105, rng.range(3, 7) as i32).unwrap(); } // minMatch + if !dict.is_empty() { cctx.load_dict(&dict).unwrap(); } + let mut dst = vec![0u8; unsafe { c::ZSTD_compressBound(src.len()) } + 1024]; + let n = if rng.chance(1, 2) { + cctx.compress2(&mut dst, &src).unwrap() + } else { + // streaming, with flushes: several blocks, no content size + let (mut dpos, mut spos) = (0, 0); + loop { + let chunk = (spos + 1 + rng.size(200_000)).min(src.len()); + let end = if chunk == src.len() { 2 } else if rng.chance(1, 3) { 1 } else { 0 }; + let (r, d, s) = cctx.stream2(&mut dst, dpos, &src[..chunk], spos, end); + dpos = d; spos = s; + if r.unwrap() == 0 && end == 2 { break dpos; } + } + }; + out.extend_from_slice(&dst[..n]); + content.extend_from_slice(&src); + } + (out, content, dict) +} + +fn mutate(rng: &mut Rng, frames: &mut Vec) { + if frames.is_empty() { return; } + for _ in 0..1 + rng.size(4) { + if frames.is_empty() { return; } + // damage is more interesting near the start of a frame + let at = if rng.chance(1, 2) { rng.size(frames.len() - 1) } else { rng.below(frames.len()) }; + match rng.below(8) { + 0 => frames[at] ^= 1 << rng.below(8), + 1 => frames[at] = rng.next() as u8, + 2 => frames.truncate(at), + 3 => { frames.remove(at); } + 4 => frames.insert(at, rng.next() as u8), + 5 => { let n = rng.size(16).min(frames.len() - at); for b in &mut frames[at..at + n] { *b = rng.next() as u8 } } + 6 => { let n = rng.size(64); let extra = data(rng, n); frames.extend_from_slice(&extra) } + _ => { let from = rng.below(frames.len()); let n = rng.size(32).min(frames.len() - from.max(at)); frames.copy_within(from..from + n, at) } + } + } +} + +fn check_sizes(frames: &[u8]) { + let p = frames.as_ptr().cast(); + unsafe { + assert_eq!(c::ZSTD_findDecompressedSize(p, frames.len()), z::find_decompressed_size(frames), "findDecompressedSize"); + assert_eq!(c::ZSTD_getFrameContentSize(p, frames.len()), z::get_frame_content_size(frames), "getFrameContentSize"); + assert_eq!(c::res(c::ZSTD_findFrameCompressedSize(p, frames.len())), rs(z::decompress::find_frame_compressed_size(frames)), "findFrameCompressedSize"); + } +} + +fn one_shot(frames: &[u8], dict: &[u8], cap: usize, prefix: bool) { + let mut cd = vec![0xAAu8; cap]; + let mut rd = vec![0xAAu8; cap]; + let mut cctx = c::DCtx::new(); + let mut rctx = z::DCtx::new(); + let (cr, rr); + if prefix { + cctx.ref_prefix(dict).unwrap(); + cr = cctx.decompress(&mut cd, frames); + // an empty prefix is no dictionary at all + rr = if dict.is_empty() { rs(rctx.decompress(&mut rd, frames)) } else { rs(z::DDict::prefix(dict).and_then(|d| rctx.decompress_using_ddict(&mut rd, frames, &d))) }; + } else { + assert_eq!(cctx.load_dict(dict).map(|_| ()), rs(rctx.load_dictionary(dict)), "loadDictionary"); + cr = cctx.decompress(&mut cd, frames); + rr = rs(rctx.decompress(&mut rd, frames)); + } + assert_eq!(cr, rr, "one-shot result (cap {cap})"); + if let Ok(n) = cr { assert!(cd[..n] == rd[..n], "one-shot output"); } +} + +fn streaming(rng: &mut Rng, frames: &[u8], dict: &[u8], expect: Option<&[u8]>) { + let mut cctx = c::DCtx::new(); + let mut rctx = z::DCtx::new(); + if rng.chance(1, 6) { + let (p, v) = (*rng.pick(&[100u32, 100, 100, 0, 99, 1]), *rng.pick(&[0i32, 9, 10, 15, 20, 27, 31, 32, -1])); + assert_eq!(cctx.set(p, v).map(|_| ()), rs(rctx.set_parameter(p as i32, v)), "setParameter {p} {v}"); + } + assert_eq!(cctx.load_dict(dict).map(|_| ()), rs(rctx.load_dictionary(dict)), "loadDictionary"); + // tiny steps only where they do not take forever + let small = frames.len() < 3_000; + let in_max = *rng.pick(if small { &[1usize, 2, 3, 7, 64, 1000, 70_000, 10_000_000] } else { &[300usize, 1000, 5000, 20_000, 70_000, 200_000, 10_000_000, 10_000_000] }); + let out_max = *rng.pick(if small && expect.is_none_or(|e| e.len() < 20_000) { &[1usize, 2, 5, 100, 4096, 16_384, 140_000, 10_000_000] } else { &[1000usize, 4096, 4096, 16_384, 16_384, 140_000, 1_000_000, 10_000_000] }); + let mut total = Vec::new(); + let (mut cbuf, mut rbuf) = (Vec::new(), Vec::new()); + let mut ipos = 0usize; + let mut stalls = 0; + loop { + let iend = (ipos + rng.range(if stalls > 0 { 1 } else { 0 }, in_max)).min(frames.len()); + let cap = rng.range(if stalls > 1 { 1 } else { 0 }, out_max); + let chunk = &frames[ipos..iend]; + if cbuf.len() < cap { cbuf.resize(cap, 0x55); rbuf.resize(cap, 0x55); } + let cd = &mut cbuf[..cap]; + let rd = &mut rbuf[..cap]; + let (cr, co, ci) = cctx.stream(cd, 0, chunk, 0); + let mut ob = z::OutBuffer { dst: rd, pos: 0 }; + let mut ib = z::InBuffer { src: chunk, pos: 0 }; + let rr = rs(rctx.decompress_stream(&mut ob, &mut ib)); + let (ro, ri) = (ob.pos, ib.pos); + assert_eq!((cr, co, ci), (rr, ro, ri), "stream step at {ipos} (in {} out {cap})", chunk.len()); + assert!(cbuf[..co] == rbuf[..ro], "stream output at {ipos}"); + total.extend_from_slice(&rbuf[..ro]); + ipos += ri; + if cr.is_err() { return; } + if ri == 0 && ro == 0 { stalls += 1 } else { stalls = 0 } + if ipos == frames.len() && (cr == Ok(0) || stalls > 3) { break; } + if cr == Ok(0) && rng.chance(1, 8) { + match rng.below(2) { + 0 => { cctx.init_stream(); rctx.init_stream(); if !dict.is_empty() { return; } } + _ => {} + } + } + } + if let Some(expect) = expect { assert!(total == expect, "stream total"); } +} + +pub fn run(seed: u64, iters: u64) { + for i in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(i); + let mut rng = Rng(s); + if i % 200 == 0 { eprintln!("dec iter {i} (seed {s})"); } + let (mut frames, content, dict) = make_frames(&mut rng); + // valid + check_sizes(&frames); + one_shot(&frames, &dict, content.len(), false); + one_shot(&frames, &dict, content.len() + rng.size(300_000), false); + if !content.is_empty() { let cap = rng.below(content.len()); one_shot(&frames, &dict, cap, false); } + streaming(&mut rng, &frames, &dict, Some(&content)); + streaming(&mut rng, &frames, &dict, Some(&content)); + // wrong or missing dictionary + if rng.chance(1, 4) { let n = rng.size(5000); let other = data(&mut rng, n); one_shot(&frames, &other, content.len(), rng.chance(1, 2)); streaming(&mut rng, &frames, &other, None); } + // damaged + for _ in 0..6 { + let mut bad = frames.clone(); + mutate(&mut rng, &mut bad); + check_sizes(&bad); + let cap = content.len() + if rng.chance(1, 2) { rng.size(200_000) } else { 0 }; + one_shot(&bad, &dict, cap, false); + streaming(&mut rng, &bad, &dict, None); + } + mutate(&mut rng, &mut frames); + } + println!("dec ok: {iters} iterations from seed {seed}"); +} diff --git a/src/zstd/wip/harness/src/extra.rs b/src/zstd/wip/harness/src/extra.rs new file mode 100644 index 000000000000..23be98f8ce66 --- /dev/null +++ b/src/zstd/wip/harness/src/extra.rs @@ -0,0 +1,556 @@ +//! Extra differential checks. +use crate::c; +use crate::data::{Rng, data}; +use bun_zstd as z; + +fn r2(r: Result) -> Result { r.map_err(|e| e as u32) } +fn r0(r: Result<(), z::ErrorCode>) -> Result { r.map(|()| 0).map_err(|e| e as u32) } +fn ed(e: u32) -> z::EndDirective { match e { 0 => z::EndDirective::Continue, 1 => z::EndDirective::Flush, _ => z::EndDirective::End } } + +fn load(dir: &str) -> Vec> { + let mut names: Vec<_> = std::fs::read_dir(dir).unwrap().map(|e| e.unwrap().path()).collect(); + names.sort(); + names.iter().map(|p| std::fs::read(p).unwrap()).collect() +} + +pub fn tdict(seed: u64, iters: u64, dict_dir: &str, sample_dir: &str) { + let dicts = load(dict_dir); + let samples = load(sample_dir); + for it in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(it); + if let Ok(only) = std::env::var("ONLY") { if only.parse::().unwrap() != s { continue; } } + let mut rng = Rng(s); + let rng = &mut rng; + if it % 100 == 0 { eprintln!("iter {it}"); } + let mut dict = rng.pick(&dicts).clone(); + if rng.chance(1, 10) { + // damage + let at = rng.below(dict.len().min(400)); + dict[at] ^= 1 << rng.below(8); + } + if rng.chance(1, 15) { let n = rng.below(dict.len()); dict.truncate(n); } + let mut a = c::CCtx::new(); + let mut b = z::CCtx::new(); + let mut log: Vec = vec![format!("seed {s} dict {}", dict.len())]; + let mt = rng.chance(1, 8); + let frames = rng.range(1, 4); + let mut all_out: Vec = Vec::new(); + let mut all_src: Vec = Vec::new(); + let mut ok_frames = true; + { + let mut set = |p: i32, v: i32, log: &mut Vec| { + let ra = a.set(p as u32, v); + let rb = r2(b.set_parameter(p, v)); + log.push(format!("set {p} {v} -> {ra:?}")); + assert_eq!(ra, rb, "{log:?}"); + }; + set(100, *rng.pick(&[-7, -1, 1, 2, 3, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 17, 18, 19, 22]), &mut log); + if rng.chance(1, 3) { set(107, rng.range(1, 9) as i32, &mut log); } + if rng.chance(1, 4) { set(101, rng.range(10, 24) as i32, &mut log); } + if rng.chance(1, 4) { set(201, 1, &mut log); } + if rng.chance(1, 5) { set(202, 0, &mut log); } + if rng.chance(1, 5) { set(200, 0, &mut log); } + if rng.chance(1, 8) { set(160, 1, &mut log); } + if rng.chance(1, 8) { set(130, rng.range(1340, 9000) as i32, &mut log); } + if rng.chance(1, 8) { set(105, rng.range(3, 7) as i32, &mut log); } + if mt { set(400, rng.range(1, 3) as i32, &mut log); } + } + let ra = a.load_dict(&dict); + let rb = r0(b.load_dictionary(&dict)); + assert_eq!(ra, rb, "{log:?}"); + for _ in 0..frames { + let mut src = Vec::new(); + let target = if mt { rng.range(600_000, 3_000_000) } else { *rng.pick(&[0usize, 50, 500, 3000, 9000, 20_000, 40_000, 200_000, 700_000]) }; + let target = if mt { target } else { rng.size(target) }; + while src.len() < target { + if rng.chance(1, 6) { let n = rng.size(3000); src.extend_from_slice(&data(rng, n)); continue; } + let f = rng.pick(&samples); + let st = rng.below(f.len()); + let n = rng.size(20000).min(f.len() - st); + src.extend_from_slice(&f[st..st + n]); + } + src.truncate(target); + let len = src.len(); + if rng.chance(1, 3) { + let ra = a.pledge(len as u64); let rb = r0(b.set_pledged_src_size(len as u64)); + assert_eq!(ra, rb); log.push(format!("pledge {len}")); + } + log.push(format!("frame {len}")); + let mode = rng.below(3); + let mut pos = 0usize; + let mut oa_all = Vec::new(); + let mut ob_all = Vec::new(); + let mut failed = false; + loop { + let left = len - pos; + let chunk = match mode { 0 => left, 1 => rng.size(100_000).min(left), _ => rng.range(0, 5000).min(left) }; + let last = pos + chunk == len; + let end = if last { 2 } else { *rng.pick(&[0u32, 0, 0, 1]) }; + let input = &src[pos..pos + chunk]; + let cap = z::compress_bound(len.max(1 << 17)) + 1024; + let mut buf = vec![0u8; cap]; + let (mut ea, mut eb) = (None, None); + let mut ip = 0; + loop { + let (r, op, np) = a.stream2(&mut buf, 0, input, ip, end); + oa_all.extend_from_slice(&buf[..op]); ip = np; + match r { Err(e) => { ea = Some(e); break; } Ok(l) => if (end == 0 && ip == chunk) || (end != 0 && l == 0) { break; } } + } + let mut ip = 0; + loop { + let mut o = z::OutBuffer { dst: &mut buf, pos: 0 }; + let mut i = z::InBuffer { src: input, pos: ip }; + let r = r2(b.compress_stream2(&mut o, &mut i, ed(end))); + let op = o.pos; ip = i.pos; + ob_all.extend_from_slice(&buf[..op]); + match r { Err(e) => { eb = Some(e); break; } Ok(l) => if (end == 0 && ip == chunk) || (end != 0 && l == 0) { break; } } + } + assert_eq!(ea, eb, "{log:?}"); + if ea.is_some() { failed = true; let _ = a.reset(1); b.reset_session(); break; } + pos += chunk; + if last { break; } + } + if std::env::var("HASH").ok().and_then(|h| h.parse::().ok()) == Some(s) { + let h = |d: &[u8]| d.iter().fold(0xcbf29ce484222325u64, |h, &b| (h ^ b as u64).wrapping_mul(0x100000001b3)); + eprintln!("HASH seed {s} C {:x} Rust {:x}", h(&oa_all), h(&ob_all)); + } + if oa_all != ob_all { + let at = oa_all.iter().zip(&ob_all).position(|(x, y)| x != y); + if let Ok(d) = std::env::var("DUMP") { + std::fs::write(format!("{d}/prev.bin"), &all_src).unwrap(); + std::fs::write(format!("{d}/src.bin"), &src).unwrap(); + std::fs::write(format!("{d}/dict.bin"), &dict).unwrap(); + std::fs::write(format!("{d}/c.zst"), &oa_all).unwrap(); + std::fs::write(format!("{d}/rust.zst"), &ob_all).unwrap(); + } + panic!("tdict: {} vs {} first diff {at:?}\n{log:?}", oa_all.len(), ob_all.len()); + } + if failed { ok_frames = false; break; } + all_out.extend_from_slice(&oa_all); + all_src.extend_from_slice(&src); + } + // decode + let mut frames_bytes = all_out.clone(); + for round in 0..4 { + if round > 0 { + if frames_bytes.is_empty() { break; } + frames_bytes = all_out.clone(); + for _ in 0..1 + rng.below(3) { + let at = rng.below(frames_bytes.len()); + frames_bytes[at] ^= 1 << rng.below(8); + } + } + let cap = all_src.len() + rng.size(1000); + let (mut cd, mut rd) = (vec![0u8; cap], vec![0u8; cap]); + let mut ca = c::DCtx::new(); + let mut cb = z::DCtx::new(); + let la = ca.load_dict(&dict).map(|_| ()); + let lb = cb.load_dictionary(&dict).map_err(|e| e as u32); + assert_eq!(la, lb, "dctx load {log:?}"); + let x = ca.decompress(&mut cd, &frames_bytes); + let y = r2(cb.decompress(&mut rd, &frames_bytes)); + assert_eq!(x, y, "decode round {round} {log:?}"); + if let Ok(n) = x { assert!(cd[..n] == rd[..n]); if round == 0 && ok_frames && la.is_ok() { assert!(cd[..n] == all_src[..], "roundtrip {log:?}"); } } + // by reference ddict + if let Ok(dd) = z::DDict::by_reference(&dict) { + let y2 = r2(z::DCtx::new().decompress_using_ddict(&mut rd, &frames_bytes, &dd)); + assert_eq!(x, y2, "ddict {log:?}"); + } + // streaming + let mut ca = c::DCtx::new(); + let mut cb = z::DCtx::new(); + let _ = ca.load_dict(&dict); let _ = cb.load_dictionary(&dict); + let mut ipos = 0; let mut stalls = 0; + let small = frames_bytes.len() < 4000 && all_src.len() < 30_000; + let inmax = *rng.pick(if small { &[7usize, 300, 5000, 100_000, 10_000_000] } else { &[3000usize, 20_000, 5000, 100_000, 10_000_000] }); + let outmax = *rng.pick(if small { &[50usize, 4096, 16384, 200_000, 1_000_000] } else { &[5000usize, 4096, 16384, 200_000, 1_000_000] }); + let (mut x1f, mut x2f) = (vec![0u8; outmax], vec![0u8; outmax]); + loop { + let iend = (ipos + rng.range(if stalls > 0 {1} else {0}, inmax)).min(frames_bytes.len()); + let cap = rng.range(if stalls > 1 {1} else {0}, outmax); + let chunk = &frames_bytes[ipos..iend]; + let (x1, x2) = (&mut x1f[..cap], &mut x2f[..cap]); + let (cr, co, ci) = ca.stream(x1, 0, chunk, 0); + let mut ob = z::OutBuffer { dst: x2, pos: 0 }; + let mut ib = z::InBuffer { src: chunk, pos: 0 }; + let rr = r2(cb.decompress_stream(&mut ob, &mut ib)); + let (ro, ri) = (ob.pos, ib.pos); + assert_eq!((cr, co, ci), (rr, ro, ri), "stream {log:?}"); + assert!(x1[..co] == x2[..ro]); + ipos += ri; + if cr.is_err() { break; } + if ri == 0 && ro == 0 { stalls += 1 } else { stalls = 0 } + if ipos == frames_bytes.len() && (cr == Ok(0) || stalls > 3) { break; } + } + } + } + println!("tdict ok seed {seed} iters {iters}"); +} + +/// A very long frame, so that indices are brought down. +pub fn long(seed: u64, total: u64, level: i32, wlog: i32, extra: &[(i32, i32)]) { + let mut rng = Rng(seed); + let mut a = c::CCtx::new(); + let mut b = z::CCtx::new(); + a.set(100, level).unwrap(); b.set_parameter(100, level).unwrap(); + if wlog != 0 { a.set(101, wlog).unwrap(); b.set_parameter(101, wlog).unwrap(); } + a.set(201, 1).unwrap(); b.set_parameter(201, 1).unwrap(); + for &(p, v) in extra { assert_eq!(a.set(p as u32, v), r2(b.set_parameter(p, v))); } + let base = data(&mut rng, 8 << 20); + let mut done = 0u64; + let cap = z::compress_bound(4 << 20) + 1024; + let (mut oa, mut ob) = (vec![0u8; cap], vec![0u8; cap]); + let mut da = c::DCtx::new(); + let mut db = z::DCtx::new(); + da.set(100, 31).unwrap(); db.set_parameter(100, 31).unwrap(); + let mut dec_a = vec![0u8; 8 << 20]; + let mut dec_b = vec![0u8; 8 << 20]; + let mut hash_src = 0u64; let mut hash_dec = 0u64; + let fnv = |h: u64, d: &[u8]| { let mut h = h; for c in d.chunks(4096) { h = (h ^ c[0] as u64 ^ ((c[c.len()-1] as u64) << 8) ^ ((c.len() as u64) << 16)).wrapping_mul(0x100000001b3); } h }; + let _ = fnv; + let mut srcsum = 0u64; let mut decsum = 0u64; + while done < total { + let st = rng.below(base.len() - (1 << 20)); + let n = rng.range(1, 1 << 20); + let mut chunk = base[st..st + n].to_vec(); + if rng.chance(1, 2) { let at = rng.below(n); chunk[at] = rng.next() as u8; } + done += n as u64; + let end = if done >= total { 2 } else { 0 }; + for &x in chunk.iter().step_by(97) { srcsum = srcsum.wrapping_mul(31).wrapping_add(x as u64); } + let mut ip = 0; + loop { + let (ra, pa, ia) = a.stream2(&mut oa, 0, &chunk, ip, end); + let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; + let mut i = z::InBuffer { src: &chunk, pos: ip }; + let rb = r2(b.compress_stream2(&mut o, &mut i, ed(end))); + let (pb, ib) = (o.pos, i.pos); + assert_eq!((ra, pa, ia), (rb, pb, ib), "long at {done}"); + assert!(oa[..pa] == ob[..pb], "long bytes at {done}"); + ip = ia; + // decode + let mut dp = 0; + loop { + let (cr, co, ci) = da.stream(&mut dec_a, 0, &oa[..pa], dp); + let mut o2 = z::OutBuffer { dst: &mut dec_b, pos: 0 }; + let mut i2 = z::InBuffer { src: &oa[..pa], pos: dp }; + let rr = r2(db.decompress_stream(&mut o2, &mut i2)); + assert_eq!((cr, co, ci), (rr, o2.pos, i2.pos), "long dec at {done}"); + assert!(dec_a[..co] == dec_b[..co], "long dec bytes at {done}"); + cr.unwrap(); + dp = ci; + hash_dec = hash_dec.wrapping_add(co as u64); + let _ = &mut decsum; + if dp == pa && co < dec_a.len() { break; } + } + if (end == 0 && ip == chunk.len()) || (end == 2 && ra == Ok(0)) { break; } + } + hash_src = hash_src.wrapping_add(n as u64); + } + assert_eq!(hash_src, hash_dec, "sizes"); + println!("long ok: {done} bytes level {level} wlog {wlog} {extra:?} {srcsum}"); +} + +/// Two frames on one context with a dictionary. +pub fn repro(dict_path: &str, sample: &str, level: i32, wlog: i32, n1: usize, n2: usize, pledge: bool) { + let dict = std::fs::read(dict_path).unwrap(); + let f = std::fs::read(sample).unwrap(); + let mut a = c::CCtx::new(); + let mut b = z::CCtx::new(); + a.set(100, level).unwrap(); b.set_parameter(100, level).unwrap(); + if wlog != 0 { a.set(101, wlog).unwrap(); b.set_parameter(101, wlog).unwrap(); } + a.load_dict(&dict).unwrap(); b.load_dictionary(&dict).unwrap(); + for (k, &n) in [n1, n2].iter().enumerate() { + let src = &f[..n.min(f.len())]; + if pledge { a.pledge(src.len() as u64).unwrap(); b.set_pledged_src_size(src.len() as u64).unwrap(); } + let cap = z::compress_bound(src.len()) + 100; + let (mut oa, mut ob) = (vec![0u8; cap], vec![0u8; cap]); + let (ra, pa, _) = a.stream2(&mut oa, 0, src, 0, 2); + let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; + let mut i = z::InBuffer { src, pos: 0 }; + let rb = r2(b.compress_stream2(&mut o, &mut i, z::EndDirective::End)); + let pb = o.pos; + let same = oa[..pa] == ob[..pb]; + let at = oa[..pa].iter().zip(&ob[..pb]).position(|(x, y)| x != y); + println!("frame {k}: len {} C {ra:?} {pa} Rust {rb:?} {pb} same {same} first diff {at:?}", src.len()); + if !same { + println!(" C : {:02x?}", &oa[..40.min(pa)]); + println!(" Rust: {:02x?}", &ob[..40.min(pb)]); + let ndiff = oa[..pa].iter().zip(&ob[..pb]).filter(|(x, y)| x != y).count(); + println!(" bytes differing: {ndiff}"); + // does the Rust frame decode? + let mut d = c::DCtx::new(); d.load_dict(&dict).unwrap(); + let mut out = vec![0u8; src.len()]; + let r = d.decompress(&mut out, &ob[..pb]); + println!(" C decoder on the Rust frame: {r:?} content ok {}", out == src); + } + } +} + +pub fn bench(path: &str) { + use std::time::Instant; + let src = std::fs::read(path).unwrap(); + let cap = z::compress_bound(src.len()); + let mut out_c = vec![0u8; cap]; + let mut out_r = vec![0u8; cap]; + let mut dec = vec![0u8; src.len()]; + println!("corpus {} bytes", src.len()); + for level in [1, 3, 5, 7, 9, 12, 15, 19] { + let reps = if level >= 15 { 1 } else if level >= 9 { 2 } else { 4 }; + let mut tc = f64::MAX; let mut tr = f64::MAX; let mut nc = 0; let mut nr = 0; + for _ in 0..reps { + let t = Instant::now(); + nc = c::res(unsafe { c::ZSTD_compress(out_c.as_mut_ptr().cast(), cap, src.as_ptr().cast(), src.len(), level) }).unwrap(); + tc = tc.min(t.elapsed().as_secs_f64()); + let t = Instant::now(); + nr = z::compress(&mut out_r, &src, level).unwrap(); + tr = tr.min(t.elapsed().as_secs_f64()); + } + assert!(out_c[..nc] == out_r[..nr]); + let mut dc = f64::MAX; let mut dr = f64::MAX; + for _ in 0..5 { + let t = Instant::now(); + c::decompress(&mut dec, &out_c[..nc]).unwrap(); + dc = dc.min(t.elapsed().as_secs_f64()); + let t = Instant::now(); + z::decompress(&mut dec, &out_c[..nc]).unwrap(); + dr = dr.min(t.elapsed().as_secs_f64()); + } + let mb = src.len() as f64 / 1e6; + println!("level {level:2}: compress C {:7.1} MB/s Rust {:7.1} MB/s ({:.2}x slower) | decompress C {:7.1} MB/s Rust {:7.1} MB/s ({:.2}x slower)", mb / tc, mb / tr, tr / tc, mb / dc, mb / dr, dr / dc); + } +} + +/// Many frames on one context, so that indices get close to the limit between frames. +pub fn frames(seed: u64, total: u64, level: i32, max_frame: usize, oneshot_end: bool) { + let mut rng = Rng(seed); + let mut a = c::CCtx::new(); + let mut b = z::CCtx::new(); + a.set(100, level).unwrap(); b.set_parameter(100, level).unwrap(); + let base = data(&mut rng, max_frame + (1 << 20)); + let cap = z::compress_bound(max_frame) + 1024; + let (mut oa, mut ob) = (vec![0u8; cap], vec![0u8; cap]); + let mut done = 0u64; let mut n = 0; + while done < total { + let len = rng.range(max_frame / 2, max_frame); + let st = rng.below(base.len() - len); + let src = &base[st..st + len]; + if oneshot_end { + let (ra, pa, ia) = a.stream2(&mut oa, 0, src, 0, 2); + let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; + let mut i = z::InBuffer { src, pos: 0 }; + let rb = r2(b.compress_stream2(&mut o, &mut i, z::EndDirective::End)); + assert_eq!((ra, pa, ia), (rb, o.pos, i.pos), "frame {n} at {done}"); + assert!(oa[..pa] == ob[..pa], "frame {n} bytes at {done}"); + } else { + let mut pos = 0; + loop { + let chunk = (1 << 20).min(len - pos); + let end = if pos + chunk == len { 2 } else { 0 }; + let mut ip = 0; + loop { + let (ra, pa, ia) = a.stream2(&mut oa, 0, &src[pos..pos + chunk], ip, end); + let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; + let mut i = z::InBuffer { src: &src[pos..pos + chunk], pos: ip }; + let rb = r2(b.compress_stream2(&mut o, &mut i, ed(end))); + assert_eq!((ra, pa, ia), (rb, o.pos, i.pos), "frame {n} at {done}"); + assert!(oa[..pa] == ob[..pa], "frame {n} bytes at {done}"); + ip = ia; + if (end == 0 && ip == chunk) || (end == 2 && ra == Ok(0)) { break; } + } + pos += chunk; + if end == 2 { break; } + } + } + done += len as u64; n += 1; + } + println!("frames ok: {n} frames {done} bytes level {level} oneshot {oneshot_end}"); +} + +/// One call on more than 4 GB. +pub fn huge(seed: u64, total: usize, level: i32) { + let mut rng = Rng(seed); + let base = data(&mut rng, 16 << 20); + let mut src = Vec::with_capacity(total); + while src.len() < total { + let st = rng.below(base.len() - (1 << 20)); + let n = rng.range(1, 1 << 20).min(total - src.len()); + src.extend_from_slice(&base[st..st + n]); + } + let cap = z::compress_bound(total); + let mut oa = vec![0u8; cap]; + let ra = c::res(unsafe { c::ZSTD_compress(oa.as_mut_ptr().cast(), cap, src.as_ptr().cast(), total, level) }); + let mut ob = vec![0u8; cap]; + let rb = r2(z::compress(&mut ob, &src, level)); + assert_eq!(ra, rb); + let n = ra.unwrap(); + assert!(oa[..n] == ob[..n], "huge bytes"); + drop(ob); + let mut dec = vec![0u8; total]; + let rd = r2(z::decompress(&mut dec, &oa[..n])); + assert_eq!(rd, Ok(total)); + assert!(dec == src, "huge roundtrip"); + let v = z::decompress_alloc(&oa[..n]).unwrap(); + assert!(v == src, "huge decompress_alloc"); + println!("huge ok {total} -> {n} level {level}"); +} + +/// Multithreaded frames that are given up, small outputs, contexts dropped in the middle. +pub fn mtx(seed: u64, iters: u64) { + for it in 0..iters { + let s = seed.wrapping_mul(1_000_003).wrapping_add(it); + let mut rng = Rng(s); + let rng = &mut rng; + if it % 50 == 0 { eprintln!("iter {it}"); } + let mut a = c::CCtx::new(); + let mut b = z::CCtx::new(); + let mut log = vec![format!("seed {s}")]; + let w = rng.range(1, 6) as i32; + for (p, v) in [(400, w), (100, *rng.pick(&[1, 1, 3, 3, 5, 7, 9, 13])), (201, rng.below(2) as i32), (401, *rng.pick(&[0, 1, 600_000, 1_000_000])), (101, *rng.pick(&[0, 0, 12, 17, 20])), (160, *rng.pick(&[0, 0, 0, 1]))] { + assert_eq!(a.set(p as u32, v), r2(b.set_parameter(p, v))); + log.push(format!("set {p} {v}")); + } + let base = data(rng, 6 << 20); + let nframes = rng.range(1, 4); + for f in 0..nframes { + let len = if rng.chance(1, 5) { rng.size(600_000) } else { rng.range(600_000, base.len()) }; + let src = &base[..len]; + let give_up_at = if rng.chance(1, 3) { Some(rng.below(len + 1)) } else { None }; + let pledge = rng.chance(1, 3) && give_up_at.is_none(); + if pledge { a.pledge(len as u64).unwrap(); b.set_pledged_src_size(len as u64).unwrap(); } + log.push(format!("frame {f} len {len} give up {give_up_at:?} pledge {pledge}")); + let outcap = *rng.pick(&[1usize, 7, 100, 4096, 16384, 1 << 20]); + let outcap = if len > 2_000_000 && outcap < 100 { 4096 } else { outcap }; + let inmax = *rng.pick(&[1000usize, 16384, 65536, 1 << 20, 8 << 20]); + let run = |which: u8, a: &mut c::CCtx, b: &mut z::CCtx, seed2: u64| -> (Vec, Option) { + let mut r = Rng(seed2); + let mut out = Vec::new(); + let mut buf = vec![0u8; outcap]; + let mut pos = 0; + loop { + let stop = give_up_at.unwrap_or(usize::MAX); + if pos >= stop { return (out, None); } + let chunk = r.range(1, inmax).min(len - pos).min(stop - pos); + let last = pos + chunk == len; + let end = if last { 2 } else { *r.pick(&[0u32, 0, 0, 0, 0, 1]) }; + let input = &src[pos..pos + chunk]; + let mut ip = 0; + loop { + let (res, op, np) = if which == 0 { a.stream2(&mut buf, 0, input, ip, end) } else { + let mut o = z::OutBuffer { dst: &mut buf, pos: 0 }; + let mut i = z::InBuffer { src: input, pos: ip }; + let res = r2(b.compress_stream2(&mut o, &mut i, ed(end))); + (res, o.pos, i.pos) + }; + out.extend_from_slice(&buf[..op]); ip = np; + match res { Err(e) => return (out, Some(e)), Ok(l) => if (end == 0 && ip == chunk) || (end != 0 && l == 0) { break; } } + } + pos += chunk; + if last { return (out, None); } + } + }; + let seed2 = rng.next(); + let (oa, ea) = run(0, &mut a, &mut b, seed2); + let (ob, eb) = run(1, &mut a, &mut b, seed2); + assert_eq!(ea, eb, "{log:?}"); + if give_up_at.is_none() && ea.is_none() { + // Flushes make sections whose cut depends on timing only when the job table is full; compare by decoding. + let mut dec = vec![0u8; len]; + assert_eq!(c::decompress(&mut dec, &ob), Ok(len), "{log:?}"); + assert!(dec == src, "roundtrip {log:?}"); + if oa != ob { log.push(format!("(frames differ: {} vs {})", oa.len(), ob.len())); eprintln!("differs: {log:?}"); } + } else { + if rng.chance(1, 4) { break; } // drop both in the middle of the frame + a.reset(1).unwrap(); b.reset_session(); + } + } + } + println!("mtx ok seed {seed} iters {iters}"); +} + +pub fn bench_small(path: &str) { + use std::time::Instant; + let all = std::fs::read(path).unwrap(); + for n in [0usize, 50, 300, 1000, 4000, 16000, 64000] { + for level in [1, 3, 9] { + let src = &all[1000..1000 + n]; + let cap = z::compress_bound(n); + let mut oc = vec![0u8; cap]; let mut or = vec![0u8; cap]; + let iters = 20000.min(40_000_000 / (n + 200)); + let mut nc = 0; let mut nr = 0; + let t = Instant::now(); + for _ in 0..iters { nc = c::res(unsafe { c::ZSTD_compress(oc.as_mut_ptr().cast(), cap, src.as_ptr().cast(), n, level) }).unwrap(); } + let tc = t.elapsed().as_secs_f64() / iters as f64; + let t = Instant::now(); + for _ in 0..iters { nr = z::compress(&mut or, src, level).unwrap(); } + let tr = t.elapsed().as_secs_f64() / iters as f64; + assert!(oc[..nc] == or[..nr]); + let mut dec = vec![0u8; n]; + let t = Instant::now(); + for _ in 0..iters { c::decompress(&mut dec, &oc[..nc]).unwrap(); } + let dc = t.elapsed().as_secs_f64() / iters as f64; + let t = Instant::now(); + for _ in 0..iters { z::decompress(&mut dec, &oc[..nc]).unwrap(); } + let dr = t.elapsed().as_secs_f64() / iters as f64; + println!("n {n:6} level {level}: compress C {:8.2} us Rust {:8.2} us ({:.2}x) | decompress C {:7.2} us Rust {:7.2} us ({:.2}x)", tc * 1e6, tr * 1e6, tr / tc, dc * 1e6, dr * 1e6, dr / dc); + } + } +} + +pub fn bench_mt(path: &str) { + use std::time::Instant; + let text = std::fs::read(path).unwrap(); + let mut rng = Rng(7); + // sections that take very different times: text, noise, zeros + let mut src = Vec::new(); + let sec = 1 << 20; + for k in 0..192 { + match k % 3 { 0 => { let st = rng.below(text.len() - sec); src.extend_from_slice(&text[st..st + sec]); } + 1 => { for _ in 0..sec / 8 { src.extend_from_slice(&rng.next().to_le_bytes()); } } + _ => src.resize(src.len() + sec, 0) } + } + let uniform: Vec = text.iter().cycle().take(src.len()).copied().collect(); + let cap = z::compress_bound(src.len()); + let mut out = vec![0u8; cap]; + for (name, data) in [("uneven", &src), ("uniform", &uniform)] { + for level in [3, 9] { + for w in [0, 1, 2, 4, 8] { + let mut best = [f64::MAX; 2]; + for _ in 0..2 { + let mut a = c::CCtx::new(); + a.set(100, level).unwrap(); a.set(400, w).unwrap(); a.set(401, 1 << 20).unwrap(); + let t = Instant::now(); + let (r, _, _) = a.stream2(&mut out, 0, data, 0, 2); assert_eq!(r, Ok(0)); + best[0] = best[0].min(t.elapsed().as_secs_f64()); + let mut b = z::CCtx::new(); + b.set_parameter(100, level).unwrap(); b.set_parameter(400, w).unwrap(); b.set_parameter(401, 1 << 20).unwrap(); + let t = Instant::now(); + let mut o = z::OutBuffer { dst: &mut out, pos: 0 }; + let mut i = z::InBuffer { src: data, pos: 0 }; + assert_eq!(b.compress_stream2(&mut o, &mut i, z::EndDirective::End), Ok(0)); + best[1] = best[1].min(t.elapsed().as_secs_f64()); + } + println!("{name:8} level {level} workers {w}: C {:6.0} ms Rust {:6.0} ms ({:.2}x)", best[0] * 1e3, best[1] * 1e3, best[1] / best[0]); + } + } + } +} + +pub fn bench_mt_flush(path: &str) { + use std::time::Instant; + let text = std::fs::read(path).unwrap(); + let mut out = vec![0u8; 1 << 20]; + for (level, wlog) in [(3, 0), (19, 0), (19, 27)] { + let n = 5000usize; + let mut a = c::CCtx::new(); + a.set(100, level).unwrap(); a.set(400, 2).unwrap(); if wlog != 0 { a.set(101, wlog).unwrap(); } + let t = Instant::now(); + for k in 0..n { let input = &text[k * 1000..k * 1000 + 1000]; let mut ip = 0; loop { let (r, _, np) = a.stream2(&mut out, 0, input, ip, 1); ip = np; if r.unwrap() == 0 { break; } } } + let tc = t.elapsed().as_secs_f64(); + let mut b = z::CCtx::new(); + b.set_parameter(100, level).unwrap(); b.set_parameter(400, 2).unwrap(); if wlog != 0 { b.set_parameter(101, wlog).unwrap(); } + let t = Instant::now(); + for k in 0..n { let input = &text[k * 1000..k * 1000 + 1000]; let mut ip = 0; loop { let mut o = z::OutBuffer { dst: &mut out, pos: 0 }; let mut i = z::InBuffer { src: input, pos: ip }; let r = b.compress_stream2(&mut o, &mut i, z::EndDirective::Flush).unwrap(); ip = i.pos; if r == 0 { break; } } } + let tr = t.elapsed().as_secs_f64(); + println!("level {level} wlog {wlog}: {n} flushes of 1000 B with 2 workers: C {:.1} us/flush, Rust {:.1} us/flush ({:.2}x)", tc / n as f64 * 1e6, tr / n as f64 * 1e6, tr / tc); + } +} diff --git a/src/zstd/wip/harness/src/main.rs b/src/zstd/wip/harness/src/main.rs new file mode 100644 index 000000000000..b4e43e2c2a8b --- /dev/null +++ b/src/zstd/wip/harness/src/main.rs @@ -0,0 +1,31 @@ +mod c; +mod comp; +mod data; +mod dec; +mod extra; +mod sizes; + +fn main() { + let args: Vec = std::env::args().collect(); + let cmd = args.get(1).map(String::as_str).unwrap_or(""); + let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); + let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); + match cmd { + "dec" => dec::run(seed, iters), + "sizes" => sizes::run(), + "bench_mt_flush" => extra::bench_mt_flush(&args[2]), + "bench_mt" => extra::bench_mt(&args[2]), + "bench_small" => extra::bench_small(&args[2]), + "mtx" => extra::mtx(seed, iters), + "frames" => extra::frames(seed, iters, args[4].parse().unwrap(), args[5].parse().unwrap(), args[6] == "1"), + "huge" => extra::huge(seed, iters as usize, args[4].parse().unwrap()), + "bench" => extra::bench(&args[2]), + "repro" => extra::repro(&args[2], &args[3], args[4].parse().unwrap(), args[5].parse().unwrap(), args[6].parse().unwrap(), args[7].parse().unwrap(), args[8] == "1"), + "tdict" => extra::tdict(seed, iters, &args[4], &args[5]), + "long" => { let ex: Vec<(i32,i32)> = args[6..].chunks(2).map(|c| (c[0].parse().unwrap(), c[1].parse().unwrap())).collect(); extra::long(seed, iters, args[4].parse().unwrap(), args[5].parse().unwrap(), &ex) } + "dict" => comp::run_dict(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(1 << 18)), + "mt" => comp::run_mt(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(8 << 20)), + "comp" => comp::run(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(1 << 20)), + _ => eprintln!("usage: zstd_harness dec "), + } +} diff --git a/src/zstd/wip/harness/src/sizes.rs b/src/zstd/wip/harness/src/sizes.rs new file mode 100644 index 000000000000..8e0a3dc711af --- /dev/null +++ b/src/zstd/wip/harness/src/sizes.rs @@ -0,0 +1 @@ +pub fn run() { println!("CCtx {} DCtx {}", size_of::(), size_of::()); } diff --git a/src/zstd/wip/harness/sync.sh b/src/zstd/wip/harness/sync.sh new file mode 100755 index 000000000000..e2d37754d968 --- /dev/null +++ b/src/zstd/wip/harness/sync.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# A copy of the crate with everything public, so that the harness can reach what Bun does not. +set -e +src=${1:-/root/repos/oven-sh/bun/src/zstd} +dst=$(dirname "$0")/zstd-pub +rm -rf "$dst" +mkdir -p "$dst" +cp -r "$src"/. "$dst"/ +find "$dst" -name '*.rs' -exec sed -i -E 's/pub\((crate|super)\) /pub /g; s/^mod /pub mod /; s/^ fn reset_session/ pub fn reset_session/; s/^(const |static |fn |struct |enum |type )/pub \1/; s/^use (common|decompress|compress)::/pub use \1::/' {} + +cat > "$dst/Cargo.toml" < Date: Tue, 27 Oct 2026 13:44:07 +0000 Subject: [PATCH 012/199] zstd: take the harness out again It needs the C sources. It stays in the history of this branch. --- src/zstd/wip/build-ref.sh | 15 - src/zstd/wip/harness/Cargo.toml | 21 -- src/zstd/wip/harness/build.rs | 6 - src/zstd/wip/harness/src/c.rs | 104 ------ src/zstd/wip/harness/src/comp.rs | 285 --------------- src/zstd/wip/harness/src/data.rs | 65 ---- src/zstd/wip/harness/src/dec.rs | 183 ---------- src/zstd/wip/harness/src/extra.rs | 556 ------------------------------ src/zstd/wip/harness/src/main.rs | 31 -- src/zstd/wip/harness/src/sizes.rs | 1 - src/zstd/wip/harness/sync.sh | 20 -- 11 files changed, 1287 deletions(-) delete mode 100755 src/zstd/wip/build-ref.sh delete mode 100644 src/zstd/wip/harness/Cargo.toml delete mode 100644 src/zstd/wip/harness/build.rs delete mode 100644 src/zstd/wip/harness/src/c.rs delete mode 100644 src/zstd/wip/harness/src/comp.rs delete mode 100644 src/zstd/wip/harness/src/data.rs delete mode 100644 src/zstd/wip/harness/src/dec.rs delete mode 100644 src/zstd/wip/harness/src/extra.rs delete mode 100644 src/zstd/wip/harness/src/main.rs delete mode 100644 src/zstd/wip/harness/src/sizes.rs delete mode 100755 src/zstd/wip/harness/sync.sh diff --git a/src/zstd/wip/build-ref.sh b/src/zstd/wip/build-ref.sh deleted file mode 100755 index 209061b63378..000000000000 --- a/src/zstd/wip/build-ref.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Builds the reference C zstd exactly as Bun's scripts/build/deps/zstd.ts did (pin f8745da6 + Bun's two patches). -set -e -cd /root/zstd-ref -SRC="common/debug common/entropy_common common/error_private common/fse_decompress common/pool common/threading common/xxhash common/zstd_common compress/fse_compress compress/hist compress/huf_compress compress/zstd_compress compress/zstd_compress_literals compress/zstd_compress_sequences compress/zstd_compress_superblock compress/zstd_double_fast compress/zstd_fast compress/zstd_lazy compress/zstd_ldm compress/zstd_opt compress/zstd_preSplit compress/zstdmt_compress decompress/huf_decompress decompress/zstd_ddict decompress/zstd_decompress decompress/zstd_decompress_block" -build() { # name, extra flags, asm? - local name=$1; shift; local asm=$1; shift - mkdir -p build/$name; local objs="" - for s in $SRC; do o=build/$name/$(basename $s).o; clang-23 -c -O2 -g -fPIC -march=nehalem -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=0 -DXXH_NAMESPACE=ZSTD_ -Izstd/lib -Izstd/lib/common "$@" zstd/lib/$s.c -o $o & objs="$objs $o"; done - if [ $asm = 1 ]; then clang-23 -c -fPIC zstd/lib/decompress/huf_decompress_amd64.S -o build/$name/huf_asm.o; objs="$objs build/$name/huf_asm.o"; fi - wait - rm -f build/$name/libzstd.a; llvm-ar-23 rcs build/$name/libzstd.a $objs -} -build ref 1 -build noasm 0 -DZSTD_DISABLE_ASM=1 -DDYNAMIC_BMI2=0 diff --git a/src/zstd/wip/harness/Cargo.toml b/src/zstd/wip/harness/Cargo.toml deleted file mode 100644 index 6be9d34131e2..000000000000 --- a/src/zstd/wip/harness/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "zstd_harness" -version = "0.0.0" -edition = "2024" - -[dependencies] -bun_zstd = { path = "zstd-pub" } - -[profile.release] -debug = "line-tables-only" -overflow-checks = true -debug-assertions = true -panic = "abort" - -[profile.bench] -codegen-units = 1 -inherits = "release" -overflow-checks = false -debug-assertions = false - -[workspace] diff --git a/src/zstd/wip/harness/build.rs b/src/zstd/wip/harness/build.rs deleted file mode 100644 index e0b8fe9e7a87..000000000000 --- a/src/zstd/wip/harness/build.rs +++ /dev/null @@ -1,6 +0,0 @@ -fn main() { - let variant = std::env::var("ZSTD_REF").unwrap_or_else(|_| "ref".into()); - println!("cargo:rerun-if-env-changed=ZSTD_REF"); - println!("cargo:rustc-link-search=native=/root/zstd-ref/build/{variant}"); - println!("cargo:rustc-link-lib=static=zstd"); -} diff --git a/src/zstd/wip/harness/src/c.rs b/src/zstd/wip/harness/src/c.rs deleted file mode 100644 index d5a82e1f6b78..000000000000 --- a/src/zstd/wip/harness/src/c.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! The reference C library. -#![allow(non_camel_case_types, dead_code)] -use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; - -#[repr(C)] -pub struct ZSTD_inBuffer { pub src: *const c_void, pub size: usize, pub pos: usize } -#[repr(C)] -pub struct ZSTD_outBuffer { pub dst: *mut c_void, pub size: usize, pub pos: usize } -pub enum ZSTD_CCtx {} -pub enum ZSTD_DCtx {} -pub enum ZSTD_DDict {} - -unsafe extern "C" { - pub fn ZSTD_compress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize, level: c_int) -> usize; - pub fn ZSTD_compressBound(size: usize) -> usize; - pub fn ZSTD_decompress(dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_isError(code: usize) -> c_uint; - pub fn ZSTD_getErrorCode(code: usize) -> c_uint; - pub fn ZSTD_getErrorName(code: usize) -> *const c_char; - pub fn ZSTD_getErrorString(code: c_uint) -> *const c_char; - pub fn ZSTD_findDecompressedSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_getFrameContentSize(src: *const c_void, size: usize) -> c_ulonglong; - pub fn ZSTD_findFrameCompressedSize(src: *const c_void, size: usize) -> usize; - pub fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; - pub fn ZSTD_freeCCtx(c: *mut ZSTD_CCtx) -> usize; - pub fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; - pub fn ZSTD_freeDCtx(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_CCtx_setPledgedSrcSize(c: *mut ZSTD_CCtx, size: c_ulonglong) -> usize; - pub fn ZSTD_CCtx_loadDictionary(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_refPrefix(c: *mut ZSTD_CCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_CCtx_setParameter(c: *mut ZSTD_CCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_CCtx_reset(c: *mut ZSTD_CCtx, r: c_uint) -> usize; - pub fn ZSTD_compress2(c: *mut ZSTD_CCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_compressStream2(c: *mut ZSTD_CCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer, e: c_uint) -> usize; - pub fn ZSTD_DCtx_loadDictionary(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_refPrefix(c: *mut ZSTD_DCtx, d: *const c_void, n: usize) -> usize; - pub fn ZSTD_DCtx_setParameter(c: *mut ZSTD_DCtx, p: c_uint, v: c_int) -> usize; - pub fn ZSTD_DCtx_reset(c: *mut ZSTD_DCtx, r: c_uint) -> usize; - pub fn ZSTD_initDStream(c: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_decompressDCtx(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize) -> usize; - pub fn ZSTD_decompressStream(c: *mut ZSTD_DCtx, o: *mut ZSTD_outBuffer, i: *mut ZSTD_inBuffer) -> usize; - pub fn ZSTD_createDDict_byReference(d: *const c_void, n: usize) -> *mut ZSTD_DDict; - pub fn ZSTD_freeDDict(d: *mut ZSTD_DDict) -> usize; - pub fn ZSTD_decompress_usingDDict(c: *mut ZSTD_DCtx, dst: *mut c_void, cap: usize, src: *const c_void, size: usize, d: *const ZSTD_DDict) -> usize; - pub fn ZSTD_defaultCLevel() -> c_int; - pub fn ZSTD_minCLevel() -> c_int; - pub fn ZSTD_maxCLevel() -> c_int; -} - -/// `Ok(value)` or `Err(error code)`. -pub fn res(code: usize) -> Result { - unsafe { if ZSTD_isError(code) != 0 { Err(ZSTD_getErrorCode(code)) } else { Ok(code) } } -} - -pub struct CCtx(pub *mut ZSTD_CCtx); -impl CCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createCCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_CCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_CCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn pledge(&mut self, n: u64) -> Result { res(unsafe { ZSTD_CCtx_setPledgedSrcSize(self.0, n) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_CCtx_reset(self.0, r) }) } - pub fn compress2(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_compress2(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - /// Returns (result, out pos, in pos). - pub fn stream2(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize, end: u32) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_compressStream2(self.0, &mut o, &mut i, end) }); - (r, o.pos, i.pos) - } -} -impl Drop for CCtx { fn drop(&mut self) { unsafe { ZSTD_freeCCtx(self.0) }; } } - -pub struct DCtx(pub *mut ZSTD_DCtx); -impl DCtx { - pub fn new() -> Self { Self(unsafe { ZSTD_createDCtx() }) } - pub fn set(&mut self, p: u32, v: i32) -> Result { res(unsafe { ZSTD_DCtx_setParameter(self.0, p, v) }) } - pub fn load_dict(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_loadDictionary(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn ref_prefix(&mut self, d: &[u8]) -> Result { res(unsafe { ZSTD_DCtx_refPrefix(self.0, d.as_ptr().cast(), d.len()) }) } - pub fn reset(&mut self, r: u32) -> Result { res(unsafe { ZSTD_DCtx_reset(self.0, r) }) } - pub fn init_stream(&mut self) { unsafe { ZSTD_initDStream(self.0) }; } - pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompressDCtx(self.0, dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) - } - pub fn stream(&mut self, dst: &mut [u8], dpos: usize, src: &[u8], spos: usize) -> (Result, usize, usize) { - let mut o = ZSTD_outBuffer { dst: dst.as_mut_ptr().cast(), size: dst.len(), pos: dpos }; - let mut i = ZSTD_inBuffer { src: src.as_ptr().cast(), size: src.len(), pos: spos }; - let r = res(unsafe { ZSTD_decompressStream(self.0, &mut o, &mut i) }); - (r, o.pos, i.pos) - } -} -impl Drop for DCtx { fn drop(&mut self) { unsafe { ZSTD_freeDCtx(self.0) }; } } - -pub fn compress(src: &[u8], level: i32) -> Vec { - let mut dst = vec![0u8; unsafe { ZSTD_compressBound(src.len()) }]; - let n = res(unsafe { ZSTD_compress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len(), level) }).unwrap(); - dst.truncate(n); - dst -} -pub fn decompress(dst: &mut [u8], src: &[u8]) -> Result { - res(unsafe { ZSTD_decompress(dst.as_mut_ptr().cast(), dst.len(), src.as_ptr().cast(), src.len()) }) -} diff --git a/src/zstd/wip/harness/src/comp.rs b/src/zstd/wip/harness/src/comp.rs deleted file mode 100644 index 586336aa7dde..000000000000 --- a/src/zstd/wip/harness/src/comp.rs +++ /dev/null @@ -1,285 +0,0 @@ -//! The compressor against the C one: same calls, same results, same bytes, same positions. -use crate::c; -use crate::data::{Rng, data}; -use bun_zstd as z; - -fn r2(r: Result) -> Result { r.map_err(|e| e as u32) } -fn r0(r: Result<(), z::ErrorCode>) -> Result { r.map(|()| 0).map_err(|e| e as u32) } - -/// A dictionary: raw content, or in zstd's format (made from a real one by the C trainer is not available, so entropy -/// tables are taken from a frame... here: raw, or magic + garbage to exercise the error path). -fn dictionary(rng: &mut Rng, base: &[u8]) -> Vec { - let len = rng.size(1 << 17); - let mut d = if rng.chance(1, 2) && !base.is_empty() { - let start = rng.below(base.len()); - let end = (start + len).min(base.len()); - base[start..end].to_vec() - } else { - data(rng, len) - }; - if rng.chance(1, 12) && d.len() >= 8 { - d[..4].copy_from_slice(&0xEC30A437u32.to_le_bytes()); - } - d -} - -fn level(rng: &mut Rng) -> i32 { - match rng.below(10) { - 0 => -(rng.range(1, 50) as i32), - 1 => rng.range(0, 30) as i32 - 4, - 2 => -(rng.size(200000) as i32), - _ => rng.range(1, 22) as i32, - } -} - -pub fn one_shot(rng: &mut Rng, max: usize) { - let len = rng.size(max); - let src = data(rng, len); - let lvl = level(rng); - let bound = unsafe { c::ZSTD_compressBound(len) }; - assert_eq!(bound, z::compress_bound(len)); - let cap = match rng.below(8) { 0 => rng.below(bound + 1), 1 => bound.saturating_sub(rng.below(20)), _ => bound + rng.below(50) }; - let mut a = vec![0u8; cap]; - let mut b = vec![0u8; cap]; - let ra = c::res(unsafe { c::ZSTD_compress(a.as_mut_ptr().cast(), cap, src.as_ptr().cast(), len, lvl) }); - let rb = r2(z::compress(&mut b, &src, lvl)); - assert_eq!(ra, rb, "one shot: len {len} level {lvl} cap {cap}"); - if let Ok(n) = ra { - if a[..n] != b[..n] { - let at = a[..n].iter().zip(&b[..n]).position(|(x, y)| x != y); - panic!("one shot bytes differ at {at:?}/{n}: len {len} level {lvl}"); - } - } -} - -const PARAMS: &[(i32, i32, i32)] = &[ - (10, 0, 1), (100, -50, 22), (101, 10, 27), (102, 6, 26), (103, 6, 26), (104, 1, 12), (105, 3, 7), (106, 0, 4096), - (107, 1, 9), (130, 0, 20000), (160, 0, 2), (161, 6, 24), (162, 4, 4096), (163, 1, 8), (164, 0, 12), - (200, 0, 1), (201, 0, 1), (202, 0, 1), -]; - -fn set_params(rng: &mut Rng, a: &mut c::CCtx, b: &mut z::CCtx, log: &mut Vec, mt: bool) { - let n = match rng.below(4) { 0 => 0, 1 => 1, 2 => rng.below(4), _ => rng.below(9) }; - for _ in 0..n { - let (p, v) = if rng.chance(1, 25) { - (rng.below(403) as i32, rng.range(0, 40) as i32 - 5) - } else { - let &(p, lo, hi) = rng.pick(PARAMS); - let v = if rng.chance(1, 20) { rng.range(0, 3000000) as i32 - 200000 } else { lo + rng.below((hi - lo + 1) as usize) as i32 }; - (p, v) - }; - if !mt && (400..=402).contains(&p) { continue; } - let ra = a.set(p as u32, v); - let rb = r2(b.set_parameter(p, v)); - log.push(format!("set {p} {v} -> {ra:?}")); - assert_eq!(ra, rb, "{log:?}"); - } -} - -/// One context, several frames, random calls. -pub fn stream(rng: &mut Rng, max: usize) { - let mut a = c::CCtx::new(); - let mut b = z::CCtx::new(); - let mut log: Vec = Vec::new(); - let frames = if rng.chance(1, 3) { rng.range(2, 5) } else { 1 }; - for _ in 0..frames { - if rng.chance(1, 8) { a.reset(1).unwrap(); b.reset_session(); log.push("reset".into()); } - set_params(rng, &mut a, &mut b, &mut log, false); - let len = rng.size(max); - let src = data(rng, len); - if rng.chance(1, 4) && std::env::var_os("NODICT").is_none() { - let d = if rng.chance(1, 10) { Vec::new() } else { dictionary(rng, &src) }; - let ra = a.load_dict(&d); - let rb = r0(b.load_dictionary(&d)); - log.push(format!("dict {} -> {ra:?}", d.len())); - assert_eq!(ra, rb, "{log:?}"); - } - if rng.chance(1, 4) { - let n = if rng.chance(1, 8) { rng.size(max) as u64 } else { len as u64 }; - let ra = a.pledge(n); - let rb = r0(b.set_pledged_src_size(n)); - log.push(format!("pledge {n} (len {len}) -> {ra:?}")); - assert_eq!(ra, rb, "{log:?}"); - } - log.push(format!("frame len {len}")); - // how the input and the output are cut - let in_mode = rng.below(4); - let out_mode = rng.below(4); - let small = len < 5000; - let mut spos_total = 0usize; - let mut calls = 0usize; - let mut failed = false; - loop { - calls += 1; - let left = len - spos_total; - let chunk = match in_mode { - 0 => left, - 1 => rng.range(0, if small { 64 } else { 100_000 }).min(left), - 2 => rng.size(1 << 19).min(left), - _ => if rng.chance(1, 2) { left } else { rng.below(left + 1) }, - }; - let cap = match out_mode { - 0 => z::compress_bound(chunk) + rng.below(100), - 1 => rng.range(0, if small { 40 } else { 50_000 }), - 2 => rng.size(1 << 18), - _ => 1 << 17, - }; - let last = spos_total + chunk == len; - let end = if last && rng.chance(3, 4) { 2 } else { *rng.pick(&[0u32, 0, 0, 1, 1, 2]) }; - let end = if calls > 4000 { 2 } else { end }; - let input = &src[spos_total..spos_total + chunk]; - let mut oa = vec![0u8; cap]; - let mut ob = vec![0u8; cap]; - let (ra, opa, ipa) = a.stream2(&mut oa, 0, input, 0, end); - let (rb, opb, ipb) = { - let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; - let mut i = z::InBuffer { src: input, pos: 0 }; - let r = r2(b.compress_stream2(&mut o, &mut i, match end { 0 => z::EndDirective::Continue, 1 => z::EndDirective::Flush, _ => z::EndDirective::End })); - (r, o.pos, i.pos) - }; - if (ra, opa, ipa) != (rb, opb, ipb) || oa[..opa] != ob[..opb] { - let at = oa.iter().zip(&ob).position(|(x, y)| x != y); - panic!("stream: call {calls} chunk {chunk} cap {cap} end {end}: C {:?} Rust {:?} first diff {at:?}\n{log:?}", (ra, opa, ipa), (rb, opb, ipb)); - } - spos_total += ipa; - if ra.is_err() { failed = true; break; } - if end == 2 && ra == Ok(0) { - if spos_total == len { break; } - log.push(format!("frame ended at {spos_total}")); - } - if calls > 6000 { panic!("no end {log:?}"); } - } - if failed { a.reset(1).unwrap(); b.reset_session(); } - } -} - -pub fn run(seed: u64, iters: u64, max: usize) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if std::env::var_os("TRACE").is_some() { eprintln!("iter {i} seed {s}"); } - if rng.chance(1, 3) { one_shot(&mut rng, max) } else { stream(&mut rng, max) } - } - println!("comp ok: seed {seed}, {iters} iterations"); -} - -/// Multithreaded: when output comes out depends on timing, so only the whole output is compared. -pub fn mt(rng: &mut Rng, max: usize) { - let mut a = c::CCtx::new(); - let mut b = z::CCtx::new(); - let mut log: Vec = Vec::new(); - let frames = if rng.chance(1, 3) { rng.range(2, 3) } else { 1 }; - for _ in 0..frames { - let mut set = |p: i32, v: i32, log: &mut Vec| { - let ra = a.set(p as u32, v); - let rb = r2(b.set_parameter(p, v)); - log.push(format!("set {p} {v} -> {ra:?}")); - assert_eq!(ra, rb, "{log:?}"); - }; - set(400, rng.range(1, 4) as i32, &mut log); - if rng.chance(1, 2) { set(401, rng.range(0, 3 << 20) as i32, &mut log); } - if rng.chance(1, 2) { set(402, rng.range(0, 9) as i32, &mut log); } - set(100, if rng.chance(1, 6) { rng.range(13, 19) as i32 } else { rng.range(1, 12) as i32 - 3 }, &mut log); - if rng.chance(1, 2) { set(101, rng.range(10, 22) as i32, &mut log); } - if rng.chance(1, 3) { set(160, 1, &mut log); if rng.chance(1, 2) { set(162, rng.range(4, 200) as i32, &mut log); } if rng.chance(1,2) { set(161, rng.range(6, 20) as i32, &mut log); } } - if rng.chance(1, 2) { set(201, 1, &mut log); } - if rng.chance(1, 8) { set(130, rng.range(1340, 60000) as i32, &mut log); } - let len = if rng.chance(1, 4) { rng.size(max) } else { rng.range(max / 4, max) }; - let src = data(rng, len); - if rng.chance(1, 5) { - let d = dictionary(rng, &src); - let ra = a.load_dict(&d); - let rb = r0(b.load_dictionary(&d)); - log.push(format!("dict {} -> {ra:?}", d.len())); - assert_eq!(ra, rb, "{log:?}"); - } - if rng.chance(1, 4) { - let ra = a.pledge(len as u64); - let rb = r0(b.set_pledged_src_size(len as u64)); - log.push(format!("pledge {len}")); - assert_eq!(ra, rb, "{log:?}"); - } - log.push(format!("frame len {len}")); - let mut out_a: Vec = Vec::new(); - let mut out_b: Vec = Vec::new(); - let mut pos = 0usize; - let in_mode = rng.below(3); - loop { - let left = len - pos; - let chunk = match in_mode { 0 => left, 1 => rng.size(1 << 21).min(left), _ => rng.range(0, 300_000).min(left) }; - let last = pos + chunk == len; - let end = if last { 2u32 } else { *rng.pick(&[0u32, 0, 0, 0, 1]) }; - let input = &src[pos..pos + chunk]; - let cap = z::compress_bound(len.max(1 << 20)) + 1024; - let mut buf = vec![0u8; cap]; - let mut err_a = None; - let mut err_b = None; - let mut ip = 0usize; - loop { - let (r, op, np) = a.stream2(&mut buf, 0, input, ip, end); - out_a.extend_from_slice(&buf[..op]); - ip = np; - match r { Err(e) => { err_a = Some(e); break; } Ok(left) => if (end == 0 && ip == chunk) || (end != 0 && left == 0) { break; } } - } - let mut ip = 0usize; - loop { - let mut o = z::OutBuffer { dst: &mut buf, pos: 0 }; - let mut i = z::InBuffer { src: input, pos: ip }; - let r = r2(b.compress_stream2(&mut o, &mut i, match end { 0 => z::EndDirective::Continue, 1 => z::EndDirective::Flush, _ => z::EndDirective::End })); - let op = o.pos; - ip = i.pos; - out_b.extend_from_slice(&buf[..op]); - match r { Err(e) => { err_b = Some(e); break; } Ok(left) => if (end == 0 && ip == chunk) || (end != 0 && left == 0) { break; } } - } - assert_eq!(err_a, err_b, "{log:?}"); - log.push(format!("chunk {chunk} end {end}")); - if err_a.is_some() { a.reset(1).unwrap(); b.reset_session(); break; } - pos += chunk; - if last { break; } - } - if out_a != out_b { - let at = out_a.iter().zip(&out_b).position(|(x, y)| x != y); - panic!("mt: {} vs {} bytes, first diff {at:?}\n{log:?}", out_a.len(), out_b.len()); - } - } -} - -pub fn run_mt(seed: u64, iters: u64, max: usize) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if std::env::var_os("TRACE").is_some() { eprintln!("iter {i} seed {s}"); } - mt(&mut rng, max); - } - println!("mt ok: seed {seed}, {iters} iterations"); -} - -/// Dictionaries, by strategy. -pub fn run_dict(seed: u64, iters: u64, max: usize) { - let mut fails = std::collections::BTreeMap::new(); - for i in 0..iters { - let mut rng = Rng(seed.wrapping_mul(1_000_003).wrapping_add(i)); - let strat = rng.range(1, 9) as i32; - let len = rng.size(max); - let src = data(&mut rng, len); - let d = dictionary(&mut rng, &src); - let pledge = rng.chance(1, 2); - let mut a = c::CCtx::new(); - let mut b = z::CCtx::new(); - a.set(107, strat).unwrap(); b.set_parameter(107, strat).unwrap(); - a.load_dict(&d).unwrap(); b.load_dictionary(&d).unwrap(); - let cap = z::compress_bound(len) + 64; - let (mut oa, mut ob) = (vec![0u8; cap], vec![0u8; cap]); - let end = if pledge { 2 } else { 1 }; - let (ra, pa, _) = a.stream2(&mut oa, 0, &src, 0, end); - let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; - let mut inp = z::InBuffer { src: &src, pos: 0 }; - let rb = r2(b.compress_stream2(&mut o, &mut inp, if pledge { z::EndDirective::End } else { z::EndDirective::Flush })); - let pb = o.pos; - let ok = ra == rb && pa == pb && oa[..pa] == ob[..pb]; - let e = fails.entry((strat, pledge)).or_insert((0u32, 0u32)); - e.0 += 1; - if !ok { e.1 += 1; if e.1 == 1 { eprintln!("first fail strat {strat} pledge {pledge} len {len} dict {} iter {i}: {ra:?} {rb:?} {pa} {pb}", d.len()); } } - } - for (k, v) in fails { println!("{k:?}: {} of {} differ", v.1, v.0); } -} diff --git a/src/zstd/wip/harness/src/data.rs b/src/zstd/wip/harness/src/data.rs deleted file mode 100644 index c48a645e1b5d..000000000000 --- a/src/zstd/wip/harness/src/data.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Deterministic inputs. -pub struct Rng(pub u64); -impl Rng { - pub fn next(&mut self) -> u64 { - // splitmix64 - self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); - z ^ (z >> 31) - } - pub fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } - pub fn range(&mut self, lo: usize, hi: usize) -> usize { lo + self.below(hi - lo + 1) } - pub fn chance(&mut self, num: usize, den: usize) -> bool { self.below(den) < num } - pub fn pick<'a, T>(&mut self, v: &'a [T]) -> &'a T { &v[self.below(v.len())] } - /// Skewed towards small values. - pub fn size(&mut self, max: usize) -> usize { - let bits = self.below(usize::BITS as usize - max.leading_zeros() as usize + 1); - (self.next() as usize & ((1usize << bits) - 1)).min(max) - } -} - -static WORDS: &[&str] = &["the ", "function ", "return ", "const ", "bun", "zstd", "\n", " ", "{", "}", "(", ");", "import ", "export ", "0", "1", "===", "await ", "async ", "null", "undefined", "\"use strict\";", "=> ", "let ", "for ", "if ", "else ", "this.", "length", "// "]; - -/// One buffer of about `len` bytes, of a kind picked by `rng`. -pub fn data(rng: &mut Rng, len: usize) -> Vec { - let mut v = Vec::with_capacity(len + 64); - match rng.below(9) { - 0 => v.resize(len, rng.next() as u8), - 1 => while v.len() < len { v.push(rng.next() as u8) }, - 2 => while v.len() < len { v.extend_from_slice(rng.pick(WORDS).as_bytes()) }, - 3 => { // few symbols, skewed - let n = rng.range(2, 40); - while v.len() < len { let a = rng.below(n); let b = rng.below(n); v.push(b'a' + a.min(b) as u8) } - } - 4 => { // LZ-like: copies of earlier parts, at all distances - while v.len() < len { - if v.len() > 8 && rng.chance(3, 4) { - let dist = 1 + rng.size(v.len() - 1); - let n = 3 + rng.size(300); - for _ in 0..n { let b = v[v.len() - dist]; v.push(b) } - } else { - for _ in 0..1 + rng.size(40) { v.push(rng.next() as u8) } - } - } - } - 5 => { // periodic with noise - let period = rng.range(1, 300); - let pat: Vec = (0..period).map(|_| rng.next() as u8).collect(); - while v.len() < len { let b = pat[v.len() % period]; v.push(if rng.chance(1, 200) { rng.next() as u8 } else { b }) } - } - 6 => { // sections of different kinds - while v.len() < len { let n = 1 + rng.size(len.max(1)); let part = data(rng, n.min(len - v.len() + 1)); v.extend_from_slice(&part) } - } - 7 => { // numbers as text - let mut x = rng.next() % 1000; - while v.len() < len { x = x.wrapping_add(rng.next() % 17); v.extend_from_slice(format!("{x},").as_bytes()) } - } - _ => { // long runs - while v.len() < len { let b = rng.next() as u8; let n = 1 + rng.size(5000); v.resize(v.len() + n, b) } - } - } - v.truncate(len); - v -} diff --git a/src/zstd/wip/harness/src/dec.rs b/src/zstd/wip/harness/src/dec.rs deleted file mode 100644 index 418ae256c1ae..000000000000 --- a/src/zstd/wip/harness/src/dec.rs +++ /dev/null @@ -1,183 +0,0 @@ -//! The decoder against the C decoder: valid frames, damaged frames, one call and streaming. -use crate::c; -use crate::data::{Rng, data}; -use bun_zstd as z; - -fn rs(r: Result) -> Result { r.map_err(|e| e as u32) } - -/// A frame (or several) made by the C compressor with parameters picked by `rng`. Returns (frames, content, dictionary). -pub fn make_frames(rng: &mut Rng) -> (Vec, Vec, Vec) { - let mut out = Vec::new(); - let mut content = Vec::new(); - let dict = if rng.chance(1, 4) { let n = rng.size(200_000); data(rng, n) } else { Vec::new() }; - let nframes = if rng.chance(1, 5) { rng.range(2, 4) } else { 1 }; - for _ in 0..nframes { - if rng.chance(1, 12) { - // skippable frame - let n = rng.size(300); - out.extend_from_slice(&(0x184D2A50u32 + rng.below(16) as u32).to_le_bytes()); - out.extend_from_slice(&(n as u32).to_le_bytes()); - out.extend_from_slice(&data(rng, n)); - continue; - } - let max = *rng.pick(&[0usize, 10, 300, 5_000, 70_000, 300_000, 1_500_000]); - let len = rng.size(max); - let mut src = data(rng, len); - if !dict.is_empty() && rng.chance(1, 2) && !src.is_empty() { - // make the dictionary useful - let n = rng.size(dict.len().min(src.len())); - let at = rng.below(dict.len() - n + 1); - let to = rng.below(src.len() - n + 1); - src[to..to + n].copy_from_slice(&dict[at..at + n]); - } - let mut cctx = c::CCtx::new(); - let level = *rng.pick(&[-50, -5, -1, 1, 1, 2, 3, 3, 3, 4, 5, 6, 7, 9, 12, 13, 16, 19, 22]); - cctx.set(100, if src.len() > 400_000 { level.min(12) } else { level }).unwrap(); - if rng.chance(1, 3) { cctx.set(201, 1).unwrap(); } // checksum - if rng.chance(1, 4) { cctx.set(200, 0).unwrap(); } // no content size - if rng.chance(1, 4) { cctx.set(101, rng.range(10, 22) as i32).unwrap(); } // windowLog - if rng.chance(1, 10) { cctx.set(130, rng.range(1340, 20_000) as i32).unwrap(); } // targetCBlockSize - if rng.chance(1, 10) { cctx.set(160, 1).unwrap(); } // LDM - if rng.chance(1, 8) { cctx.set(105, rng.range(3, 7) as i32).unwrap(); } // minMatch - if !dict.is_empty() { cctx.load_dict(&dict).unwrap(); } - let mut dst = vec![0u8; unsafe { c::ZSTD_compressBound(src.len()) } + 1024]; - let n = if rng.chance(1, 2) { - cctx.compress2(&mut dst, &src).unwrap() - } else { - // streaming, with flushes: several blocks, no content size - let (mut dpos, mut spos) = (0, 0); - loop { - let chunk = (spos + 1 + rng.size(200_000)).min(src.len()); - let end = if chunk == src.len() { 2 } else if rng.chance(1, 3) { 1 } else { 0 }; - let (r, d, s) = cctx.stream2(&mut dst, dpos, &src[..chunk], spos, end); - dpos = d; spos = s; - if r.unwrap() == 0 && end == 2 { break dpos; } - } - }; - out.extend_from_slice(&dst[..n]); - content.extend_from_slice(&src); - } - (out, content, dict) -} - -fn mutate(rng: &mut Rng, frames: &mut Vec) { - if frames.is_empty() { return; } - for _ in 0..1 + rng.size(4) { - if frames.is_empty() { return; } - // damage is more interesting near the start of a frame - let at = if rng.chance(1, 2) { rng.size(frames.len() - 1) } else { rng.below(frames.len()) }; - match rng.below(8) { - 0 => frames[at] ^= 1 << rng.below(8), - 1 => frames[at] = rng.next() as u8, - 2 => frames.truncate(at), - 3 => { frames.remove(at); } - 4 => frames.insert(at, rng.next() as u8), - 5 => { let n = rng.size(16).min(frames.len() - at); for b in &mut frames[at..at + n] { *b = rng.next() as u8 } } - 6 => { let n = rng.size(64); let extra = data(rng, n); frames.extend_from_slice(&extra) } - _ => { let from = rng.below(frames.len()); let n = rng.size(32).min(frames.len() - from.max(at)); frames.copy_within(from..from + n, at) } - } - } -} - -fn check_sizes(frames: &[u8]) { - let p = frames.as_ptr().cast(); - unsafe { - assert_eq!(c::ZSTD_findDecompressedSize(p, frames.len()), z::find_decompressed_size(frames), "findDecompressedSize"); - assert_eq!(c::ZSTD_getFrameContentSize(p, frames.len()), z::get_frame_content_size(frames), "getFrameContentSize"); - assert_eq!(c::res(c::ZSTD_findFrameCompressedSize(p, frames.len())), rs(z::decompress::find_frame_compressed_size(frames)), "findFrameCompressedSize"); - } -} - -fn one_shot(frames: &[u8], dict: &[u8], cap: usize, prefix: bool) { - let mut cd = vec![0xAAu8; cap]; - let mut rd = vec![0xAAu8; cap]; - let mut cctx = c::DCtx::new(); - let mut rctx = z::DCtx::new(); - let (cr, rr); - if prefix { - cctx.ref_prefix(dict).unwrap(); - cr = cctx.decompress(&mut cd, frames); - // an empty prefix is no dictionary at all - rr = if dict.is_empty() { rs(rctx.decompress(&mut rd, frames)) } else { rs(z::DDict::prefix(dict).and_then(|d| rctx.decompress_using_ddict(&mut rd, frames, &d))) }; - } else { - assert_eq!(cctx.load_dict(dict).map(|_| ()), rs(rctx.load_dictionary(dict)), "loadDictionary"); - cr = cctx.decompress(&mut cd, frames); - rr = rs(rctx.decompress(&mut rd, frames)); - } - assert_eq!(cr, rr, "one-shot result (cap {cap})"); - if let Ok(n) = cr { assert!(cd[..n] == rd[..n], "one-shot output"); } -} - -fn streaming(rng: &mut Rng, frames: &[u8], dict: &[u8], expect: Option<&[u8]>) { - let mut cctx = c::DCtx::new(); - let mut rctx = z::DCtx::new(); - if rng.chance(1, 6) { - let (p, v) = (*rng.pick(&[100u32, 100, 100, 0, 99, 1]), *rng.pick(&[0i32, 9, 10, 15, 20, 27, 31, 32, -1])); - assert_eq!(cctx.set(p, v).map(|_| ()), rs(rctx.set_parameter(p as i32, v)), "setParameter {p} {v}"); - } - assert_eq!(cctx.load_dict(dict).map(|_| ()), rs(rctx.load_dictionary(dict)), "loadDictionary"); - // tiny steps only where they do not take forever - let small = frames.len() < 3_000; - let in_max = *rng.pick(if small { &[1usize, 2, 3, 7, 64, 1000, 70_000, 10_000_000] } else { &[300usize, 1000, 5000, 20_000, 70_000, 200_000, 10_000_000, 10_000_000] }); - let out_max = *rng.pick(if small && expect.is_none_or(|e| e.len() < 20_000) { &[1usize, 2, 5, 100, 4096, 16_384, 140_000, 10_000_000] } else { &[1000usize, 4096, 4096, 16_384, 16_384, 140_000, 1_000_000, 10_000_000] }); - let mut total = Vec::new(); - let (mut cbuf, mut rbuf) = (Vec::new(), Vec::new()); - let mut ipos = 0usize; - let mut stalls = 0; - loop { - let iend = (ipos + rng.range(if stalls > 0 { 1 } else { 0 }, in_max)).min(frames.len()); - let cap = rng.range(if stalls > 1 { 1 } else { 0 }, out_max); - let chunk = &frames[ipos..iend]; - if cbuf.len() < cap { cbuf.resize(cap, 0x55); rbuf.resize(cap, 0x55); } - let cd = &mut cbuf[..cap]; - let rd = &mut rbuf[..cap]; - let (cr, co, ci) = cctx.stream(cd, 0, chunk, 0); - let mut ob = z::OutBuffer { dst: rd, pos: 0 }; - let mut ib = z::InBuffer { src: chunk, pos: 0 }; - let rr = rs(rctx.decompress_stream(&mut ob, &mut ib)); - let (ro, ri) = (ob.pos, ib.pos); - assert_eq!((cr, co, ci), (rr, ro, ri), "stream step at {ipos} (in {} out {cap})", chunk.len()); - assert!(cbuf[..co] == rbuf[..ro], "stream output at {ipos}"); - total.extend_from_slice(&rbuf[..ro]); - ipos += ri; - if cr.is_err() { return; } - if ri == 0 && ro == 0 { stalls += 1 } else { stalls = 0 } - if ipos == frames.len() && (cr == Ok(0) || stalls > 3) { break; } - if cr == Ok(0) && rng.chance(1, 8) { - match rng.below(2) { - 0 => { cctx.init_stream(); rctx.init_stream(); if !dict.is_empty() { return; } } - _ => {} - } - } - } - if let Some(expect) = expect { assert!(total == expect, "stream total"); } -} - -pub fn run(seed: u64, iters: u64) { - for i in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(i); - let mut rng = Rng(s); - if i % 200 == 0 { eprintln!("dec iter {i} (seed {s})"); } - let (mut frames, content, dict) = make_frames(&mut rng); - // valid - check_sizes(&frames); - one_shot(&frames, &dict, content.len(), false); - one_shot(&frames, &dict, content.len() + rng.size(300_000), false); - if !content.is_empty() { let cap = rng.below(content.len()); one_shot(&frames, &dict, cap, false); } - streaming(&mut rng, &frames, &dict, Some(&content)); - streaming(&mut rng, &frames, &dict, Some(&content)); - // wrong or missing dictionary - if rng.chance(1, 4) { let n = rng.size(5000); let other = data(&mut rng, n); one_shot(&frames, &other, content.len(), rng.chance(1, 2)); streaming(&mut rng, &frames, &other, None); } - // damaged - for _ in 0..6 { - let mut bad = frames.clone(); - mutate(&mut rng, &mut bad); - check_sizes(&bad); - let cap = content.len() + if rng.chance(1, 2) { rng.size(200_000) } else { 0 }; - one_shot(&bad, &dict, cap, false); - streaming(&mut rng, &bad, &dict, None); - } - mutate(&mut rng, &mut frames); - } - println!("dec ok: {iters} iterations from seed {seed}"); -} diff --git a/src/zstd/wip/harness/src/extra.rs b/src/zstd/wip/harness/src/extra.rs deleted file mode 100644 index 23be98f8ce66..000000000000 --- a/src/zstd/wip/harness/src/extra.rs +++ /dev/null @@ -1,556 +0,0 @@ -//! Extra differential checks. -use crate::c; -use crate::data::{Rng, data}; -use bun_zstd as z; - -fn r2(r: Result) -> Result { r.map_err(|e| e as u32) } -fn r0(r: Result<(), z::ErrorCode>) -> Result { r.map(|()| 0).map_err(|e| e as u32) } -fn ed(e: u32) -> z::EndDirective { match e { 0 => z::EndDirective::Continue, 1 => z::EndDirective::Flush, _ => z::EndDirective::End } } - -fn load(dir: &str) -> Vec> { - let mut names: Vec<_> = std::fs::read_dir(dir).unwrap().map(|e| e.unwrap().path()).collect(); - names.sort(); - names.iter().map(|p| std::fs::read(p).unwrap()).collect() -} - -pub fn tdict(seed: u64, iters: u64, dict_dir: &str, sample_dir: &str) { - let dicts = load(dict_dir); - let samples = load(sample_dir); - for it in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(it); - if let Ok(only) = std::env::var("ONLY") { if only.parse::().unwrap() != s { continue; } } - let mut rng = Rng(s); - let rng = &mut rng; - if it % 100 == 0 { eprintln!("iter {it}"); } - let mut dict = rng.pick(&dicts).clone(); - if rng.chance(1, 10) { - // damage - let at = rng.below(dict.len().min(400)); - dict[at] ^= 1 << rng.below(8); - } - if rng.chance(1, 15) { let n = rng.below(dict.len()); dict.truncate(n); } - let mut a = c::CCtx::new(); - let mut b = z::CCtx::new(); - let mut log: Vec = vec![format!("seed {s} dict {}", dict.len())]; - let mt = rng.chance(1, 8); - let frames = rng.range(1, 4); - let mut all_out: Vec = Vec::new(); - let mut all_src: Vec = Vec::new(); - let mut ok_frames = true; - { - let mut set = |p: i32, v: i32, log: &mut Vec| { - let ra = a.set(p as u32, v); - let rb = r2(b.set_parameter(p, v)); - log.push(format!("set {p} {v} -> {ra:?}")); - assert_eq!(ra, rb, "{log:?}"); - }; - set(100, *rng.pick(&[-7, -1, 1, 2, 3, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 17, 18, 19, 22]), &mut log); - if rng.chance(1, 3) { set(107, rng.range(1, 9) as i32, &mut log); } - if rng.chance(1, 4) { set(101, rng.range(10, 24) as i32, &mut log); } - if rng.chance(1, 4) { set(201, 1, &mut log); } - if rng.chance(1, 5) { set(202, 0, &mut log); } - if rng.chance(1, 5) { set(200, 0, &mut log); } - if rng.chance(1, 8) { set(160, 1, &mut log); } - if rng.chance(1, 8) { set(130, rng.range(1340, 9000) as i32, &mut log); } - if rng.chance(1, 8) { set(105, rng.range(3, 7) as i32, &mut log); } - if mt { set(400, rng.range(1, 3) as i32, &mut log); } - } - let ra = a.load_dict(&dict); - let rb = r0(b.load_dictionary(&dict)); - assert_eq!(ra, rb, "{log:?}"); - for _ in 0..frames { - let mut src = Vec::new(); - let target = if mt { rng.range(600_000, 3_000_000) } else { *rng.pick(&[0usize, 50, 500, 3000, 9000, 20_000, 40_000, 200_000, 700_000]) }; - let target = if mt { target } else { rng.size(target) }; - while src.len() < target { - if rng.chance(1, 6) { let n = rng.size(3000); src.extend_from_slice(&data(rng, n)); continue; } - let f = rng.pick(&samples); - let st = rng.below(f.len()); - let n = rng.size(20000).min(f.len() - st); - src.extend_from_slice(&f[st..st + n]); - } - src.truncate(target); - let len = src.len(); - if rng.chance(1, 3) { - let ra = a.pledge(len as u64); let rb = r0(b.set_pledged_src_size(len as u64)); - assert_eq!(ra, rb); log.push(format!("pledge {len}")); - } - log.push(format!("frame {len}")); - let mode = rng.below(3); - let mut pos = 0usize; - let mut oa_all = Vec::new(); - let mut ob_all = Vec::new(); - let mut failed = false; - loop { - let left = len - pos; - let chunk = match mode { 0 => left, 1 => rng.size(100_000).min(left), _ => rng.range(0, 5000).min(left) }; - let last = pos + chunk == len; - let end = if last { 2 } else { *rng.pick(&[0u32, 0, 0, 1]) }; - let input = &src[pos..pos + chunk]; - let cap = z::compress_bound(len.max(1 << 17)) + 1024; - let mut buf = vec![0u8; cap]; - let (mut ea, mut eb) = (None, None); - let mut ip = 0; - loop { - let (r, op, np) = a.stream2(&mut buf, 0, input, ip, end); - oa_all.extend_from_slice(&buf[..op]); ip = np; - match r { Err(e) => { ea = Some(e); break; } Ok(l) => if (end == 0 && ip == chunk) || (end != 0 && l == 0) { break; } } - } - let mut ip = 0; - loop { - let mut o = z::OutBuffer { dst: &mut buf, pos: 0 }; - let mut i = z::InBuffer { src: input, pos: ip }; - let r = r2(b.compress_stream2(&mut o, &mut i, ed(end))); - let op = o.pos; ip = i.pos; - ob_all.extend_from_slice(&buf[..op]); - match r { Err(e) => { eb = Some(e); break; } Ok(l) => if (end == 0 && ip == chunk) || (end != 0 && l == 0) { break; } } - } - assert_eq!(ea, eb, "{log:?}"); - if ea.is_some() { failed = true; let _ = a.reset(1); b.reset_session(); break; } - pos += chunk; - if last { break; } - } - if std::env::var("HASH").ok().and_then(|h| h.parse::().ok()) == Some(s) { - let h = |d: &[u8]| d.iter().fold(0xcbf29ce484222325u64, |h, &b| (h ^ b as u64).wrapping_mul(0x100000001b3)); - eprintln!("HASH seed {s} C {:x} Rust {:x}", h(&oa_all), h(&ob_all)); - } - if oa_all != ob_all { - let at = oa_all.iter().zip(&ob_all).position(|(x, y)| x != y); - if let Ok(d) = std::env::var("DUMP") { - std::fs::write(format!("{d}/prev.bin"), &all_src).unwrap(); - std::fs::write(format!("{d}/src.bin"), &src).unwrap(); - std::fs::write(format!("{d}/dict.bin"), &dict).unwrap(); - std::fs::write(format!("{d}/c.zst"), &oa_all).unwrap(); - std::fs::write(format!("{d}/rust.zst"), &ob_all).unwrap(); - } - panic!("tdict: {} vs {} first diff {at:?}\n{log:?}", oa_all.len(), ob_all.len()); - } - if failed { ok_frames = false; break; } - all_out.extend_from_slice(&oa_all); - all_src.extend_from_slice(&src); - } - // decode - let mut frames_bytes = all_out.clone(); - for round in 0..4 { - if round > 0 { - if frames_bytes.is_empty() { break; } - frames_bytes = all_out.clone(); - for _ in 0..1 + rng.below(3) { - let at = rng.below(frames_bytes.len()); - frames_bytes[at] ^= 1 << rng.below(8); - } - } - let cap = all_src.len() + rng.size(1000); - let (mut cd, mut rd) = (vec![0u8; cap], vec![0u8; cap]); - let mut ca = c::DCtx::new(); - let mut cb = z::DCtx::new(); - let la = ca.load_dict(&dict).map(|_| ()); - let lb = cb.load_dictionary(&dict).map_err(|e| e as u32); - assert_eq!(la, lb, "dctx load {log:?}"); - let x = ca.decompress(&mut cd, &frames_bytes); - let y = r2(cb.decompress(&mut rd, &frames_bytes)); - assert_eq!(x, y, "decode round {round} {log:?}"); - if let Ok(n) = x { assert!(cd[..n] == rd[..n]); if round == 0 && ok_frames && la.is_ok() { assert!(cd[..n] == all_src[..], "roundtrip {log:?}"); } } - // by reference ddict - if let Ok(dd) = z::DDict::by_reference(&dict) { - let y2 = r2(z::DCtx::new().decompress_using_ddict(&mut rd, &frames_bytes, &dd)); - assert_eq!(x, y2, "ddict {log:?}"); - } - // streaming - let mut ca = c::DCtx::new(); - let mut cb = z::DCtx::new(); - let _ = ca.load_dict(&dict); let _ = cb.load_dictionary(&dict); - let mut ipos = 0; let mut stalls = 0; - let small = frames_bytes.len() < 4000 && all_src.len() < 30_000; - let inmax = *rng.pick(if small { &[7usize, 300, 5000, 100_000, 10_000_000] } else { &[3000usize, 20_000, 5000, 100_000, 10_000_000] }); - let outmax = *rng.pick(if small { &[50usize, 4096, 16384, 200_000, 1_000_000] } else { &[5000usize, 4096, 16384, 200_000, 1_000_000] }); - let (mut x1f, mut x2f) = (vec![0u8; outmax], vec![0u8; outmax]); - loop { - let iend = (ipos + rng.range(if stalls > 0 {1} else {0}, inmax)).min(frames_bytes.len()); - let cap = rng.range(if stalls > 1 {1} else {0}, outmax); - let chunk = &frames_bytes[ipos..iend]; - let (x1, x2) = (&mut x1f[..cap], &mut x2f[..cap]); - let (cr, co, ci) = ca.stream(x1, 0, chunk, 0); - let mut ob = z::OutBuffer { dst: x2, pos: 0 }; - let mut ib = z::InBuffer { src: chunk, pos: 0 }; - let rr = r2(cb.decompress_stream(&mut ob, &mut ib)); - let (ro, ri) = (ob.pos, ib.pos); - assert_eq!((cr, co, ci), (rr, ro, ri), "stream {log:?}"); - assert!(x1[..co] == x2[..ro]); - ipos += ri; - if cr.is_err() { break; } - if ri == 0 && ro == 0 { stalls += 1 } else { stalls = 0 } - if ipos == frames_bytes.len() && (cr == Ok(0) || stalls > 3) { break; } - } - } - } - println!("tdict ok seed {seed} iters {iters}"); -} - -/// A very long frame, so that indices are brought down. -pub fn long(seed: u64, total: u64, level: i32, wlog: i32, extra: &[(i32, i32)]) { - let mut rng = Rng(seed); - let mut a = c::CCtx::new(); - let mut b = z::CCtx::new(); - a.set(100, level).unwrap(); b.set_parameter(100, level).unwrap(); - if wlog != 0 { a.set(101, wlog).unwrap(); b.set_parameter(101, wlog).unwrap(); } - a.set(201, 1).unwrap(); b.set_parameter(201, 1).unwrap(); - for &(p, v) in extra { assert_eq!(a.set(p as u32, v), r2(b.set_parameter(p, v))); } - let base = data(&mut rng, 8 << 20); - let mut done = 0u64; - let cap = z::compress_bound(4 << 20) + 1024; - let (mut oa, mut ob) = (vec![0u8; cap], vec![0u8; cap]); - let mut da = c::DCtx::new(); - let mut db = z::DCtx::new(); - da.set(100, 31).unwrap(); db.set_parameter(100, 31).unwrap(); - let mut dec_a = vec![0u8; 8 << 20]; - let mut dec_b = vec![0u8; 8 << 20]; - let mut hash_src = 0u64; let mut hash_dec = 0u64; - let fnv = |h: u64, d: &[u8]| { let mut h = h; for c in d.chunks(4096) { h = (h ^ c[0] as u64 ^ ((c[c.len()-1] as u64) << 8) ^ ((c.len() as u64) << 16)).wrapping_mul(0x100000001b3); } h }; - let _ = fnv; - let mut srcsum = 0u64; let mut decsum = 0u64; - while done < total { - let st = rng.below(base.len() - (1 << 20)); - let n = rng.range(1, 1 << 20); - let mut chunk = base[st..st + n].to_vec(); - if rng.chance(1, 2) { let at = rng.below(n); chunk[at] = rng.next() as u8; } - done += n as u64; - let end = if done >= total { 2 } else { 0 }; - for &x in chunk.iter().step_by(97) { srcsum = srcsum.wrapping_mul(31).wrapping_add(x as u64); } - let mut ip = 0; - loop { - let (ra, pa, ia) = a.stream2(&mut oa, 0, &chunk, ip, end); - let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; - let mut i = z::InBuffer { src: &chunk, pos: ip }; - let rb = r2(b.compress_stream2(&mut o, &mut i, ed(end))); - let (pb, ib) = (o.pos, i.pos); - assert_eq!((ra, pa, ia), (rb, pb, ib), "long at {done}"); - assert!(oa[..pa] == ob[..pb], "long bytes at {done}"); - ip = ia; - // decode - let mut dp = 0; - loop { - let (cr, co, ci) = da.stream(&mut dec_a, 0, &oa[..pa], dp); - let mut o2 = z::OutBuffer { dst: &mut dec_b, pos: 0 }; - let mut i2 = z::InBuffer { src: &oa[..pa], pos: dp }; - let rr = r2(db.decompress_stream(&mut o2, &mut i2)); - assert_eq!((cr, co, ci), (rr, o2.pos, i2.pos), "long dec at {done}"); - assert!(dec_a[..co] == dec_b[..co], "long dec bytes at {done}"); - cr.unwrap(); - dp = ci; - hash_dec = hash_dec.wrapping_add(co as u64); - let _ = &mut decsum; - if dp == pa && co < dec_a.len() { break; } - } - if (end == 0 && ip == chunk.len()) || (end == 2 && ra == Ok(0)) { break; } - } - hash_src = hash_src.wrapping_add(n as u64); - } - assert_eq!(hash_src, hash_dec, "sizes"); - println!("long ok: {done} bytes level {level} wlog {wlog} {extra:?} {srcsum}"); -} - -/// Two frames on one context with a dictionary. -pub fn repro(dict_path: &str, sample: &str, level: i32, wlog: i32, n1: usize, n2: usize, pledge: bool) { - let dict = std::fs::read(dict_path).unwrap(); - let f = std::fs::read(sample).unwrap(); - let mut a = c::CCtx::new(); - let mut b = z::CCtx::new(); - a.set(100, level).unwrap(); b.set_parameter(100, level).unwrap(); - if wlog != 0 { a.set(101, wlog).unwrap(); b.set_parameter(101, wlog).unwrap(); } - a.load_dict(&dict).unwrap(); b.load_dictionary(&dict).unwrap(); - for (k, &n) in [n1, n2].iter().enumerate() { - let src = &f[..n.min(f.len())]; - if pledge { a.pledge(src.len() as u64).unwrap(); b.set_pledged_src_size(src.len() as u64).unwrap(); } - let cap = z::compress_bound(src.len()) + 100; - let (mut oa, mut ob) = (vec![0u8; cap], vec![0u8; cap]); - let (ra, pa, _) = a.stream2(&mut oa, 0, src, 0, 2); - let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; - let mut i = z::InBuffer { src, pos: 0 }; - let rb = r2(b.compress_stream2(&mut o, &mut i, z::EndDirective::End)); - let pb = o.pos; - let same = oa[..pa] == ob[..pb]; - let at = oa[..pa].iter().zip(&ob[..pb]).position(|(x, y)| x != y); - println!("frame {k}: len {} C {ra:?} {pa} Rust {rb:?} {pb} same {same} first diff {at:?}", src.len()); - if !same { - println!(" C : {:02x?}", &oa[..40.min(pa)]); - println!(" Rust: {:02x?}", &ob[..40.min(pb)]); - let ndiff = oa[..pa].iter().zip(&ob[..pb]).filter(|(x, y)| x != y).count(); - println!(" bytes differing: {ndiff}"); - // does the Rust frame decode? - let mut d = c::DCtx::new(); d.load_dict(&dict).unwrap(); - let mut out = vec![0u8; src.len()]; - let r = d.decompress(&mut out, &ob[..pb]); - println!(" C decoder on the Rust frame: {r:?} content ok {}", out == src); - } - } -} - -pub fn bench(path: &str) { - use std::time::Instant; - let src = std::fs::read(path).unwrap(); - let cap = z::compress_bound(src.len()); - let mut out_c = vec![0u8; cap]; - let mut out_r = vec![0u8; cap]; - let mut dec = vec![0u8; src.len()]; - println!("corpus {} bytes", src.len()); - for level in [1, 3, 5, 7, 9, 12, 15, 19] { - let reps = if level >= 15 { 1 } else if level >= 9 { 2 } else { 4 }; - let mut tc = f64::MAX; let mut tr = f64::MAX; let mut nc = 0; let mut nr = 0; - for _ in 0..reps { - let t = Instant::now(); - nc = c::res(unsafe { c::ZSTD_compress(out_c.as_mut_ptr().cast(), cap, src.as_ptr().cast(), src.len(), level) }).unwrap(); - tc = tc.min(t.elapsed().as_secs_f64()); - let t = Instant::now(); - nr = z::compress(&mut out_r, &src, level).unwrap(); - tr = tr.min(t.elapsed().as_secs_f64()); - } - assert!(out_c[..nc] == out_r[..nr]); - let mut dc = f64::MAX; let mut dr = f64::MAX; - for _ in 0..5 { - let t = Instant::now(); - c::decompress(&mut dec, &out_c[..nc]).unwrap(); - dc = dc.min(t.elapsed().as_secs_f64()); - let t = Instant::now(); - z::decompress(&mut dec, &out_c[..nc]).unwrap(); - dr = dr.min(t.elapsed().as_secs_f64()); - } - let mb = src.len() as f64 / 1e6; - println!("level {level:2}: compress C {:7.1} MB/s Rust {:7.1} MB/s ({:.2}x slower) | decompress C {:7.1} MB/s Rust {:7.1} MB/s ({:.2}x slower)", mb / tc, mb / tr, tr / tc, mb / dc, mb / dr, dr / dc); - } -} - -/// Many frames on one context, so that indices get close to the limit between frames. -pub fn frames(seed: u64, total: u64, level: i32, max_frame: usize, oneshot_end: bool) { - let mut rng = Rng(seed); - let mut a = c::CCtx::new(); - let mut b = z::CCtx::new(); - a.set(100, level).unwrap(); b.set_parameter(100, level).unwrap(); - let base = data(&mut rng, max_frame + (1 << 20)); - let cap = z::compress_bound(max_frame) + 1024; - let (mut oa, mut ob) = (vec![0u8; cap], vec![0u8; cap]); - let mut done = 0u64; let mut n = 0; - while done < total { - let len = rng.range(max_frame / 2, max_frame); - let st = rng.below(base.len() - len); - let src = &base[st..st + len]; - if oneshot_end { - let (ra, pa, ia) = a.stream2(&mut oa, 0, src, 0, 2); - let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; - let mut i = z::InBuffer { src, pos: 0 }; - let rb = r2(b.compress_stream2(&mut o, &mut i, z::EndDirective::End)); - assert_eq!((ra, pa, ia), (rb, o.pos, i.pos), "frame {n} at {done}"); - assert!(oa[..pa] == ob[..pa], "frame {n} bytes at {done}"); - } else { - let mut pos = 0; - loop { - let chunk = (1 << 20).min(len - pos); - let end = if pos + chunk == len { 2 } else { 0 }; - let mut ip = 0; - loop { - let (ra, pa, ia) = a.stream2(&mut oa, 0, &src[pos..pos + chunk], ip, end); - let mut o = z::OutBuffer { dst: &mut ob, pos: 0 }; - let mut i = z::InBuffer { src: &src[pos..pos + chunk], pos: ip }; - let rb = r2(b.compress_stream2(&mut o, &mut i, ed(end))); - assert_eq!((ra, pa, ia), (rb, o.pos, i.pos), "frame {n} at {done}"); - assert!(oa[..pa] == ob[..pa], "frame {n} bytes at {done}"); - ip = ia; - if (end == 0 && ip == chunk) || (end == 2 && ra == Ok(0)) { break; } - } - pos += chunk; - if end == 2 { break; } - } - } - done += len as u64; n += 1; - } - println!("frames ok: {n} frames {done} bytes level {level} oneshot {oneshot_end}"); -} - -/// One call on more than 4 GB. -pub fn huge(seed: u64, total: usize, level: i32) { - let mut rng = Rng(seed); - let base = data(&mut rng, 16 << 20); - let mut src = Vec::with_capacity(total); - while src.len() < total { - let st = rng.below(base.len() - (1 << 20)); - let n = rng.range(1, 1 << 20).min(total - src.len()); - src.extend_from_slice(&base[st..st + n]); - } - let cap = z::compress_bound(total); - let mut oa = vec![0u8; cap]; - let ra = c::res(unsafe { c::ZSTD_compress(oa.as_mut_ptr().cast(), cap, src.as_ptr().cast(), total, level) }); - let mut ob = vec![0u8; cap]; - let rb = r2(z::compress(&mut ob, &src, level)); - assert_eq!(ra, rb); - let n = ra.unwrap(); - assert!(oa[..n] == ob[..n], "huge bytes"); - drop(ob); - let mut dec = vec![0u8; total]; - let rd = r2(z::decompress(&mut dec, &oa[..n])); - assert_eq!(rd, Ok(total)); - assert!(dec == src, "huge roundtrip"); - let v = z::decompress_alloc(&oa[..n]).unwrap(); - assert!(v == src, "huge decompress_alloc"); - println!("huge ok {total} -> {n} level {level}"); -} - -/// Multithreaded frames that are given up, small outputs, contexts dropped in the middle. -pub fn mtx(seed: u64, iters: u64) { - for it in 0..iters { - let s = seed.wrapping_mul(1_000_003).wrapping_add(it); - let mut rng = Rng(s); - let rng = &mut rng; - if it % 50 == 0 { eprintln!("iter {it}"); } - let mut a = c::CCtx::new(); - let mut b = z::CCtx::new(); - let mut log = vec![format!("seed {s}")]; - let w = rng.range(1, 6) as i32; - for (p, v) in [(400, w), (100, *rng.pick(&[1, 1, 3, 3, 5, 7, 9, 13])), (201, rng.below(2) as i32), (401, *rng.pick(&[0, 1, 600_000, 1_000_000])), (101, *rng.pick(&[0, 0, 12, 17, 20])), (160, *rng.pick(&[0, 0, 0, 1]))] { - assert_eq!(a.set(p as u32, v), r2(b.set_parameter(p, v))); - log.push(format!("set {p} {v}")); - } - let base = data(rng, 6 << 20); - let nframes = rng.range(1, 4); - for f in 0..nframes { - let len = if rng.chance(1, 5) { rng.size(600_000) } else { rng.range(600_000, base.len()) }; - let src = &base[..len]; - let give_up_at = if rng.chance(1, 3) { Some(rng.below(len + 1)) } else { None }; - let pledge = rng.chance(1, 3) && give_up_at.is_none(); - if pledge { a.pledge(len as u64).unwrap(); b.set_pledged_src_size(len as u64).unwrap(); } - log.push(format!("frame {f} len {len} give up {give_up_at:?} pledge {pledge}")); - let outcap = *rng.pick(&[1usize, 7, 100, 4096, 16384, 1 << 20]); - let outcap = if len > 2_000_000 && outcap < 100 { 4096 } else { outcap }; - let inmax = *rng.pick(&[1000usize, 16384, 65536, 1 << 20, 8 << 20]); - let run = |which: u8, a: &mut c::CCtx, b: &mut z::CCtx, seed2: u64| -> (Vec, Option) { - let mut r = Rng(seed2); - let mut out = Vec::new(); - let mut buf = vec![0u8; outcap]; - let mut pos = 0; - loop { - let stop = give_up_at.unwrap_or(usize::MAX); - if pos >= stop { return (out, None); } - let chunk = r.range(1, inmax).min(len - pos).min(stop - pos); - let last = pos + chunk == len; - let end = if last { 2 } else { *r.pick(&[0u32, 0, 0, 0, 0, 1]) }; - let input = &src[pos..pos + chunk]; - let mut ip = 0; - loop { - let (res, op, np) = if which == 0 { a.stream2(&mut buf, 0, input, ip, end) } else { - let mut o = z::OutBuffer { dst: &mut buf, pos: 0 }; - let mut i = z::InBuffer { src: input, pos: ip }; - let res = r2(b.compress_stream2(&mut o, &mut i, ed(end))); - (res, o.pos, i.pos) - }; - out.extend_from_slice(&buf[..op]); ip = np; - match res { Err(e) => return (out, Some(e)), Ok(l) => if (end == 0 && ip == chunk) || (end != 0 && l == 0) { break; } } - } - pos += chunk; - if last { return (out, None); } - } - }; - let seed2 = rng.next(); - let (oa, ea) = run(0, &mut a, &mut b, seed2); - let (ob, eb) = run(1, &mut a, &mut b, seed2); - assert_eq!(ea, eb, "{log:?}"); - if give_up_at.is_none() && ea.is_none() { - // Flushes make sections whose cut depends on timing only when the job table is full; compare by decoding. - let mut dec = vec![0u8; len]; - assert_eq!(c::decompress(&mut dec, &ob), Ok(len), "{log:?}"); - assert!(dec == src, "roundtrip {log:?}"); - if oa != ob { log.push(format!("(frames differ: {} vs {})", oa.len(), ob.len())); eprintln!("differs: {log:?}"); } - } else { - if rng.chance(1, 4) { break; } // drop both in the middle of the frame - a.reset(1).unwrap(); b.reset_session(); - } - } - } - println!("mtx ok seed {seed} iters {iters}"); -} - -pub fn bench_small(path: &str) { - use std::time::Instant; - let all = std::fs::read(path).unwrap(); - for n in [0usize, 50, 300, 1000, 4000, 16000, 64000] { - for level in [1, 3, 9] { - let src = &all[1000..1000 + n]; - let cap = z::compress_bound(n); - let mut oc = vec![0u8; cap]; let mut or = vec![0u8; cap]; - let iters = 20000.min(40_000_000 / (n + 200)); - let mut nc = 0; let mut nr = 0; - let t = Instant::now(); - for _ in 0..iters { nc = c::res(unsafe { c::ZSTD_compress(oc.as_mut_ptr().cast(), cap, src.as_ptr().cast(), n, level) }).unwrap(); } - let tc = t.elapsed().as_secs_f64() / iters as f64; - let t = Instant::now(); - for _ in 0..iters { nr = z::compress(&mut or, src, level).unwrap(); } - let tr = t.elapsed().as_secs_f64() / iters as f64; - assert!(oc[..nc] == or[..nr]); - let mut dec = vec![0u8; n]; - let t = Instant::now(); - for _ in 0..iters { c::decompress(&mut dec, &oc[..nc]).unwrap(); } - let dc = t.elapsed().as_secs_f64() / iters as f64; - let t = Instant::now(); - for _ in 0..iters { z::decompress(&mut dec, &oc[..nc]).unwrap(); } - let dr = t.elapsed().as_secs_f64() / iters as f64; - println!("n {n:6} level {level}: compress C {:8.2} us Rust {:8.2} us ({:.2}x) | decompress C {:7.2} us Rust {:7.2} us ({:.2}x)", tc * 1e6, tr * 1e6, tr / tc, dc * 1e6, dr * 1e6, dr / dc); - } - } -} - -pub fn bench_mt(path: &str) { - use std::time::Instant; - let text = std::fs::read(path).unwrap(); - let mut rng = Rng(7); - // sections that take very different times: text, noise, zeros - let mut src = Vec::new(); - let sec = 1 << 20; - for k in 0..192 { - match k % 3 { 0 => { let st = rng.below(text.len() - sec); src.extend_from_slice(&text[st..st + sec]); } - 1 => { for _ in 0..sec / 8 { src.extend_from_slice(&rng.next().to_le_bytes()); } } - _ => src.resize(src.len() + sec, 0) } - } - let uniform: Vec = text.iter().cycle().take(src.len()).copied().collect(); - let cap = z::compress_bound(src.len()); - let mut out = vec![0u8; cap]; - for (name, data) in [("uneven", &src), ("uniform", &uniform)] { - for level in [3, 9] { - for w in [0, 1, 2, 4, 8] { - let mut best = [f64::MAX; 2]; - for _ in 0..2 { - let mut a = c::CCtx::new(); - a.set(100, level).unwrap(); a.set(400, w).unwrap(); a.set(401, 1 << 20).unwrap(); - let t = Instant::now(); - let (r, _, _) = a.stream2(&mut out, 0, data, 0, 2); assert_eq!(r, Ok(0)); - best[0] = best[0].min(t.elapsed().as_secs_f64()); - let mut b = z::CCtx::new(); - b.set_parameter(100, level).unwrap(); b.set_parameter(400, w).unwrap(); b.set_parameter(401, 1 << 20).unwrap(); - let t = Instant::now(); - let mut o = z::OutBuffer { dst: &mut out, pos: 0 }; - let mut i = z::InBuffer { src: data, pos: 0 }; - assert_eq!(b.compress_stream2(&mut o, &mut i, z::EndDirective::End), Ok(0)); - best[1] = best[1].min(t.elapsed().as_secs_f64()); - } - println!("{name:8} level {level} workers {w}: C {:6.0} ms Rust {:6.0} ms ({:.2}x)", best[0] * 1e3, best[1] * 1e3, best[1] / best[0]); - } - } - } -} - -pub fn bench_mt_flush(path: &str) { - use std::time::Instant; - let text = std::fs::read(path).unwrap(); - let mut out = vec![0u8; 1 << 20]; - for (level, wlog) in [(3, 0), (19, 0), (19, 27)] { - let n = 5000usize; - let mut a = c::CCtx::new(); - a.set(100, level).unwrap(); a.set(400, 2).unwrap(); if wlog != 0 { a.set(101, wlog).unwrap(); } - let t = Instant::now(); - for k in 0..n { let input = &text[k * 1000..k * 1000 + 1000]; let mut ip = 0; loop { let (r, _, np) = a.stream2(&mut out, 0, input, ip, 1); ip = np; if r.unwrap() == 0 { break; } } } - let tc = t.elapsed().as_secs_f64(); - let mut b = z::CCtx::new(); - b.set_parameter(100, level).unwrap(); b.set_parameter(400, 2).unwrap(); if wlog != 0 { b.set_parameter(101, wlog).unwrap(); } - let t = Instant::now(); - for k in 0..n { let input = &text[k * 1000..k * 1000 + 1000]; let mut ip = 0; loop { let mut o = z::OutBuffer { dst: &mut out, pos: 0 }; let mut i = z::InBuffer { src: input, pos: ip }; let r = b.compress_stream2(&mut o, &mut i, z::EndDirective::Flush).unwrap(); ip = i.pos; if r == 0 { break; } } } - let tr = t.elapsed().as_secs_f64(); - println!("level {level} wlog {wlog}: {n} flushes of 1000 B with 2 workers: C {:.1} us/flush, Rust {:.1} us/flush ({:.2}x)", tc / n as f64 * 1e6, tr / n as f64 * 1e6, tr / tc); - } -} diff --git a/src/zstd/wip/harness/src/main.rs b/src/zstd/wip/harness/src/main.rs deleted file mode 100644 index b4e43e2c2a8b..000000000000 --- a/src/zstd/wip/harness/src/main.rs +++ /dev/null @@ -1,31 +0,0 @@ -mod c; -mod comp; -mod data; -mod dec; -mod extra; -mod sizes; - -fn main() { - let args: Vec = std::env::args().collect(); - let cmd = args.get(1).map(String::as_str).unwrap_or(""); - let seed: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); - let iters: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); - match cmd { - "dec" => dec::run(seed, iters), - "sizes" => sizes::run(), - "bench_mt_flush" => extra::bench_mt_flush(&args[2]), - "bench_mt" => extra::bench_mt(&args[2]), - "bench_small" => extra::bench_small(&args[2]), - "mtx" => extra::mtx(seed, iters), - "frames" => extra::frames(seed, iters, args[4].parse().unwrap(), args[5].parse().unwrap(), args[6] == "1"), - "huge" => extra::huge(seed, iters as usize, args[4].parse().unwrap()), - "bench" => extra::bench(&args[2]), - "repro" => extra::repro(&args[2], &args[3], args[4].parse().unwrap(), args[5].parse().unwrap(), args[6].parse().unwrap(), args[7].parse().unwrap(), args[8] == "1"), - "tdict" => extra::tdict(seed, iters, &args[4], &args[5]), - "long" => { let ex: Vec<(i32,i32)> = args[6..].chunks(2).map(|c| (c[0].parse().unwrap(), c[1].parse().unwrap())).collect(); extra::long(seed, iters, args[4].parse().unwrap(), args[5].parse().unwrap(), &ex) } - "dict" => comp::run_dict(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(1 << 18)), - "mt" => comp::run_mt(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(8 << 20)), - "comp" => comp::run(seed, iters, args.get(4).and_then(|s| s.parse().ok()).unwrap_or(1 << 20)), - _ => eprintln!("usage: zstd_harness dec "), - } -} diff --git a/src/zstd/wip/harness/src/sizes.rs b/src/zstd/wip/harness/src/sizes.rs deleted file mode 100644 index 8e0a3dc711af..000000000000 --- a/src/zstd/wip/harness/src/sizes.rs +++ /dev/null @@ -1 +0,0 @@ -pub fn run() { println!("CCtx {} DCtx {}", size_of::(), size_of::()); } diff --git a/src/zstd/wip/harness/sync.sh b/src/zstd/wip/harness/sync.sh deleted file mode 100755 index e2d37754d968..000000000000 --- a/src/zstd/wip/harness/sync.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -# A copy of the crate with everything public, so that the harness can reach what Bun does not. -set -e -src=${1:-/root/repos/oven-sh/bun/src/zstd} -dst=$(dirname "$0")/zstd-pub -rm -rf "$dst" -mkdir -p "$dst" -cp -r "$src"/. "$dst"/ -find "$dst" -name '*.rs' -exec sed -i -E 's/pub\((crate|super)\) /pub /g; s/^mod /pub mod /; s/^ fn reset_session/ pub fn reset_session/; s/^(const |static |fn |struct |enum |type )/pub \1/; s/^use (common|decompress|compress)::/pub use \1::/' {} + -cat > "$dst/Cargo.toml" < Date: Tue, 27 Oct 2026 13:58:10 +0000 Subject: [PATCH 013/199] zstd: decode sequences as fast as the C library The loop that decodes and executes sequences took 50% more instructions than the C one. Now it works on local copies of its state, which stay in registers (what is behind a reference is stored and loaded again around every write to the output), and the fast path is written with slices that shrink instead of positions, so that the compiler sees that most bounds hold. A match that starts before the prefix takes the slow path. 15 MB of source text at level 3: 977 MB/s before, 1219 MB/s now, C 1217 MB/s. --- src/zstd/common/bitstream.rs | 44 +++-- src/zstd/decompress/block.rs | 352 ++++++++++++++++++++++------------- 2 files changed, 246 insertions(+), 150 deletions(-) diff --git a/src/zstd/common/bitstream.rs b/src/zstd/common/bitstream.rs index 035d5badeff9..0e7561c91e2b 100644 --- a/src/zstd/common/bitstream.rs +++ b/src/zstd/common/bitstream.rs @@ -98,11 +98,10 @@ pub(crate) enum DStreamStatus { /// `BIT_DStream_t` #[derive(Clone, Copy)] pub(crate) struct BitDStream<'a> { + /// The stream up to the end of what `container` was read from: its last 8 bytes, or all of it if it is shorter. src: &'a [u8], pub(crate) container: u64, pub(crate) bits_consumed: u32, - /// Where `container` was read from. - pos: usize, } impl<'a> BitDStream<'a> { @@ -121,7 +120,6 @@ impl<'a> BitDStream<'a> { src, container: u64::from_le_bytes(*tail), bits_consumed: padding, - pos: src.len() - CONTAINER_BYTES, }); } if last_byte == 0 { @@ -133,7 +131,6 @@ impl<'a> BitDStream<'a> { src, container: u64::from_le_bytes(bytes), bits_consumed: padding + (CONTAINER_BYTES - src.len()) as u32 * 8, - pos: 0, }) } @@ -141,10 +138,9 @@ impl<'a> BitDStream<'a> { #[inline] pub(crate) fn resume(src: &'a [u8], pos: usize, bits_consumed: u32) -> Result { Ok(Self { - src, + src: src.sub(..pos + CONTAINER_BYTES)?, container: read_le64(src, pos)?, bits_consumed, - pos, }) } @@ -187,18 +183,27 @@ impl<'a> BitDStream<'a> { value } + /// `ptr - start` + #[inline(always)] + fn pos(&self) -> usize { + self.src.len().saturating_sub(CONTAINER_BYTES) + } + + /// Goes `nb_bytes` back, which is no more than `pos`. #[inline(always)] - fn load(&mut self, pos: usize) { - self.pos = pos; - if let Ok(container) = read_le64(self.src, pos) { - self.container = container; + fn load(&mut self, nb_bytes: usize) { + if let Some(src) = self.src.get(..self.src.len().wrapping_sub(nb_bytes)) + && let Some(tail) = src.last_chunk() + { + self.src = src; + self.container = u64::from_le_bytes(*tail); } } /// `BIT_reloadDStreamFast` #[inline(always)] pub(crate) fn reload_fast(&mut self) -> DStreamStatus { - if self.pos < CONTAINER_BYTES { + if self.src.len() < 2 * CONTAINER_BYTES { return DStreamStatus::Overflow; } self.reload_internal() @@ -207,9 +212,9 @@ impl<'a> BitDStream<'a> { /// `BIT_reloadDStream_internal`: `pos` is at least 8 and at most 64 bits are consumed. #[inline(always)] fn reload_internal(&mut self) -> DStreamStatus { - let pos = self.pos.wrapping_sub((self.bits_consumed >> 3) as usize); + let nb_bytes = (self.bits_consumed >> 3) as usize; self.bits_consumed &= 7; - self.load(pos); + self.load(nb_bytes); DStreamStatus::Unfinished } @@ -219,10 +224,11 @@ impl<'a> BitDStream<'a> { if self.bits_consumed > CONTAINER_BITS { return DStreamStatus::Overflow; } - if self.pos >= CONTAINER_BYTES { + if self.src.len() >= 2 * CONTAINER_BYTES { return self.reload_internal(); } - if self.pos == 0 { + let pos = self.pos(); + if pos == 0 { if self.bits_consumed < CONTAINER_BITS { return DStreamStatus::EndOfBuffer; } @@ -230,18 +236,18 @@ impl<'a> BitDStream<'a> { } let mut nb_bytes = (self.bits_consumed >> 3) as usize; let mut result = DStreamStatus::Unfinished; - if self.pos < nb_bytes { - nb_bytes = self.pos; + if pos < nb_bytes { + nb_bytes = pos; result = DStreamStatus::EndOfBuffer; } self.bits_consumed -= nb_bytes as u32 * 8; - self.load(self.pos - nb_bytes); + self.load(nb_bytes); result } /// `BIT_endOfDStream` #[inline(always)] pub(crate) fn is_at_end(&self) -> bool { - self.pos == 0 && self.bits_consumed == CONTAINER_BITS + self.pos() == 0 && self.bits_consumed == CONTAINER_BITS } } diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 5d53cb4cbef2..63556b0307e3 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -527,31 +527,33 @@ impl BlockDecoder { self.fse_entropy = true; } - let block_end = start + capacity; let oend = if literals.location == LitLocation::InDst { start + block_size_max + WILDCOPY_OVERLENGTH } else { - block_end + start + capacity }; let split = literals.location == LitLocation::Split; - let in_dst_size = literals.size.saturating_sub(LIT_BUFFER_EXTRA_SIZE); + let lits = match literals.in_src { + Some(start) => src.sub(start..)?, + None => lit_buffer.as_slice(), + }; + // How many would be in the output buffer, if some. + let in_dst_size = if split { + literals.size - LIT_BUFFER_EXTRA_SIZE + } else { + literals.size + }; let mut exec = Executor { - buf: buf.sub_mut(..block_end)?, + buf: buf.sub_mut(..oend)?, op: start, - oend, prefix_start, ext_dict, - lits: match literals.in_src { - Some(start) => src.sub(start..)?, - None => lit_buffer.as_slice(), - }, - lit_pos: 0, - lit_limit: if split { in_dst_size } else { literals.size }, - lit_size: literals.size, + lits: lits.sub(..in_dst_size + WILDCOPY_OVERLENGTH)?, split, - // `litBufferEnd` is `WILDCOPY_OVERLENGTH` short of how far the block is expected to write. - lit_in_dst: (start + capacity.min(block_size_max)) - .wrapping_sub(WILDCOPY_OVERLENGTH + in_dst_size), + lits_extra: lits.sub(in_dst_size..literals.size + WILDCOPY_OVERLENGTH)?, + // `WILDCOPY_OVERLENGTH` short of how far the block is expected to write. + lit_end_in_dst: (start + capacity.min(block_size_max)) + .wrapping_sub(WILDCOPY_OVERLENGTH), }; if nb_seq > 0 { @@ -635,12 +637,13 @@ fn long_offset_share(of_table: &SeqTable<{ 1 << OFF_FSE_LOG }>) -> u32 { /// `seq_t` #[derive(Clone, Copy, Default)] struct Sequence { - lit_length: usize, - match_length: usize, + lit_length: u32, + match_length: u32, offset: usize, } /// `seqState_t` +#[derive(Clone, Copy)] struct SeqState<'a> { bit_d: BitDStream<'a>, state_ll: usize, @@ -688,8 +691,8 @@ impl<'a> SeqState<'a> { let ll_d_info = self.ll_table.get(self.state_ll); let ml_d_info = self.ml_table.get(self.state_ml); let of_d_info = self.of_table.get(self.state_off); - let mut match_length = ml_d_info.base_value as usize; - let mut lit_length = ll_d_info.base_value as usize; + let mut match_length = ml_d_info.base_value; + let mut lit_length = ll_d_info.base_value; let of_base = of_d_info.base_value as usize; let ll_bits = u32::from(ll_d_info.nb_additional_bits); let ml_bits = u32::from(ml_d_info.nb_additional_bits); @@ -724,14 +727,14 @@ impl<'a> SeqState<'a> { } if ml_bits > 0 { - match_length += self.bit_d.read_bits_fast(ml_bits) as usize; + match_length += self.bit_d.read_bits_fast(ml_bits) as u32; } const ACCUMULATOR_MIN: u32 = 57; if total_bits >= ACCUMULATOR_MIN - (LL_FSE_LOG + ML_FSE_LOG + OFF_FSE_LOG) { self.bit_d.reload(); } if ll_bits > 0 { - lit_length += self.bit_d.read_bits_fast(ll_bits) as usize; + lit_length += self.bit_d.read_bits_fast(ll_bits) as u32; } if !is_last_seq { @@ -778,39 +781,83 @@ fn copy_match_exact(buf: &mut [u8], src: usize, dst: usize, len: usize) -> Resul } /// Turns sequences into output. -struct Executor<'a> { - /// Ends where the block has to. - buf: &'a mut [u8], +struct Executor<'buf, 'a> { + /// Ends where the block has to stop writing: `oend`. + buf: &'buf mut [u8], op: usize, - /// How far the C code would let the block write. - oend: usize, prefix_start: usize, ext_dict: ExtDict<'a>, - /// At least `WILDCOPY_OVERLENGTH` longer than `lit_size`. + /// The literals that are left, up to `litBufferEnd`, and `WILDCOPY_OVERLENGTH` bytes after them. lits: &'a [u8], - lit_pos: usize, - /// `litBufferEnd` - lit_limit: usize, - lit_size: usize, - /// `litBufferLocation == ZSTD_split`: the literals up to `lit_limit` would be in the output buffer. + /// `litBufferLocation == ZSTD_split`: `lits` would be in the output buffer. split: bool, - /// Where in `buf` they would start. - lit_in_dst: usize, + /// The literals that come after `lits` then, likewise. + lits_extra: &'a [u8], + /// Where in `buf` `lits` would end then. + lit_end_in_dst: usize, } -impl Executor<'_> { - /// The match of a sequence whose literals end at `o_lit_end`. `FAST`: 32 bytes after the match may be written. +impl<'a> Executor<'_, 'a> { + /// `litBufferEnd - litPtr` #[inline(always)] - fn copy_match(&mut self, o_lit_end: usize, sequence: Sequence) -> Result<()> { - let Sequence { - mut match_length, - offset, - .. - } = sequence; - let mut op = o_lit_end; - let in_prefix = o_lit_end - self.prefix_start; - let mut match_pos = o_lit_end.wrapping_sub(offset); + fn lits_left(&self) -> usize { + self.lits.len().saturating_sub(WILDCOPY_OVERLENGTH) + } + + /// The same, for a while. A loop works on a copy that is a local variable and whose address nothing takes, which + /// can stay in registers. What is behind a reference is written and read again for every sequence. + #[inline(always)] + fn fork(&mut self) -> Executor<'_, 'a> { + Executor { + buf: &mut *self.buf, + ..*self + } + } + + #[inline(always)] + fn join(&mut self, (op, lits, split): (usize, &'a [u8], bool)) { + (self.op, self.lits, self.split) = (op, lits, split); + } + + #[inline(always)] + fn exec_sequence_end(&mut self, sequence: Sequence) -> Result<()> { + let mut fork = self.fork(); + fork.exec_sequence_end_cold(sequence)?; + let state = (fork.op, fork.lits, fork.split); + self.join(state); + Ok(()) + } + + /// `ZSTD_execSequenceEnd` and `ZSTD_execSequenceEndSplitLitBuffer`: nothing is written past the sequence. Also what + /// `ZSTD_execSequence` does of a match that starts before the prefix. + #[cold] + #[inline(never)] + fn exec_sequence_end_cold(&mut self, sequence: Sequence) -> Result<()> { + let lit_length = sequence.lit_length as usize; + let mut match_length = sequence.match_length as usize; + let offset = sequence.offset; + let mut op = self.op; + + if lit_length + match_length > self.buf.len() - op { + return Err(ErrorCode::DstSizeTooSmall); + } + if lit_length > self.lits_left() { + return Err(ErrorCode::CorruptionDetected); + } + if self.split { + let lit_ptr = self.lit_end_in_dst.wrapping_sub(self.lits_left()); + if op > lit_ptr && op < lit_ptr.wrapping_add(lit_length) { + return Err(ErrorCode::DstSizeTooSmall); + } + } + self.buf.write_at(op, self.lits.sub(..lit_length)?)?; + self.lits = self.lits.sub(lit_length..)?; + op += lit_length; + self.op = op + match_length; + + let in_prefix = op - self.prefix_start; + let mut match_pos = op.wrapping_sub(offset); if offset > in_prefix { if offset > in_prefix + self.ext_dict.len() { return Err(ErrorCode::CorruptionDetected); @@ -825,24 +872,85 @@ impl Executor<'_> { match_length -= back; match_pos = self.prefix_start; } + copy_match_exact(self.buf, match_pos, op, match_length) + } + + /// `ZSTD_execSequence` and `ZSTD_execSequenceSplitLitBuffer`, if there is room to copy in chunks and the match is + /// in the prefix. Otherwise `None`, and what is changed is only output that the sequence is to overwrite. + /// + /// The conditions are such that the compiler sees that most of the `?` cannot fail. + #[inline(always)] + fn exec_sequence_fast(&mut self, sequence: Sequence) -> Option<()> { + let lit_length = sequence.lit_length as usize; + let match_length = sequence.match_length as usize; + let offset = sequence.offset; + let op = self.op; + + let room = self.buf.get_mut(op..)?; + if lit_length + match_length + WILDCOPY_OVERLENGTH > room.len() + || lit_length + WILDCOPY_OVERLENGTH > self.lits.len() + { + return None; + } + if SPLIT { + // The output must not catch up with the literals. + let i_lit_end = (self.lit_end_in_dst) + .wrapping_sub(self.lits.len() - WILDCOPY_OVERLENGTH) + .wrapping_add(lit_length); + if op + lit_length + match_length + WILDCOPY_OVERLENGTH > i_lit_end { + return None; + } + } - if !FAST { - return copy_match_exact(self.buf, match_pos, op, match_length); + *room.first_chunk_mut::<16>()? = *self.lits.first_chunk()?; + if lit_length > 16 { + room.get_mut(16..lit_length)? + .copy_from_slice(self.lits.get(16..lit_length)?); } + let o_lit_end = op + lit_length; + if offset > o_lit_end - self.prefix_start { + return None; + } + let (before, room) = self.buf.split_at_mut_checked(o_lit_end)?; + let from = before.get(before.len().checked_sub(offset)?..)?; if offset >= 16 { // `ZSTD_wildcopy(ZSTD_no_overlap)` - let mut done = 0usize; - loop { - copy_within::<16>(self.buf, match_pos + done, op + done)?; - done += 16; - if done >= match_length { - return Ok(()); + *room.first_chunk_mut::<16>()? = *from.first_chunk()?; + if match_length > 16 { + let rounded = (match_length + 15) & !15; + if offset >= rounded { + room.get_mut(16..rounded)? + .copy_from_slice(from.get(16..rounded)?); + } else { + let mut done = 16; + while done < match_length { + let to = o_lit_end + done; + copy_within::<16>(self.buf, to - offset, to).ok()?; + done += 16; + } } } + } else { + self.copy_close_match(o_lit_end, offset, match_length) + .ok()?; } - // `ZSTD_overlapCopy8`: after 8 bytes, the source is at least 8 bytes behind. + self.lits = self.lits.get(lit_length..)?; + self.op = o_lit_end + match_length; + Some(()) + } + + /// `ZSTD_overlapCopy8` and `ZSTD_wildcopy(ZSTD_overlap_src_before_dst)`: a match less than 16 bytes back. + #[inline(always)] + fn copy_close_match( + &mut self, + mut op: usize, + offset: usize, + match_length: usize, + ) -> Result<()> { + let mut match_pos = op - offset; + // After 8 bytes, the source is at least 8 bytes behind. if offset < 8 { const DEC32_TABLE: [usize; 8] = [0, 1, 2, 1, 4, 4, 4, 4]; const DEC64_TABLE: [usize; 8] = [8, 8, 8, 7, 8, 9, 10, 11]; @@ -859,7 +967,6 @@ impl Executor<'_> { match_pos += 8; } op += 8; - // `ZSTD_wildcopy(ZSTD_overlap_src_before_dst)` let mut done = 8usize; while done < match_length { copy_within::<8>(self.buf, match_pos, op)?; @@ -870,92 +977,71 @@ impl Executor<'_> { Ok(()) } - /// `ZSTD_execSequenceEnd` and `ZSTD_execSequenceEndSplitLitBuffer`: near the end of a buffer. - #[cold] - #[inline(never)] - fn exec_sequence_end(&mut self, sequence: Sequence) -> Result<()> { - let op = self.op; - let o_lit_end = op + sequence.lit_length; - let sequence_length = sequence.lit_length + sequence.match_length; - - if sequence_length > self.oend - op { - return Err(ErrorCode::DstSizeTooSmall); - } - if sequence.lit_length > self.lit_limit - self.lit_pos { - return Err(ErrorCode::CorruptionDetected); - } - if self.split { - let lit_ptr = self.lit_in_dst.wrapping_add(self.lit_pos); - if op > lit_ptr && op < lit_ptr + sequence.lit_length { - return Err(ErrorCode::DstSizeTooSmall); - } + #[inline(always)] + fn exec_sequence(&mut self, sequence: Sequence) -> Result<()> { + match self.exec_sequence_fast::(sequence) { + Some(()) => Ok(()), + None => self.exec_sequence_end(sequence), } - - self.buf - .write_at(op, self.lits.span(self.lit_pos, sequence.lit_length)?)?; - self.lit_pos += sequence.lit_length; - self.copy_match::(o_lit_end, sequence)?; - self.op = op + sequence_length; - Ok(()) } - /// `ZSTD_execSequence` and `ZSTD_execSequenceSplitLitBuffer` + /// The first sequence whose literals do not end among those that would be in the output buffer. #[inline(always)] - fn exec_sequence(&mut self, sequence: Sequence) -> Result<()> { - let op = self.op; - let o_lit_end = op + sequence.lit_length; - let o_match_end = o_lit_end + sequence.match_length; - let i_lit_end = self.lit_pos + sequence.lit_length; - // How far a copy in chunks may write, plus `WILDCOPY_OVERLENGTH`. - let limit = if self.split { - self.lit_in_dst.wrapping_add(i_lit_end) - } else { - self.oend - }; - - if i_lit_end > self.lit_limit || o_match_end + WILDCOPY_OVERLENGTH > limit { - return self.exec_sequence_end(sequence); - } + fn exec_sequence_leaving_split(&mut self, sequence: Sequence) -> Result<()> { + let mut fork = self.fork(); + fork.exec_sequence_leaving_split_cold(sequence)?; + let state = (fork.op, fork.lits, fork.split); + self.join(state); + Ok(()) + } - *self.buf.array_at_mut::<16>(op)? = *self.lits.array_at(self.lit_pos)?; - if sequence.lit_length > 16 { - self.buf - .write_at(op + 16, self.lits.sub(self.lit_pos + 16..i_lit_end)?)?; + #[cold] + #[inline(never)] + fn exec_sequence_leaving_split_cold(&mut self, mut sequence: Sequence) -> Result<()> { + let leftover_lit = self.lits_left(); + if leftover_lit > 0 { + if leftover_lit > self.buf.len() - self.op { + return Err(ErrorCode::DstSizeTooSmall); + } + self.buf.write_at(self.op, self.lits.sub(..leftover_lit)?)?; + sequence.lit_length -= leftover_lit as u32; + self.op += leftover_lit; } - self.lit_pos = i_lit_end; - self.copy_match::(o_lit_end, sequence)?; - self.op = o_match_end; - Ok(()) + self.split = false; + self.lits = self.lits_extra; + self.exec_sequence::(sequence) } - /// A sequence, whose literals may start among those that would be in the output buffer and end among the others. #[inline(always)] - fn exec_sequence_maybe_split(&mut self, mut sequence: Sequence) -> Result<()> { - if self.split && self.lit_pos + sequence.lit_length > self.lit_limit { - let leftover_lit = self.lit_limit - self.lit_pos; - if leftover_lit > 0 { - if leftover_lit > self.oend - self.op { - return Err(ErrorCode::DstSizeTooSmall); - } - self.buf - .write_at(self.op, self.lits.span(self.lit_pos, leftover_lit)?)?; - sequence.lit_length -= leftover_lit; - self.op += leftover_lit; - self.lit_pos += leftover_lit; - } - self.split = false; - self.lit_limit = self.lit_size; + fn exec_sequence_maybe_split(&mut self, sequence: Sequence) -> Result<()> { + if !self.split { + self.exec_sequence::(sequence) + } else if sequence.lit_length as usize > self.lits_left() { + self.exec_sequence_leaving_split(sequence) + } else { + self.exec_sequence::(sequence) } - self.exec_sequence(sequence) } /// `ZSTD_decompressSequences_body` and `ZSTD_decompressSequences_bodySplitLitBuffer` fn run(&mut self, seq_state: &mut SeqState<'_>, nb_seq: usize) -> Result<()> { - for remaining in (1..=nb_seq).rev() { - let sequence = seq_state.decode_sequence(remaining == 1); - self.exec_sequence_maybe_split(sequence)?; + let mut exec = self.fork(); + let mut state = *seq_state; + let mut remaining = nb_seq; + while exec.split && remaining > 0 { + let sequence = state.decode_sequence(remaining == 1); + remaining -= 1; + exec.exec_sequence_maybe_split(sequence)?; } - if !seq_state.bit_d.is_at_end() { + while remaining > 0 { + let sequence = state.decode_sequence(remaining == 1); + remaining -= 1; + exec.exec_sequence::(sequence)?; + } + let exec = (exec.op, exec.lits, exec.split); + self.join(exec); + *seq_state = state; + if !state.bit_d.is_at_end() { return Err(ErrorCode::CorruptionDetected); } Ok(()) @@ -987,13 +1073,17 @@ impl Executor<'_> { /// The literals after the last sequence. fn last_literals(&mut self) -> Result<()> { - let last_ll_size = self.lit_size - self.lit_pos; - if last_ll_size > self.oend - self.op { + let extra = if self.split { self.lits_extra } else { &[] }; + let parts = [self.lits, extra] + .map(|part| part.get(..part.len().saturating_sub(WILDCOPY_OVERLENGTH))); + let parts = parts.map(Option::unwrap_or_default); + if parts.iter().map(|part| part.len()).sum::() > self.buf.len() - self.op { return Err(ErrorCode::DstSizeTooSmall); } - self.buf - .write_at(self.op, self.lits.span(self.lit_pos, last_ll_size)?)?; - self.op += last_ll_size; + for part in parts { + self.buf.write_at(self.op, part)?; + self.op += part.len(); + } Ok(()) } } From a20882ebbc25e7fb8e5bc06aef5a476908916f48 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 14:22:47 +0000 Subject: [PATCH 014/199] zstd: faster match finders and sequence encoder, and what mordant found - fast and dfast without a dictionary work on 32-bit positions in the part of the input that they can read (Near), where a bounds check is one comparison, read each position once, and index tables through a mask. An index that is too low wraps around and fails the same check as one that is out of bounds. - SeqSink: the sequence store as a local variable while a hot loop adds to it. - The row match finder compares 8 tags at a time instead of one, and reads the bytes that a longer match has to have once per length instead of once per candidate. Level 12 goes from 1.43x to 1.16x the time of the C library. - The bit writer clamps its position where it writes, which removes the check. - The parts of generic functions that do not depend on the parameter are functions of their own (NoDictBlock, TwoSegments, SavedRep, Unsorted, ChainSearch), tuples of two of the same type are a struct or a range, and the size of the loaded header is in the stage that it belongs to. --- src/zstd/common/bitstream.rs | 14 +- src/zstd/compress/double_fast.rs | 583 ++++++++++++------------- src/zstd/compress/fast.rs | 708 +++++++++++++++++++------------ src/zstd/compress/fse.rs | 11 +- src/zstd/compress/internal.rs | 194 ++++++++- src/zstd/compress/lazy.rs | 193 ++++++--- src/zstd/compress/ldm.rs | 21 +- src/zstd/compress/params.rs | 49 +-- src/zstd/compress/sequences.rs | 28 +- src/zstd/decompress/huf.rs | 3 +- src/zstd/decompress/mod.rs | 94 ++-- src/zstd/lib.rs | 2 +- 12 files changed, 1162 insertions(+), 738 deletions(-) diff --git a/src/zstd/common/bitstream.rs b/src/zstd/common/bitstream.rs index 0e7561c91e2b..6798ff23443d 100644 --- a/src/zstd/common/bitstream.rs +++ b/src/zstd/common/bitstream.rs @@ -50,13 +50,14 @@ impl<'a> BitCStream<'a> { #[inline(always)] fn flush(&mut self) { let nb_bytes = (self.bit_pos >> 3) as usize; - if let Ok(out) = self.dst.array_at_mut(self.pos) { + // At the end of the output the stream stops advancing, which `close` reports. + let pos = self.pos.min(self.end); + if let Ok(out) = self.dst.array_at_mut(pos) { *out = self.container.to_le_bytes(); } - self.pos += nb_bytes; + self.pos = pos + nb_bytes; self.bit_pos &= 7; - // All 8 bytes leave only if `add_bits` was given more than fits. - self.container = self.container.checked_shr(nb_bytes as u32 * 8).unwrap_or(0); + self.container >>= (nb_bytes as u32 * 8) & (CONTAINER_BITS - 1); } /// `BIT_flushBitsFast`: the caller knows that the output is large enough. @@ -65,13 +66,10 @@ impl<'a> BitCStream<'a> { self.flush(); } - /// `BIT_flushBits`: at the end of the output the stream stops advancing, which [`Self::close`] reports. + /// `BIT_flushBits` #[inline(always)] pub(crate) fn flush_bits(&mut self) { self.flush(); - if self.pos > self.end { - self.pos = self.end; - } } /// `BIT_closeCStream`: the size of the stream, or 0 if it did not fit. diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 741c372c3cb6..a4b99966c902 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -1,11 +1,11 @@ //! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. -use super::fast::{FAST_HASH_FILL_STEP, Located, count_backwards, restore_rep}; +use super::fast::{FAST_HASH_FILL_STEP, NoDictBlock, TwoSegments, count_backwards, restore_rep}; use super::internal::{ BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, - compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, - write_tagged_index, + REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, count, count_2segments, hash_of, hash_ptr, hash8, index_overlap_check, + offset_to_offbase, table_get, table_put, write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -110,112 +110,101 @@ fn compress_block_double_fast_no_dict_generic( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { - const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; + const K_STEP_INCR: u32 = 1 << K_SEARCH_STRENGTH; + let Some(block) = NoDictBlock::new(ms, rep, src)? else { + return Ok(src.end - src.start); + }; + let NoDictBlock { + near, + base_index, + candidates, + prefix_lowest_index, + prefix_lowest, + ilimit, + mut offset_1, + mut offset_2, + .. + } = block; let h_bits_l = ms.c_params.hash_log; let h_bits_s = ms.c_params.chain_log; - let window_log = ms.c_params.window_log; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, window_log); - let prefix_lowest = src.pos_of(prefix_lowest_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip = istart + usize::from(istart == prefix_lowest); - let [mut offset_1, mut offset_2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let current = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(current, window_log); - let max_rep = current.wrapping_sub(window_low); - if offset_2 > max_rep { - offset_saved2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved1 = offset_1; - offset_1 = 0; - } - } - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); + let mut anchor = block.istart; + let mut ip = block.first; + let mut seq_store = SeqSink::new(seq_store); + let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; + let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; // One turn per match. 'start: loop { - let mut step = 1usize; + let mut step = 1; let mut next_step = ip + K_STEP_INCR; let mut ip1 = ip + step; - if ip1 > ilimit { + if ip1 as usize > ilimit { break 'start; } - let mut hl0 = hash_ptr(buf, ip, h_bits_l, 8)?; - let mut idxl0 = hash_long.at(hl0)?; + // The 8 bytes at `ip`. + let mut bytes0 = near.read64(ip).ok_or(BUG)?; + let mut hl0 = hash8(bytes0, h_bits_l, 0); + let mut idxl0 = table_get(hash_long, hl0); let mut curr; let m_length = 'match_stored: { // One turn per position. Ends with the offset and the length of a match, which `ip` is moved to the start // of. let (offset, m_length, hl1) = loop { - let hs0 = hash_ptr(buf, ip, h_bits_s, MLS)?; - let idxs0 = hash_small.at(hs0)?; - curr = src.index_of(ip); - hash_long.put(hl0, curr)?; - hash_small.put(hs0, curr)?; - - if (offset_1 > 0) - & (read_le32(buf, (ip + 1).wrapping_sub(offset_1 as usize))? - == read_le32(buf, ip + 1)?) + let hs0 = hash_of::(bytes0, h_bits_s); + let idxs0 = table_get(hash_small, hs0); + curr = base_index.wrapping_add(ip); + table_put(hash_long, hl0, curr); + table_put(hash_small, hs0, curr); + + if offset_1 > 0 + && near.read32((ip + 1).wrapping_sub(offset_1)) == Some((bytes0 >> 8) as u32) { - let m_length = count( - buf, - ip + 1 + 4, - buf, - (ip + 1 + 4).wrapping_sub(offset_1 as usize), - iend, - )? + 4; ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + let m_length = near.count(ip + 4, (ip + 4).wrapping_sub(offset_1)) + 4; + seq_store.store_seq( + ip - anchor, + near, + anchor, + REPCODE1_TO_OFFBASE, + m_length, + )?; break 'match_stored m_length; } - let hl1 = hash_ptr(buf, ip1, h_bits_l, 8)?; + let bytes1 = near.read64(ip1).ok_or(BUG)?; + let hl1 = hash8(bytes1, h_bits_l, 0); - let matchl0 = src.pos_of(idxl0); - if idxl0 >= prefix_lowest_index && read_le64(buf, matchl0)? == read_le64(buf, ip)? { - let m_length = count(buf, ip + 8, buf, matchl0 + 8, iend)? + 8; - let offset = ip.wrapping_sub(matchl0) as u32; - let back = count_backwards(buf, ip, anchor, buf, matchl0, prefix_lowest); + if candidates.read64(idxl0.wrapping_sub(prefix_lowest_index)) == Some(bytes0) { + let matchl0 = idxl0 - base_index; + let m_length = near.count(ip + 8, matchl0 + 8) + 8; + let offset = ip - matchl0; + let back = near.count_backwards(ip, anchor, matchl0, prefix_lowest); ip -= back; break (offset, m_length + back, hl1); } - let idxl1 = hash_long.at(hl1)?; + let idxl1 = table_get(hash_long, hl1); - let mut matchs0 = src.pos_of(idxs0); - if idxs0 >= prefix_lowest_index && read_le32(buf, matchs0)? == read_le32(buf, ip)? { + if candidates.read32(idxs0.wrapping_sub(prefix_lowest_index)) == Some(bytes0 as u32) + { // A short match: a longer one at the next position is preferred. - let mut m_length = count(buf, ip + 4, buf, matchs0 + 4, iend)? + 4; - let mut offset = ip.wrapping_sub(matchs0) as u32; - let matchl1 = src.pos_of(idxl1); - if idxl1 > prefix_lowest_index - && read_le64(buf, matchl1)? == read_le64(buf, ip1)? - { - let l1len = count(buf, ip1 + 8, buf, matchl1 + 8, iend)? + 8; + let mut matchs0 = idxs0 - base_index; + let mut m_length = near.count(ip + 4, matchs0 + 4) + 4; + let mut offset = ip - matchs0; + let matchl1 = idxl1.wrapping_sub(base_index); + if idxl1 > prefix_lowest_index && near.read64(matchl1) == Some(bytes1) { + let l1len = near.count(ip1 + 8, matchl1 + 8) + 8; if l1len > m_length { ip = ip1; m_length = l1len; - offset = ip.wrapping_sub(matchl1) as u32; + offset = ip - matchl1; matchs0 = matchl1; } } - let back = count_backwards(buf, ip, anchor, buf, matchs0, prefix_lowest); + let back = near.count_backwards(ip, anchor, matchs0, prefix_lowest); ip -= back; break (offset, m_length + back, hl1); } @@ -227,9 +216,10 @@ fn compress_block_double_fast_no_dict_generic( ip = ip1; ip1 += step; + bytes0 = bytes1; hl0 = hl1; idxl0 = idxl1; - if ip1 > ilimit { + if ip1 as usize > ilimit { break 'start; } }; @@ -238,11 +228,11 @@ fn compress_block_double_fast_no_dict_generic( offset_1 = offset; // With a larger step, ip1 may be past the end of the match. if step < 4 { - hash_long.put(hl1, src.index_of(ip1))?; + table_put(hash_long, hl1, base_index.wrapping_add(ip1)); } seq_store.store_seq( ip - anchor, - buf, + near, anchor, offset_to_offbase(offset), m_length, @@ -253,37 +243,53 @@ fn compress_block_double_fast_no_dict_generic( ip += m_length; anchor = ip; - if ip <= ilimit { + if ip as usize <= ilimit { let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; - - while ip <= ilimit - && (offset_2 > 0) - & (read_le32(buf, ip)? == read_le32(buf, ip.wrapping_sub(offset_2 as usize))?) - { - let r_length = count( - buf, - ip + 4, - buf, - (ip + 4).wrapping_sub(offset_2 as usize), - iend, - )? + 4; + let to_insert = near.read64(index_to_insert - base_index).ok_or(BUG)?; + let before2 = near.read64(ip - 2).ok_or(BUG)?; + let before1 = near.read64(ip - 1).ok_or(BUG)?; + table_put(hash_long, hash8(to_insert, h_bits_l, 0), index_to_insert); + table_put( + hash_long, + hash8(before2, h_bits_l, 0), + base_index.wrapping_add(ip - 2), + ); + table_put( + hash_small, + hash_of::(to_insert, h_bits_s), + index_to_insert, + ); + table_put( + hash_small, + hash_of::(before1, h_bits_s), + base_index.wrapping_add(ip - 1), + ); + + while ip as usize <= ilimit && offset_2 > 0 { + let bytes = near.read64(ip).ok_or(BUG)?; + if near.read32(ip.wrapping_sub(offset_2)) != Some(bytes as u32) { + break; + } + let r_length = near.count(ip + 4, (ip + 4).wrapping_sub(offset_2)) + 4; core::mem::swap(&mut offset_1, &mut offset_2); - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, src.index_of(ip))?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, src.index_of(ip))?; - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; + let index = base_index.wrapping_add(ip); + table_put(hash_small, hash_of::(bytes, h_bits_s), index); + table_put(hash_long, hash8(bytes, h_bits_l, 0), index); + seq_store.store_seq(0, near, anchor, REPCODE1_TO_OFFBASE, r_length)?; ip += r_length; anchor = ip; } } } - restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); - Ok(iend - anchor) + restore_rep( + rep, + offset_1, + offset_2, + block.offset_saved1, + block.offset_saved2, + ); + Ok((near.len() - anchor) as usize) } /// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` @@ -292,63 +298,27 @@ fn compress_block_double_fast_dict_match_state_generic( seq_store: &mut SeqStore, rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, + seg: &TwoSegments<'_>, ) -> Result { let h_bits_l = ms.c_params.hash_log; let h_bits_s = ms.c_params.chain_log; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_lowest_index = ms.get_lowest_prefix_index(end_index, ms.c_params.window_log); - let prefix_lowest = src.pos_of(prefix_lowest_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; let [mut offset_1, mut offset_2, _] = *rep; - let dms = src.dms.as_ref().ok_or(BUG)?; let dict_hash_long = dms.ms.hash_table.as_slice(); let dict_hash_small = dms.ms.chain_table.as_slice(); - let dict_start_index = dms.ms.window.dict_limit; - let dict_start = dms.pos_of(dict_start_index); - let dict_end = dms.end; - // What to subtract from an index below prefix_lowest_index to get the index in the dictionary. - let dict_index_delta = prefix_lowest_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); let dict_h_bits_l = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; let dict_h_bits_s = dms.ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; - let dict_and_prefix_length = istart - .wrapping_sub(prefix_lowest) - .wrapping_add(dict_end.wrapping_sub(dict_start)) as u32; - - let locate = |index: u32| { - if index < prefix_lowest_index { - Located { - buf: dms.buf, - pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_lowest, - end: iend, - } - } - }; let hash_long = ms.hash_table.as_mut_slice(); let hash_small = ms.chain_table.as_mut_slice(); - let mut anchor = istart; - let mut ip = istart + usize::from(dict_and_prefix_length == 0); + let mut anchor = seg.istart; + let mut ip = seg.istart + usize::from(seg.dict_and_prefix_length() == 0); - 'search: while ip < ilimit { - let h2 = hash_ptr(buf, ip, h_bits_l, 8)?; - let h = hash_ptr(buf, ip, h_bits_s, MLS)?; - let dict_hash_and_tag_l = hash_ptr(buf, ip, dict_h_bits_l, 8)?; - let dict_hash_and_tag_s = hash_ptr(buf, ip, dict_h_bits_s, MLS)?; + 'search: while ip < seg.ilimit { + let h2 = hash_ptr(seg.buf, ip, h_bits_l, 8)?; + let h = hash_ptr(seg.buf, ip, h_bits_s, MLS)?; + let dict_hash_and_tag_l = hash_ptr(seg.buf, ip, dict_h_bits_l, 8)?; + let dict_hash_and_tag_s = hash_ptr(seg.buf, ip, dict_h_bits_s, MLS)?; let dict_match_index_and_tag_l = dict_hash_long.at(dict_hash_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS)?; let dict_match_index_and_tag_s = @@ -361,74 +331,81 @@ fn compress_block_double_fast_dict_match_state_generic( let match_index_l = hash_long.at(h2)?; let mut match_index_s = hash_small.at(h)?; let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); + let rep_match = seg.locate(rep_index); hash_long.put(h2, curr)?; hash_small.put(h, curr)?; let m_length = 'match_stored: { - if index_overlap_check(prefix_lowest_index, rep_index) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? + if index_overlap_check(seg.prefix_start_index, rep_index) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(seg.buf, ip + 1)? { let m_length = count_2segments( - buf, + seg.buf, ip + 1 + 4, rep_match.buf, rep_match.pos + 4, - iend, + seg.iend, rep_match.end, - prefix_lowest, + seg.prefix_start, )? + 4; ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + seq_store.store_seq(ip - anchor, seg.buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; break 'match_stored m_length; } // The offset and the length of a match, which `ip` is moved to the start of. let (offset, m_length) = 'match_found: { let match_long = src.pos_of(match_index_l); - if match_index_l >= prefix_lowest_index - && read_le64(buf, match_long)? == read_le64(buf, ip)? + if match_index_l >= seg.prefix_start_index + && read_le64(seg.buf, match_long)? == read_le64(seg.buf, ip)? { - let m_length = count(buf, ip + 8, buf, match_long + 8, iend)? + 8; + let m_length = count(seg.buf, ip + 8, seg.buf, match_long + 8, seg.iend)? + 8; let offset = ip.wrapping_sub(match_long) as u32; - let back = count_backwards(buf, ip, anchor, buf, match_long, prefix_lowest); + let back = + count_backwards(seg.buf, ip, anchor, seg.buf, match_long, seg.prefix_start); ip -= back; break 'match_found (offset, m_length + back); } else if dict_tags_match_l { let dict_match_index_l = dict_match_index_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match_l = dms.pos_of(dict_match_index_l); - if dict_match_index_l > dict_start_index - && read_le64(dms.buf, dict_match_l)? == read_le64(buf, ip)? + let dict_match_l = seg.dict_pos_of(dict_match_index_l); + if dict_match_index_l > seg.dict_start_index + && read_le64(seg.dict_buf, dict_match_l)? == read_le64(seg.buf, ip)? { let m_length = count_2segments( - buf, + seg.buf, ip + 8, - dms.buf, + seg.dict_buf, dict_match_l + 8, - iend, - dict_end, - prefix_lowest, + seg.iend, + seg.dict_end, + seg.prefix_start, )? + 8; let offset = curr .wrapping_sub(dict_match_index_l) - .wrapping_sub(dict_index_delta); - let back = - count_backwards(buf, ip, anchor, dms.buf, dict_match_l, dict_start); + .wrapping_sub(seg.dict_index_delta); + let back = count_backwards( + seg.buf, + ip, + anchor, + seg.dict_buf, + dict_match_l, + seg.dict_start, + ); ip -= back; break 'match_found (offset, m_length + back); } } - let short_found = if match_index_s > prefix_lowest_index { - read_le32(buf, src.pos_of(match_index_s))? == read_le32(buf, ip)? + let short_found = if match_index_s > seg.prefix_start_index { + read_le32(seg.buf, src.pos_of(match_index_s))? == read_le32(seg.buf, ip)? } else if dict_tags_match_s { let dict_match_index_s = dict_match_index_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS; - match_index_s = dict_match_index_s.wrapping_add(dict_index_delta); - dict_match_index_s > dict_start_index - && read_le32(dms.buf, dms.pos_of(dict_match_index_s))? - == read_le32(buf, ip)? + match_index_s = dict_match_index_s.wrapping_add(seg.dict_index_delta); + dict_match_index_s > seg.dict_start_index + && read_le32(seg.dict_buf, seg.dict_pos_of(dict_match_index_s))? + == read_le32(seg.buf, ip)? } else { false }; @@ -438,8 +415,8 @@ fn compress_block_double_fast_dict_match_state_generic( } // A short match: a long one at the next position is preferred. - let hl3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; - let dict_hash_and_tag_l3 = hash_ptr(buf, ip + 1, dict_h_bits_l, 8)?; + let hl3 = hash_ptr(seg.buf, ip + 1, h_bits_l, 8)?; + let dict_hash_and_tag_l3 = hash_ptr(seg.buf, ip + 1, dict_h_bits_l, 8)?; let match_index_l3 = hash_long.at(hl3)?; let dict_match_index_and_tag_l3 = dict_hash_long.at(dict_hash_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS)?; @@ -448,56 +425,63 @@ fn compress_block_double_fast_dict_match_state_generic( let match_l3 = src.pos_of(match_index_l3); hash_long.put(hl3, curr + 1)?; - if match_index_l3 >= prefix_lowest_index - && read_le64(buf, match_l3)? == read_le64(buf, ip + 1)? + if match_index_l3 >= seg.prefix_start_index + && read_le64(seg.buf, match_l3)? == read_le64(seg.buf, ip + 1)? { - let m_length = count(buf, ip + 9, buf, match_l3 + 8, iend)? + 8; + let m_length = count(seg.buf, ip + 9, seg.buf, match_l3 + 8, seg.iend)? + 8; ip += 1; let offset = ip.wrapping_sub(match_l3) as u32; - let back = count_backwards(buf, ip, anchor, buf, match_l3, prefix_lowest); + let back = + count_backwards(seg.buf, ip, anchor, seg.buf, match_l3, seg.prefix_start); ip -= back; break 'match_found (offset, m_length + back); } else if dict_tags_match_l3 { let dict_match_index_l3 = dict_match_index_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match_l3 = dms.pos_of(dict_match_index_l3); - if dict_match_index_l3 > dict_start_index - && read_le64(dms.buf, dict_match_l3)? == read_le64(buf, ip + 1)? + let dict_match_l3 = seg.dict_pos_of(dict_match_index_l3); + if dict_match_index_l3 > seg.dict_start_index + && read_le64(seg.dict_buf, dict_match_l3)? == read_le64(seg.buf, ip + 1)? { let m_length = count_2segments( - buf, + seg.buf, ip + 1 + 8, - dms.buf, + seg.dict_buf, dict_match_l3 + 8, - iend, - dict_end, - prefix_lowest, + seg.iend, + seg.dict_end, + seg.prefix_start, )? + 8; ip += 1; let offset = curr .wrapping_add(1) .wrapping_sub(dict_match_index_l3) - .wrapping_sub(dict_index_delta); - let back = - count_backwards(buf, ip, anchor, dms.buf, dict_match_l3, dict_start); + .wrapping_sub(seg.dict_index_delta); + let back = count_backwards( + seg.buf, + ip, + anchor, + seg.dict_buf, + dict_match_l3, + seg.dict_start, + ); ip -= back; break 'match_found (offset, m_length + back); } } - let match_s = locate(match_index_s); + let match_s = seg.locate(match_index_s); let m_length = count_2segments( - buf, + seg.buf, ip + 4, match_s.buf, match_s.pos + 4, - iend, + seg.iend, match_s.end, - prefix_lowest, + seg.prefix_start, )? + 4; let offset = curr.wrapping_sub(match_index_s); let back = - count_backwards(buf, ip, anchor, match_s.buf, match_s.pos, match_s.start); + count_backwards(seg.buf, ip, anchor, match_s.buf, match_s.pos, match_s.start); ip -= back; (offset, m_length + back) }; @@ -506,7 +490,7 @@ fn compress_block_double_fast_dict_match_state_generic( offset_1 = offset; seq_store.store_seq( ip - anchor, - buf, + seg.buf, anchor, offset_to_offbase(offset), m_length, @@ -517,36 +501,45 @@ fn compress_block_double_fast_dict_match_state_generic( ip += m_length; anchor = ip; - if ip <= ilimit { + if ip <= seg.ilimit { let index_to_insert = curr + 2; let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + hash_long.put(hash_ptr(seg.buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put( + hash_ptr(seg.buf, ip - 2, h_bits_l, 8)?, + src.index_of(ip - 2), + )?; + hash_small.put( + hash_ptr(seg.buf, to_insert, h_bits_s, MLS)?, + index_to_insert, + )?; + hash_small.put( + hash_ptr(seg.buf, ip - 1, h_bits_s, MLS)?, + src.index_of(ip - 1), + )?; - while ip <= ilimit { + while ip <= seg.ilimit { let current2 = src.index_of(ip); let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !(index_overlap_check(prefix_lowest_index, rep_index2) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) + let rep_match2 = seg.locate(rep_index2); + if !(index_overlap_check(seg.prefix_start_index, rep_index2) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(seg.buf, ip)?) { break; } let rep_length2 = count_2segments( - buf, + seg.buf, ip + 4, rep_match2.buf, rep_match2.pos + 4, - iend, + seg.iend, rep_match2.end, - prefix_lowest, + seg.prefix_start, )? + 4; core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; + seq_store.store_seq(0, seg.buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_small.put(hash_ptr(seg.buf, ip, h_bits_s, MLS)?, current2)?; + hash_long.put(hash_ptr(seg.buf, ip, h_bits_l, 8)?, current2)?; ip += rep_length2; anchor = ip; } @@ -556,7 +549,7 @@ fn compress_block_double_fast_dict_match_state_generic( let [rep0, rep1, _] = rep; *rep0 = offset_1; *rep1 = offset_2; - Ok(iend - anchor) + Ok(seg.iend - anchor) } /// `ZSTD_compressBlock_doubleFast` @@ -581,11 +574,16 @@ pub(crate) fn compress_block_double_fast_dict_match_state( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { + let prefix_lowest_index = + ms.get_lowest_prefix_index(src.index_of(src.end), ms.c_params.window_log); + let Some(seg) = &TwoSegments::with_dict_match_state(src, prefix_lowest_index)? else { + return Ok(src.end - src.start); + }; match ms.c_params.min_match { - 5 => compress_block_double_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), + 5 => compress_block_double_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src, seg), + 6 => compress_block_double_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src, seg), + 7 => compress_block_double_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src, seg), + _ => compress_block_double_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src, seg), } } @@ -595,101 +593,64 @@ fn compress_block_double_fast_ext_dict_generic( seq_store: &mut SeqStore, rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, + seg: &TwoSegments<'_>, ) -> Result { let h_bits_l = ms.c_params.hash_log; let h_bits_s = ms.c_params.chain_log; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); - let dict_start_index = low_limit; - let dict_limit = ms.window.dict_limit; - let prefix_start_index = dict_limit.max(low_limit); - let prefix_start = src.pos_of(prefix_start_index); - let dict_start = src.dict_pos_of(dict_start_index); - let dict_end = src.dict_pos_of(prefix_start_index); let [mut offset_1, mut offset_2, _] = *rep; - - // Nothing of the older segment is within reach. - if prefix_start_index == dict_start_index { - return compress_block_double_fast(ms, seq_store, rep, src); - } - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: src.dict_buf, - pos: src.dict_pos_of(index), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; let hash_long = ms.hash_table.as_mut_slice(); let hash_small = ms.chain_table.as_mut_slice(); - let mut anchor = istart; - let mut ip = istart; + let mut anchor = seg.istart; + let mut ip = seg.istart; - while ip < ilimit { - let h_small = hash_ptr(buf, ip, h_bits_s, MLS)?; + while ip < seg.ilimit { + let h_small = hash_ptr(seg.buf, ip, h_bits_s, MLS)?; let match_index = hash_small.at(h_small)?; - let match_s = locate(match_index); + let match_s = seg.locate(match_index); - let h_long = hash_ptr(buf, ip, h_bits_l, 8)?; + let h_long = hash_ptr(seg.buf, ip, h_bits_l, 8)?; let match_long_index = hash_long.at(h_long)?; - let match_long = locate(match_long_index); + let match_long = seg.locate(match_long_index); let curr = src.index_of(ip); let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); + let rep_match = seg.locate(rep_index); hash_long.put(h_long, curr)?; hash_small.put(h_small, curr)?; let m_length; - if (index_overlap_check(prefix_start_index, rep_index) - & (offset_1 <= curr.wrapping_add(1).wrapping_sub(dict_start_index))) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip + 1)? + if (index_overlap_check(seg.prefix_start_index, rep_index) + & (offset_1 <= curr.wrapping_add(1).wrapping_sub(seg.dict_start_index))) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(seg.buf, ip + 1)? { m_length = count_2segments( - buf, + seg.buf, ip + 1 + 4, rep_match.buf, rep_match.pos + 4, - iend, + seg.iend, rep_match.end, - prefix_start, + seg.prefix_start, )? + 4; ip += 1; - seq_store.store_seq(ip - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + seq_store.store_seq(ip - anchor, seg.buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; } else { // The match, how much of it is known, and its offset. `ip` is moved to where it is for. - let (found, known, offset) = if match_long_index > dict_start_index - && read_le64(match_long.buf, match_long.pos)? == read_le64(buf, ip)? + let (found, known, offset) = if match_long_index > seg.dict_start_index + && read_le64(match_long.buf, match_long.pos)? == read_le64(seg.buf, ip)? { (match_long, 8, curr.wrapping_sub(match_long_index)) - } else if match_index > dict_start_index - && read_le32(match_s.buf, match_s.pos)? == read_le32(buf, ip)? + } else if match_index > seg.dict_start_index + && read_le32(match_s.buf, match_s.pos)? == read_le32(seg.buf, ip)? { // A short match: a long one at the next position is preferred. - let h3 = hash_ptr(buf, ip + 1, h_bits_l, 8)?; + let h3 = hash_ptr(seg.buf, ip + 1, h_bits_l, 8)?; let match_index3 = hash_long.at(h3)?; - let match3 = locate(match_index3); + let match3 = seg.locate(match_index3); hash_long.put(h3, curr + 1)?; - if match_index3 > dict_start_index - && read_le64(match3.buf, match3.pos)? == read_le64(buf, ip + 1)? + if match_index3 > seg.dict_start_index + && read_le64(match3.buf, match3.pos)? == read_le64(seg.buf, ip + 1)? { ip += 1; (match3, 8, curr.wrapping_add(1).wrapping_sub(match_index3)) @@ -701,22 +662,22 @@ fn compress_block_double_fast_ext_dict_generic( continue; }; let forwards = count_2segments( - buf, + seg.buf, ip + known, found.buf, found.pos + known, - iend, + seg.iend, found.end, - prefix_start, + seg.prefix_start, )? + known; - let back = count_backwards(buf, ip, anchor, found.buf, found.pos, found.start); + let back = count_backwards(seg.buf, ip, anchor, found.buf, found.pos, found.start); ip -= back; m_length = forwards + back; offset_2 = offset_1; offset_1 = offset; seq_store.store_seq( ip - anchor, - buf, + seg.buf, anchor, offset_to_offbase(offset), m_length, @@ -726,37 +687,46 @@ fn compress_block_double_fast_ext_dict_generic( ip += m_length; anchor = ip; - if ip <= ilimit { + if ip <= seg.ilimit { let index_to_insert = curr + 2; let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put(hash_ptr(buf, ip - 2, h_bits_l, 8)?, src.index_of(ip - 2))?; - hash_small.put(hash_ptr(buf, to_insert, h_bits_s, MLS)?, index_to_insert)?; - hash_small.put(hash_ptr(buf, ip - 1, h_bits_s, MLS)?, src.index_of(ip - 1))?; + hash_long.put(hash_ptr(seg.buf, to_insert, h_bits_l, 8)?, index_to_insert)?; + hash_long.put( + hash_ptr(seg.buf, ip - 2, h_bits_l, 8)?, + src.index_of(ip - 2), + )?; + hash_small.put( + hash_ptr(seg.buf, to_insert, h_bits_s, MLS)?, + index_to_insert, + )?; + hash_small.put( + hash_ptr(seg.buf, ip - 1, h_bits_s, MLS)?, + src.index_of(ip - 1), + )?; - while ip <= ilimit { + while ip <= seg.ilimit { let current2 = src.index_of(ip); let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !((index_overlap_check(prefix_start_index, rep_index2) - & (offset_2 <= current2.wrapping_sub(dict_start_index))) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip)?) + let rep_match2 = seg.locate(rep_index2); + if !((index_overlap_check(seg.prefix_start_index, rep_index2) + & (offset_2 <= current2.wrapping_sub(seg.dict_start_index))) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(seg.buf, ip)?) { break; } let rep_length2 = count_2segments( - buf, + seg.buf, ip + 4, rep_match2.buf, rep_match2.pos + 4, - iend, + seg.iend, rep_match2.end, - prefix_start, + seg.prefix_start, )? + 4; core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_small.put(hash_ptr(buf, ip, h_bits_s, MLS)?, current2)?; - hash_long.put(hash_ptr(buf, ip, h_bits_l, 8)?, current2)?; + seq_store.store_seq(0, seg.buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_small.put(hash_ptr(seg.buf, ip, h_bits_s, MLS)?, current2)?; + hash_long.put(hash_ptr(seg.buf, ip, h_bits_l, 8)?, current2)?; ip += rep_length2; anchor = ip; } @@ -766,7 +736,7 @@ fn compress_block_double_fast_ext_dict_generic( let [rep0, rep1, _] = rep; *rep0 = offset_1; *rep1 = offset_2; - Ok(iend - anchor) + Ok(seg.iend - anchor) } /// `ZSTD_compressBlock_doubleFast_extDict` @@ -776,10 +746,17 @@ pub(crate) fn compress_block_double_fast_ext_dict( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { + let Some(seg) = &TwoSegments::with_ext_dict(ms, src)? else { + return Ok(src.end - src.start); + }; + // Nothing of the older segment is within reach. + if seg.prefix_start_index == seg.dict_start_index { + return compress_block_double_fast(ms, seq_store, rep, src); + } match ms.c_params.min_match { - 5 => compress_block_double_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), + 5 => compress_block_double_fast_ext_dict_generic::<5>(ms, seq_store, rep, src, seg), + 6 => compress_block_double_fast_ext_dict_generic::<6>(ms, seq_store, rep, src, seg), + 7 => compress_block_double_fast_ext_dict_generic::<7>(ms, seq_store, rep, src, seg), + _ => compress_block_double_fast_ext_dict_generic::<4>(ms, seq_store, rep, src, seg), } } diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index f543cad9ae16..e5bed54e5169 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -1,10 +1,10 @@ //! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. use super::internal::{ - BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, - compare_packed_tags, count, count_2segments, hash_ptr, index_overlap_check, offset_to_offbase, - write_tagged_index, + BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Near, Prefix, + REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, count, count_2segments, hash_of, hash_ptr, index_overlap_check, + offset_to_offbase, table_get, table_put, write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -25,6 +25,116 @@ pub(super) struct Located<'a> { pub(super) end: usize, } +/// What `fast` and `dfast` work out before they look for matches in a block, when indices lead to two segments: the +/// current one, and below `prefix_start_index` either the older one of the window or the dictionary's. +pub(super) struct TwoSegments<'a> { + /// The buffer of the current segment, up to the end of the block. + pub(super) buf: &'a [u8], + base: usize, + pub(super) istart: usize, + pub(super) iend: usize, + /// The last position that [`HASH_READ_SIZE`] bytes can be read at. + pub(super) ilimit: usize, + pub(super) prefix_start_index: u32, + pub(super) prefix_start: usize, + pub(super) dict_buf: &'a [u8], + dict_base: usize, + /// What to subtract from an index below `prefix_start_index` to get an index of the other segment. + pub(super) dict_index_delta: u32, + pub(super) dict_start_index: u32, + pub(super) dict_start: usize, + pub(super) dict_end: usize, +} + +impl<'a> TwoSegments<'a> { + /// The other segment is the dictionary's, which has its own match state. `None` if the block is too short to look + /// for anything. + pub(super) fn with_dict_match_state( + src: &BlockSrc<'a>, + prefix_start_index: u32, + ) -> Result> { + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_start_index = dms.ms.window.dict_limit; + let Some(ilimit) = src.end.checked_sub(HASH_READ_SIZE) else { + return Ok(None); + }; + Ok(Some(Self { + buf: src.buf.sub(..src.end)?, + base: src.base, + istart: src.start, + iend: src.end, + ilimit, + prefix_start_index, + prefix_start: src.pos_of(prefix_start_index), + dict_buf: dms.buf, + dict_base: dms.base, + dict_index_delta: prefix_start_index + .wrapping_sub(dms.end.wrapping_sub(dms.base) as u32), + dict_start_index, + dict_start: dms.pos_of(dict_start_index), + dict_end: dms.end, + })) + } + + /// The other segment is the older one of the window, as far as it is within reach. `None` if the block is too short + /// to look for anything. + pub(super) fn with_ext_dict(ms: &MatchState, src: &BlockSrc<'a>) -> Result> { + let low_limit = ms.get_lowest_match_index(src.index_of(src.end), ms.c_params.window_log); + let prefix_start_index = ms.window.dict_limit.max(low_limit); + let Some(ilimit) = src.end.checked_sub(HASH_READ_SIZE) else { + return Ok(None); + }; + Ok(Some(Self { + buf: src.buf.sub(..src.end)?, + base: src.base, + istart: src.start, + iend: src.end, + ilimit, + prefix_start_index, + prefix_start: src.pos_of(prefix_start_index), + dict_buf: src.dict_buf, + dict_base: src.dict_base, + dict_index_delta: 0, + dict_start_index: low_limit, + dict_start: src.dict_pos_of(low_limit), + dict_end: src.dict_pos_of(prefix_start_index), + })) + } + + /// How many bytes matches can be in before the block. + pub(super) const fn dict_and_prefix_length(&self) -> u32 { + self.istart + .wrapping_sub(self.prefix_start) + .wrapping_add(self.dict_end) + .wrapping_sub(self.dict_start) as u32 + } + + /// The position in `dict_buf` of an index of the other segment. + #[inline(always)] + pub(super) const fn dict_pos_of(&self, index: u32) -> usize { + self.dict_base.wrapping_add(index as usize) + } + + #[inline(always)] + pub(super) const fn locate(&self, index: u32) -> Located<'a> { + if index < self.prefix_start_index { + Located { + buf: self.dict_buf, + pos: self.dict_pos_of(index.wrapping_sub(self.dict_index_delta)), + start: self.dict_start, + end: self.dict_end, + } + } else { + Located { + buf: self.buf, + pos: self.base.wrapping_add(index as usize), + start: self.prefix_start, + end: self.iend, + } + } + } +} + /// How many bytes before `ip`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. #[inline(always)] pub(super) fn count_backwards( @@ -45,6 +155,35 @@ pub(super) fn count_backwards( .count() } +/// The repeat offsets at the start of a block. Those that reach further than `max_rep` are 0, and saved. +pub(super) struct SavedRep { + pub(super) offset_1: u32, + pub(super) offset_2: u32, + pub(super) offset_saved1: u32, + pub(super) offset_saved2: u32, +} + +impl SavedRep { + pub(super) const fn new(rep: &[u32; ZSTD_REP_NUM], max_rep: u32) -> Self { + let [mut offset_1, mut offset_2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + if offset_2 > max_rep { + offset_saved2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved1 = offset_1; + offset_1 = 0; + } + Self { + offset_1, + offset_2, + offset_saved1, + offset_saved2, + } + } +} + /// The repeat offsets to leave at the end of a block, at the start of which those that reached too far were saved and /// set to 0. #[inline(always)] @@ -167,6 +306,73 @@ fn match4_found( Ok(match_idx >= idx_low_limit && read_le32(buf, current)? == read_le32(buf, match_pos)?) } +/// What `fast` and `dfast` work out before they look for matches in a block, when there is no dictionary and the window +/// is one segment. +pub(super) struct NoDictBlock<'a> { + /// The input from `base_index`, the lowest index that is read, up to the end of the block. Positions are in this. + pub(super) near: Near<'a>, + pub(super) base_index: u32, + /// The part of it where matches can be, which starts at `prefix_lowest_index`. A lower index, less that one, wraps + /// around to a position past the end. + pub(super) candidates: Near<'a>, + pub(super) prefix_lowest_index: u32, + pub(super) prefix_lowest: u32, + pub(super) istart: u32, + /// The first position to look at. + pub(super) first: u32, + /// The last position that [`HASH_READ_SIZE`] bytes can be read at. + pub(super) ilimit: usize, + pub(super) offset_1: u32, + pub(super) offset_2: u32, + /// The repeat offsets that reach too far, which are 0 above. + pub(super) offset_saved1: u32, + pub(super) offset_saved2: u32, +} + +impl<'a> NoDictBlock<'a> { + /// `None` if the block is too short to look for anything. + pub(super) fn new( + ms: &MatchState, + rep: &[u32; ZSTD_REP_NUM], + src: &BlockSrc<'a>, + ) -> Result> { + let window_log = ms.c_params.window_log; + let start_index = src.index_of(src.start); + let prefix_lowest_index = ms.get_lowest_prefix_index(src.index_of(src.end), window_log); + let first_index = start_index + u32::from(start_index == prefix_lowest_index); + let window_low = ms.get_lowest_prefix_index(first_index, window_log); + + // A repeat offset reaches as far as the window did at the start of the block. + let base_index = window_low.min(prefix_lowest_index); + let near = Near::new(src.buf.sub(src.pos_of(base_index)..src.end)?)?; + let prefix_lowest = prefix_lowest_index - base_index; + let Some(ilimit) = near.bytes().len().checked_sub(HASH_READ_SIZE) else { + return Ok(None); + }; + + let SavedRep { + offset_1, + offset_2, + offset_saved1, + offset_saved2, + } = SavedRep::new(rep, first_index.wrapping_sub(window_low)); + Ok(Some(Self { + near, + base_index, + candidates: Near::new(near.bytes().sub(prefix_lowest as usize..)?)?, + prefix_lowest_index, + prefix_lowest, + istart: start_index - base_index, + first: first_index - base_index, + ilimit, + offset_1, + offset_2, + offset_saved1, + offset_saved2, + })) + } +} + /// `ZSTD_compressBlock_fast_noDict_generic` /// /// Each turn of the inner loop looks at two positions, `ip0` and `ip1`, for a match, and at `ip2` for a repeat. The @@ -177,42 +383,30 @@ fn compress_block_fast_no_dict_generic( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { - const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); + const K_STEP_INCR: u32 = 1 << (K_SEARCH_STRENGTH - 1); + let Some(block) = NoDictBlock::new(ms, rep, src)? else { + return Ok(src.end - src.start); + }; + let NoDictBlock { + near, + base_index, + candidates, + prefix_lowest_index: prefix_start_index, + prefix_lowest: prefix_start, + ilimit, + offset_1: mut rep_offset1, + offset_2: mut rep_offset2, + .. + } = block; let hlog = ms.c_params.hash_log; - let window_log = ms.c_params.window_log; let target_length = ms.c_params.target_length; // The least step is 2, as two positions are looked at per turn. - let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let prefix_start_index = ms.get_lowest_prefix_index(end_index, window_log); - let prefix_start = src.pos_of(prefix_start_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip0 = istart + usize::from(istart == prefix_start); - let [mut rep_offset1, mut rep_offset2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let curr = src.index_of(ip0); - let window_low = ms.get_lowest_prefix_index(curr, window_log); - let max_rep = curr.wrapping_sub(window_low); - if rep_offset2 > max_rep { - offset_saved2 = rep_offset2; - rep_offset2 = 0; - } - if rep_offset1 > max_rep { - offset_saved1 = rep_offset1; - rep_offset1 = 0; - } - } - let hash_table = ms.hash_table.as_mut_slice(); + let step_size = target_length + u32::from(target_length == 0) + 1; + let mut anchor = block.istart; + let mut ip0 = block.first; + let mut seq_store = SeqSink::new(seq_store); + let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; 'start: loop { let mut step = step_size; @@ -221,71 +415,69 @@ fn compress_block_fast_no_dict_generic( let mut ip1 = ip0 + 1; let mut ip2 = ip0 + step; let mut ip3 = ip2 + 1; - if ip3 >= ilimit { + if ip3 as usize >= ilimit { break 'start; } - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; - let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; - let mut match_idx = hash_table.at(hash0)?; + // The 8 bytes at `ip0` and at `ip1`. + let mut bytes0 = near.read64(ip0).ok_or(BUG)?; + let mut bytes1 = near.read64(ip1).ok_or(BUG)?; + let mut hash0 = hash_of::(bytes0, hlog); + let mut hash1 = hash_of::(bytes1, hlog); + let mut match_idx = table_get(hash_table, hash0); let mut current0; // Ends with the start and the length so far of a repeat match, or with `None` for a match at `match_idx`. let rep_match = loop { - let rval = read_le32(buf, ip2.wrapping_sub(rep_offset1 as usize))?; + let bytes2 = near.read64(ip2).ok_or(BUG)?; + let rval = near.read32(ip2.wrapping_sub(rep_offset1)); - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; + current0 = base_index.wrapping_add(ip0); + table_put(hash_table, hash0, current0); - if (read_le32(buf, ip2)? == rval) & (rep_offset1 > 0) { + if (rval == Some(bytes2 as u32)) & (rep_offset1 > 0) { ip0 = ip2; - let match0 = ip0.wrapping_sub(rep_offset1 as usize); - let back = usize::from(buf.at(ip0 - 1)? == buf.at(match0.wrapping_sub(1))?); + let match0 = ip0 - rep_offset1; + let back = u32::from(near.read8(ip0 - 1) == near.read8(match0.wrapping_sub(1))); ip0 -= back; // The hash of ip1 is there, and ip1 is before the end of the match. - hash_table.put(hash1, src.index_of(ip1))?; + table_put(hash_table, hash1, base_index.wrapping_add(ip1)); break Some((match0.wrapping_sub(back), 4 + back)); } - if match4_found( - buf, - ip0, - src.pos_of(match_idx), - match_idx, - prefix_start_index, - )? { - hash_table.put(hash1, src.index_of(ip1))?; + if candidates.read32(match_idx.wrapping_sub(prefix_start_index)) == Some(bytes0 as u32) + { + table_put(hash_table, hash1, base_index.wrapping_add(ip1)); break None; } - match_idx = hash_table.at(hash1)?; + match_idx = table_get(hash_table, hash1); hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + bytes0 = bytes1; + bytes1 = bytes2; + hash1 = hash_of::(bytes1, hlog); ip0 = ip1; ip1 = ip2; ip2 = ip3; - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; + current0 = base_index.wrapping_add(ip0); + table_put(hash_table, hash0, current0); - if match4_found( - buf, - ip0, - src.pos_of(match_idx), - match_idx, - prefix_start_index, - )? { + if candidates.read32(match_idx.wrapping_sub(prefix_start_index)) == Some(bytes0 as u32) + { // ip1 is certain to be before the end of the match only if the step is small. if step <= 4 { - hash_table.put(hash1, src.index_of(ip1))?; + table_put(hash_table, hash1, base_index.wrapping_add(ip1)); } break None; } - match_idx = hash_table.at(hash1)?; + match_idx = table_get(hash_table, hash1); hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + bytes0 = bytes1; + bytes1 = near.read64(ip2).ok_or(BUG)?; + hash1 = hash_of::(bytes1, hlog); ip0 = ip1; ip1 = ip2; @@ -296,7 +488,7 @@ fn compress_block_fast_no_dict_generic( step += 1; next_step += K_STEP_INCR; } - if ip3 >= ilimit { + if ip3 as usize >= ilimit { break 'start; } }; @@ -304,51 +496,57 @@ fn compress_block_fast_no_dict_generic( let (offcode, match0, mut m_length) = match rep_match { Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), None => { - let match0 = src.pos_of(match_idx); + let match0 = match_idx - base_index; rep_offset2 = rep_offset1; - rep_offset1 = ip0.wrapping_sub(match0) as u32; - let back = count_backwards(buf, ip0, anchor, buf, match0, prefix_start); + rep_offset1 = ip0 - match0; + let back = near.count_backwards(ip0, anchor, match0, prefix_start); ip0 -= back; (offset_to_offbase(rep_offset1), match0 - back, 4 + back) } }; - m_length += count(buf, ip0 + m_length, buf, match0 + m_length, iend)?; - seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; + m_length += near.count(ip0 + m_length, match0 + m_length); + seq_store.store_seq(ip0 - anchor, near, anchor, offcode, m_length)?; ip0 += m_length; anchor = ip0; - if ip0 <= ilimit { - hash_table.put( - hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, - current0 + 2, - )?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; - - if rep_offset2 > 0 { - while ip0 <= ilimit - && read_le32(buf, ip0)? - == read_le32(buf, ip0.wrapping_sub(rep_offset2 as usize))? - { - let r_length = count( - buf, - ip0 + 4, - buf, - (ip0 + 4).wrapping_sub(rep_offset2 as usize), - iend, - )? + 4; - core::mem::swap(&mut rep_offset1, &mut rep_offset2); - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; - ip0 += r_length; - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, r_length)?; - anchor = ip0; + if ip0 as usize <= ilimit { + let to_insert = near.read64(current0 + 2 - base_index).ok_or(BUG)?; + table_put(hash_table, hash_of::(to_insert, hlog), current0 + 2); + let before2 = near.read64(ip0 - 2).ok_or(BUG)?; + table_put( + hash_table, + hash_of::(before2, hlog), + base_index.wrapping_add(ip0 - 2), + ); + + while rep_offset2 > 0 && ip0 as usize <= ilimit { + let bytes = near.read64(ip0).ok_or(BUG)?; + if near.read32(ip0.wrapping_sub(rep_offset2)) != Some(bytes as u32) { + break; } + let r_length = near.count(ip0 + 4, (ip0 + 4).wrapping_sub(rep_offset2)) + 4; + core::mem::swap(&mut rep_offset1, &mut rep_offset2); + table_put( + hash_table, + hash_of::(bytes, hlog), + base_index.wrapping_add(ip0), + ); + ip0 += r_length; + seq_store.store_seq(0, near, anchor, REPCODE1_TO_OFFBASE, r_length)?; + anchor = ip0; } } } - restore_rep(rep, rep_offset1, rep_offset2, offset_saved1, offset_saved2); - Ok(iend - anchor) + restore_rep( + rep, + rep_offset1, + rep_offset2, + block.offset_saved1, + block.offset_saved2, + ); + Ok((near.len() - anchor) as usize) } /// `ZSTD_compressBlock_fast` @@ -372,63 +570,27 @@ fn compress_block_fast_dict_match_state_generic( seq_store: &mut SeqStore, rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, + seg: &TwoSegments<'_>, ) -> Result { const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; let hlog = ms.c_params.hash_log; let target_length = ms.c_params.target_length; let step_size = (target_length + u32::from(target_length == 0)) as usize; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let prefix_start_index = ms.window.dict_limit; - let prefix_start = src.pos_of(prefix_start_index); - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; let [mut offset_1, mut offset_2, _] = *rep; - let dms = src.dms.as_ref().ok_or(BUG)?; let dict_hash_table = dms.ms.hash_table.as_slice(); - let dict_start_index = dms.ms.window.dict_limit; - let dict_start = dms.pos_of(dict_start_index); - let dict_end = dms.end; - // What to subtract from an index below prefix_start_index to get the index in the dictionary. - let dict_index_delta = prefix_start_index.wrapping_sub(dict_end.wrapping_sub(dms.base) as u32); - let dict_and_prefix_length = istart - .wrapping_sub(prefix_start) - .wrapping_add(dict_end) - .wrapping_sub(dict_start) as u32; let dict_h_bits = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: dms.buf, - pos: dms.pos_of(index.wrapping_sub(dict_index_delta)), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; let hash_table = ms.hash_table.as_mut_slice(); - let mut anchor = istart; - let mut ip0 = istart + usize::from(dict_and_prefix_length == 0); - let mut ip1 = istart + step_size; + let mut anchor = seg.istart; + let mut ip0 = seg.istart + usize::from(seg.dict_and_prefix_length() == 0); + let mut ip1 = seg.istart + step_size; - 'search: while ip1 <= ilimit { - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; + 'search: while ip1 <= seg.ilimit { + let mut hash0 = hash_ptr(seg.buf, ip0, hlog, MLS)?; - let dict_hash_and_tag0 = hash_ptr(buf, ip0, dict_h_bits, MLS)?; + let dict_hash_and_tag0 = hash_ptr(seg.buf, ip0, dict_h_bits, MLS)?; let mut dict_match_index_and_tag = dict_hash_table.at(dict_hash_and_tag0 >> ZSTD_SHORT_CACHE_TAG_BITS)?; let mut dict_tags_match = @@ -441,56 +603,69 @@ fn compress_block_fast_dict_match_state_generic( let m_length = loop { let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = locate(rep_index); - let hash1 = hash_ptr(buf, ip1, hlog, MLS)?; - let dict_hash_and_tag1 = hash_ptr(buf, ip1, dict_h_bits, MLS)?; + let rep_match = seg.locate(rep_index); + let hash1 = hash_ptr(seg.buf, ip1, hlog, MLS)?; + let dict_hash_and_tag1 = hash_ptr(seg.buf, ip1, dict_h_bits, MLS)?; hash_table.put(hash0, curr)?; - if index_overlap_check(prefix_start_index, rep_index) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip0 + 1)? + if index_overlap_check(seg.prefix_start_index, rep_index) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(seg.buf, ip0 + 1)? { let m_length = count_2segments( - buf, + seg.buf, ip0 + 1 + 4, rep_match.buf, rep_match.pos + 4, - iend, + seg.iend, rep_match.end, - prefix_start, + seg.prefix_start, )? + 4; ip0 += 1; - seq_store.store_seq(ip0 - anchor, buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; + seq_store.store_seq( + ip0 - anchor, + seg.buf, + anchor, + REPCODE1_TO_OFFBASE, + m_length, + )?; break m_length; } if dict_tags_match { let dict_match_index = dict_match_index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match = dms.pos_of(dict_match_index); + let dict_match = seg.dict_pos_of(dict_match_index); // A match in the dictionary is only taken if there is no candidate in the prefix. - if dict_match_index > dict_start_index - && read_le32(dms.buf, dict_match)? == read_le32(buf, ip0)? - && match_index <= prefix_start_index + if dict_match_index > seg.dict_start_index + && read_le32(seg.dict_buf, dict_match)? == read_le32(seg.buf, ip0)? + && match_index <= seg.prefix_start_index { let offset = curr .wrapping_sub(dict_match_index) - .wrapping_sub(dict_index_delta); + .wrapping_sub(seg.dict_index_delta); let mut m_length = count_2segments( - buf, + seg.buf, ip0 + 4, - dms.buf, + seg.dict_buf, dict_match + 4, - iend, - dict_end, - prefix_start, + seg.iend, + seg.dict_end, + seg.prefix_start, )? + 4; - let back = count_backwards(buf, ip0, anchor, dms.buf, dict_match, dict_start); + let back = count_backwards( + seg.buf, + ip0, + anchor, + seg.dict_buf, + dict_match, + seg.dict_start, + ); ip0 -= back; m_length += back; offset_2 = offset_1; offset_1 = offset; seq_store.store_seq( ip0 - anchor, - buf, + seg.buf, anchor, offset_to_offbase(offset), m_length, @@ -500,17 +675,18 @@ fn compress_block_fast_dict_match_state_generic( } let match_pos = src.pos_of(match_index); - if match4_found(buf, ip0, match_pos, match_index, prefix_start_index)? { + if match4_found(seg.buf, ip0, match_pos, match_index, seg.prefix_start_index)? { let offset = ip0.wrapping_sub(match_pos) as u32; - let mut m_length = count(buf, ip0 + 4, buf, match_pos + 4, iend)? + 4; - let back = count_backwards(buf, ip0, anchor, buf, match_pos, prefix_start); + let mut m_length = count(seg.buf, ip0 + 4, seg.buf, match_pos + 4, seg.iend)? + 4; + let back = + count_backwards(seg.buf, ip0, anchor, seg.buf, match_pos, seg.prefix_start); ip0 -= back; m_length += back; offset_2 = offset_1; offset_1 = offset; seq_store.store_seq( ip0 - anchor, - buf, + seg.buf, anchor, offset_to_offbase(offset), m_length, @@ -530,7 +706,7 @@ fn compress_block_fast_dict_match_state_generic( } ip0 = ip1; ip1 += step; - if ip1 > ilimit { + if ip1 > seg.ilimit { break 'search; } @@ -541,31 +717,37 @@ fn compress_block_fast_dict_match_state_generic( ip0 += m_length; anchor = ip0; - if ip0 <= ilimit { - hash_table.put(hash_ptr(buf, src.pos_of(curr + 2), hlog, MLS)?, curr + 2)?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + if ip0 <= seg.ilimit { + hash_table.put( + hash_ptr(seg.buf, src.pos_of(curr + 2), hlog, MLS)?, + curr + 2, + )?; + hash_table.put( + hash_ptr(seg.buf, ip0 - 2, hlog, MLS)?, + src.index_of(ip0 - 2), + )?; - while ip0 <= ilimit { + while ip0 <= seg.ilimit { let current2 = src.index_of(ip0); let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !(index_overlap_check(prefix_start_index, rep_index2) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) + let rep_match2 = seg.locate(rep_index2); + if !(index_overlap_check(seg.prefix_start_index, rep_index2) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(seg.buf, ip0)?) { break; } let rep_length2 = count_2segments( - buf, + seg.buf, ip0 + 4, rep_match2.buf, rep_match2.pos + 4, - iend, + seg.iend, rep_match2.end, - prefix_start, + seg.prefix_start, )? + 4; core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, current2)?; + seq_store.store_seq(0, seg.buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_table.put(hash_ptr(seg.buf, ip0, hlog, MLS)?, current2)?; ip0 += rep_length2; anchor = ip0; } @@ -577,7 +759,7 @@ fn compress_block_fast_dict_match_state_generic( let [rep0, rep1, _] = rep; *rep0 = offset_1; *rep1 = offset_2; - Ok(iend - anchor) + Ok(seg.iend - anchor) } /// `ZSTD_compressBlock_fast_dictMatchState` @@ -587,11 +769,14 @@ pub(crate) fn compress_block_fast_dict_match_state( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { + let Some(seg) = &TwoSegments::with_dict_match_state(src, ms.window.dict_limit)? else { + return Ok(src.end - src.start); + }; match ms.c_params.min_match { - 5 => compress_block_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src), + 5 => compress_block_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src, seg), + 6 => compress_block_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src, seg), + 7 => compress_block_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src, seg), + _ => compress_block_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src, seg), } } @@ -601,72 +786,29 @@ fn compress_block_fast_ext_dict_generic( seq_store: &mut SeqStore, rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, + seg: &TwoSegments<'_>, ) -> Result { const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); let hlog = ms.c_params.hash_log; let target_length = ms.c_params.target_length; let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; - let buf = src.buf.sub(..src.end)?; - let istart = src.start; - let iend = src.end; - let end_index = src.index_of(iend); - let low_limit = ms.get_lowest_match_index(end_index, ms.c_params.window_log); - let dict_start_index = low_limit; - let dict_start = src.dict_pos_of(dict_start_index); - let dict_limit = ms.window.dict_limit; - let prefix_start_index = dict_limit.max(low_limit); - let prefix_start = src.pos_of(prefix_start_index); - let dict_end = src.dict_pos_of(prefix_start_index); - - // Nothing of the older segment is within reach. - if prefix_start_index == dict_start_index { - return compress_block_fast(ms, seq_store, rep, src); - } - // Nothing is looked for in a block this short. - let Some(ilimit) = iend.checked_sub(HASH_READ_SIZE) else { - return Ok(iend - istart); - }; - - let mut anchor = istart; - let mut ip0 = istart; - let [mut offset_1, mut offset_2, _] = *rep; - let (mut offset_saved1, mut offset_saved2) = (0, 0); - { - let curr = src.index_of(ip0); - let max_rep = curr.wrapping_sub(dict_start_index); - if offset_2 >= max_rep { - offset_saved2 = offset_2; - offset_2 = 0; - } - if offset_1 >= max_rep { - offset_saved1 = offset_1; - offset_1 = 0; - } - } + let mut anchor = seg.istart; + let mut ip0 = seg.istart; + let max_rep = src.index_of(ip0).wrapping_sub(seg.dict_start_index); + // As far as `max_rep` is too far too. + let SavedRep { + mut offset_1, + mut offset_2, + offset_saved1, + offset_saved2, + } = SavedRep::new(rep, max_rep.saturating_sub(1)); - let locate = |index: u32| { - if index < prefix_start_index { - Located { - buf: src.dict_buf, - pos: src.dict_pos_of(index), - start: dict_start, - end: dict_end, - } - } else { - Located { - buf, - pos: src.pos_of(index), - start: prefix_start, - end: iend, - } - } - }; // Whether there is a match for the position `ip` at `idx`, which is from the table. let match_found = |ip: usize, idx: u32| -> Result { - let candidate = locate(idx); - Ok(idx >= dict_start_index - && read_le32(candidate.buf, candidate.pos)? == read_le32(buf, ip)?) + let candidate = seg.locate(idx); + Ok(idx >= seg.dict_start_index + && read_le32(candidate.buf, candidate.pos)? == read_le32(seg.buf, ip)?) }; let hash_table = ms.hash_table.as_mut_slice(); @@ -677,12 +819,12 @@ fn compress_block_fast_ext_dict_generic( let mut ip1 = ip0 + 1; let mut ip2 = ip0 + step; let mut ip3 = ip2 + 1; - if ip3 >= ilimit { + if ip3 >= seg.ilimit { break 'start; } - let mut hash0 = hash_ptr(buf, ip0, hlog, MLS)?; - let mut hash1 = hash_ptr(buf, ip1, hlog, MLS)?; + let mut hash0 = hash_ptr(seg.buf, ip0, hlog, MLS)?; + let mut hash1 = hash_ptr(seg.buf, ip1, hlog, MLS)?; let mut idx = hash_table.at(hash0)?; let mut current0; @@ -690,10 +832,11 @@ fn compress_block_fast_ext_dict_generic( let rep_match = loop { let current2 = src.index_of(ip2); let rep_index = current2.wrapping_sub(offset_1); - let mut rep_match = locate(rep_index); + let mut rep_match = seg.locate(rep_index); // The 4 bytes have to be in one segment, with a byte before them. - let rep_found = ((prefix_start_index.wrapping_sub(rep_index) >= 4) & (offset_1 > 0)) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(buf, ip2)?; + let rep_found = ((seg.prefix_start_index.wrapping_sub(rep_index) >= 4) + & (offset_1 > 0)) + && read_le32(rep_match.buf, rep_match.pos)? == read_le32(seg.buf, ip2)?; current0 = src.index_of(ip0); hash_table.put(hash0, current0)?; @@ -701,7 +844,7 @@ fn compress_block_fast_ext_dict_generic( if rep_found { ip0 = ip2; let back = usize::from( - buf.at(ip0 - 1)? == rep_match.buf.at(rep_match.pos.wrapping_sub(1))?, + seg.buf.at(ip0 - 1)? == rep_match.buf.at(rep_match.pos.wrapping_sub(1))?, ); ip0 -= back; rep_match.pos = rep_match.pos.wrapping_sub(back); @@ -714,7 +857,7 @@ fn compress_block_fast_ext_dict_generic( idx = hash_table.at(hash1)?; hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + hash1 = hash_ptr(seg.buf, ip2, hlog, MLS)?; ip0 = ip1; ip1 = ip2; @@ -729,7 +872,7 @@ fn compress_block_fast_ext_dict_generic( idx = hash_table.at(hash1)?; hash0 = hash1; - hash1 = hash_ptr(buf, ip2, hlog, MLS)?; + hash1 = hash_ptr(seg.buf, ip2, hlog, MLS)?; ip0 = ip1; ip1 = ip2; @@ -740,7 +883,7 @@ fn compress_block_fast_ext_dict_generic( step += 1; next_step += K_STEP_INCR; } - if ip3 >= ilimit { + if ip3 >= seg.ilimit { break 'start; } }; @@ -749,10 +892,11 @@ fn compress_block_fast_ext_dict_generic( Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), None => { let offset = current0.wrapping_sub(idx); - let mut match0 = locate(idx); + let mut match0 = seg.locate(idx); offset_2 = offset_1; offset_1 = offset; - let back = count_backwards(buf, ip0, anchor, match0.buf, match0.pos, match0.start); + let back = + count_backwards(seg.buf, ip0, anchor, match0.buf, match0.pos, match0.start); ip0 -= back; match0.pos -= back; (offset_to_offbase(offset), match0, 4 + back) @@ -760,15 +904,15 @@ fn compress_block_fast_ext_dict_generic( }; m_length += count_2segments( - buf, + seg.buf, ip0 + m_length, match0.buf, match0.pos + m_length, - iend, + seg.iend, match0.end, - prefix_start, + seg.prefix_start, )?; - seq_store.store_seq(ip0 - anchor, buf, anchor, offcode, m_length)?; + seq_store.store_seq(ip0 - anchor, seg.buf, anchor, offcode, m_length)?; ip0 += m_length; anchor = ip0; @@ -776,33 +920,36 @@ fn compress_block_fast_ext_dict_generic( hash_table.put(hash1, src.index_of(ip1))?; } - if ip0 <= ilimit { + if ip0 <= seg.ilimit { hash_table.put( - hash_ptr(buf, src.pos_of(current0 + 2), hlog, MLS)?, + hash_ptr(seg.buf, src.pos_of(current0 + 2), hlog, MLS)?, current0 + 2, )?; - hash_table.put(hash_ptr(buf, ip0 - 2, hlog, MLS)?, src.index_of(ip0 - 2))?; + hash_table.put( + hash_ptr(seg.buf, ip0 - 2, hlog, MLS)?, + src.index_of(ip0 - 2), + )?; - while ip0 <= ilimit { + while ip0 <= seg.ilimit { let rep_index2 = src.index_of(ip0).wrapping_sub(offset_2); - let rep_match2 = locate(rep_index2); - if !((index_overlap_check(prefix_start_index, rep_index2) & (offset_2 > 0)) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(buf, ip0)?) + let rep_match2 = seg.locate(rep_index2); + if !((index_overlap_check(seg.prefix_start_index, rep_index2) & (offset_2 > 0)) + && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(seg.buf, ip0)?) { break; } let rep_length2 = count_2segments( - buf, + seg.buf, ip0 + 4, rep_match2.buf, rep_match2.pos + 4, - iend, + seg.iend, rep_match2.end, - prefix_start, + seg.prefix_start, )? + 4; core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_table.put(hash_ptr(buf, ip0, hlog, MLS)?, src.index_of(ip0))?; + seq_store.store_seq(0, seg.buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; + hash_table.put(hash_ptr(seg.buf, ip0, hlog, MLS)?, src.index_of(ip0))?; ip0 += rep_length2; anchor = ip0; } @@ -810,7 +957,7 @@ fn compress_block_fast_ext_dict_generic( } restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); - Ok(iend - anchor) + Ok(seg.iend - anchor) } /// `ZSTD_compressBlock_fast_extDict` @@ -820,10 +967,17 @@ pub(crate) fn compress_block_fast_ext_dict( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { + let Some(seg) = &TwoSegments::with_ext_dict(ms, src)? else { + return Ok(src.end - src.start); + }; + // Nothing of the older segment is within reach. + if seg.prefix_start_index == seg.dict_start_index { + return compress_block_fast(ms, seq_store, rep, src); + } match ms.c_params.min_match { - 5 => compress_block_fast_ext_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_ext_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_ext_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_ext_dict_generic::<4>(ms, seq_store, rep, src), + 5 => compress_block_fast_ext_dict_generic::<5>(ms, seq_store, rep, src, seg), + 6 => compress_block_fast_ext_dict_generic::<6>(ms, seq_store, rep, src, seg), + 7 => compress_block_fast_ext_dict_generic::<7>(ms, seq_store, rep, src, seg), + _ => compress_block_fast_ext_dict_generic::<4>(ms, seq_store, rep, src, seg), } } diff --git a/src/zstd/compress/fse.rs b/src/zstd/compress/fse.rs index 7d57ceda36e4..999f24140c31 100644 --- a/src/zstd/compress/fse.rs +++ b/src/zstd/compress/fse.rs @@ -37,9 +37,12 @@ pub(crate) struct FseCTable { pub(crate) table_log: u32, pub(crate) max_symbol_value: u32, pub(crate) state_table: [u16; 1 << ML_FSE_LOG], - pub(crate) symbol_tt: [FseSymbolTransform; MAX_SEQ as usize + 1], + /// More than `MAX_SEQ + 1`, so that a masked symbol is known to be in bounds. + pub(crate) symbol_tt: [FseSymbolTransform; SYMBOL_TT_SIZE], } +const SYMBOL_TT_SIZE: usize = (MAX_SEQ as usize + 1).next_power_of_two(); + impl FseCTable { pub(crate) const fn new() -> Self { Self { @@ -49,7 +52,7 @@ impl FseCTable { symbol_tt: [FseSymbolTransform { delta_find_state: 0, delta_nb_bits: 0, - }; MAX_SEQ as usize + 1], + }; SYMBOL_TT_SIZE], } } @@ -237,7 +240,9 @@ impl<'a> FseCState<'a> { #[inline(always)] pub(crate) fn encode_symbol(&mut self, bit_c: &mut BitCStream<'_>, symbol: u32) { let symbol_tt = &self.table.symbol_tt; - let transform = symbol_tt.at(symbol as usize).unwrap_or_default(); + let transform = symbol_tt + .at(symbol as usize & (SYMBOL_TT_SIZE - 1)) + .unwrap_or_default(); let nb_bits_out = ((self.value + transform.delta_nb_bits as usize) >> 16) as u32; bit_c.add_bits(self.value as u64, nb_bits_out); self.step(nb_bits_out, transform.delta_find_state); diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 3f2aab1365fe..a732e979a2de 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -7,7 +7,7 @@ use crate::common::{ ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, highbit32, read_le32, read_le64, write_le24}; +use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64, write_le24}; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; pub(crate) const HASH_READ_SIZE: usize = 8; @@ -415,6 +415,79 @@ impl SeqStore { } } +/// A [`SeqStore`] while a hot loop adds to it: what changes with every sequence is here, in a local variable, which can +/// stay in registers. It goes back to the store on drop. +pub(crate) struct SeqSink<'a> { + store: &'a mut SeqStore, + sequences: Vec, + lit_buffer: Vec, + lit_len: u32, +} + +impl<'a> SeqSink<'a> { + #[inline(always)] + pub(crate) fn new(store: &'a mut SeqStore) -> Self { + Self { + sequences: core::mem::take(&mut store.sequences), + lit_buffer: core::mem::take(&mut store.lit_buffer), + lit_len: store.lit_len as u32, + store, + } + } + + /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `src` from `literals` on. + #[inline(always)] + pub(crate) fn store_seq( + &mut self, + lit_length: u32, + src: Near<'_>, + literals: u32, + off_base: u32, + match_length: u32, + ) -> Result<()> { + let lit_len = self.lit_len as usize; + let room = self.lit_buffer.get_mut(lit_len..lit_len + 16); + match ( + src.chunk16(literals), + room.and_then(<[u8]>::first_chunk_mut::<16>), + ) { + (Some(from), Some(to)) if lit_length <= 16 => *to = *from, + _ => self.lit_buffer.write_at( + lit_len, + src.bytes().span(literals as usize, lit_length as usize)?, + )?, + } + self.lit_len += lit_length; + + let ml_base = match_length.wrapping_sub(MINMATCH); + if lit_length > 0xFFFF || ml_base > 0xFFFF { + self.store.long_length_type = if ml_base > 0xFFFF { + LongLengthType::MatchLength + } else { + LongLengthType::LiteralLength + }; + self.store.long_length_pos = self.sequences.len() as u32; + } + self.sequences + .push_within_capacity(SeqDef { + off_base, + lit_length: lit_length as u16, + ml_base: ml_base as u16, + }) + .map(drop) + .map_err(|_| BUG) + } +} + +impl Drop for SeqSink<'_> { + #[inline(always)] + fn drop(&mut self) { + self.store.sequences = core::mem::take(&mut self.sequences); + self.store.lit_buffer = core::mem::take(&mut self.lit_buffer); + self.store.lit_len = self.lit_len as usize; + } +} + pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; /// `OFFSET_TO_OFFBASE`: `offset` is not 0. @@ -571,6 +644,113 @@ pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { .count() } +/// Input whose positions fit in 32 bits, which makes a bounds check a single comparison. The hot loops of the match +/// finders read through this. +#[derive(Clone, Copy)] +pub(crate) struct Near<'a>(&'a [u8]); + +impl<'a> Near<'a> { + #[inline] + pub(crate) fn new(bytes: &'a [u8]) -> Result { + if u32::try_from(bytes.len()).is_err() { + return Err(BUG); + } + Ok(Self(bytes)) + } + + #[inline(always)] + pub(crate) const fn bytes(self) -> &'a [u8] { + self.0 + } + + #[inline(always)] + pub(crate) const fn len(self) -> u32 { + self.0.len() as u32 + } + + #[inline(always)] + pub(crate) fn read32(self, pos: u32) -> Option { + let pos = pos as usize; + let bytes = self.0.get(pos..pos + 4)?.first_chunk()?; + Some(u32::from_le_bytes(*bytes)) + } + + #[inline(always)] + pub(crate) fn read64(self, pos: u32) -> Option { + let pos = pos as usize; + let bytes = self.0.get(pos..pos + 8)?.first_chunk()?; + Some(u64::from_le_bytes(*bytes)) + } + + #[inline(always)] + fn chunk16(self, pos: u32) -> Option<&'a [u8; 16]> { + let pos = pos as usize; + self.0.get(pos..pos + 16)?.first_chunk() + } + + #[inline(always)] + pub(crate) fn read8(self, pos: u32) -> Option { + self.0.get(pos as usize).copied() + } + + /// `ZSTD_count`, up to the end. `match_pos` is less than `pos`. + #[inline(always)] + pub(crate) fn count(self, pos: u32, match_pos: u32) -> u32 { + let mut len = 0; + while let (Some(bytes), Some(match_bytes)) = + (self.read64(pos + len), self.read64(match_pos + len)) + { + let diff = bytes ^ match_bytes; + if diff != 0 { + return len + (diff.trailing_zeros() >> 3); + } + len += 8; + } + while let Some(byte) = self.read8(pos + len) + && self.read8(match_pos + len) == Some(byte) + { + len += 1; + } + len + } + + /// How many bytes before `pos`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. + #[inline(always)] + pub(crate) fn count_backwards( + self, + pos: u32, + anchor: u32, + match_pos: u32, + match_low: u32, + ) -> u32 { + let mut len = 0; + while pos - len > anchor + && match_pos - len > match_low + && let Some(byte) = self.read8(pos - len - 1) + && self.read8(match_pos - len - 1) == Some(byte) + { + len += 1; + } + len + } +} + +/// `table[hash]`. The size of `table` is a power of 2 that `hash` is less than, which the mask tells the compiler. +#[inline(always)] +pub(crate) fn table_get(table: &[u32], hash: usize) -> u32 { + let cell = table.get(hash & table.len().wrapping_sub(1)); + cell.copied().unwrap_or(0) +} + +/// `table[hash] = index`, likewise. +#[inline(always)] +pub(crate) fn table_put(table: &mut [u32], hash: usize, index: u32) { + let mask = table.len().wrapping_sub(1); + if let Some(cell) = table.get_mut(hash & mask) { + *cell = index; + } +} + /// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. #[inline(always)] pub(crate) fn count( @@ -645,6 +825,18 @@ pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) } +/// [`hash_ptr`] of the position that the 8 bytes `bytes` were read at. +#[inline(always)] +pub(crate) const fn hash_of(bytes: u64, h_bits: u32) -> usize { + match MLS { + 5 => hash5(bytes, h_bits, 0), + 6 => hash6(bytes, h_bits, 0), + 7 => hash7(bytes, h_bits, 0), + 8 => hash8(bytes, h_bits, 0), + _ => hash4(bytes as u32, h_bits, 0) as usize, + } +} + /// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, /// else 8. #[inline(always)] diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index c19d790c6159..de614529d6e3 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -3,10 +3,10 @@ use core::hint::{Locality, prefetch_read}; use super::internal::{ - BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Prefix, REPCODE1_TO_OFFBASE, - SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, - hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, offbase_to_offset, - offset_to_offbase, + BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Near, Prefix, + REPCODE1_TO_OFFBASE, SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, + count, count_2segments, hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, + offbase_to_offset, offset_to_offbase, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -52,6 +52,48 @@ fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Ok(()) } +/// What [`insert_dubt1`] works out about the position it sorts. +struct Unsorted<'a> { + bt_mask: u32, + dict_limit: u32, + dict_end: usize, + prefix_start: usize, + /// The buffer that the position is in, where it is in it, and where its segment ends. + ip_buf: &'a [u8], + ip: usize, + iend: usize, + window_low: u32, +} + +impl<'a> Unsorted<'a> { + fn new(ms: &MatchState, src: &BlockSrc<'a>, curr: u32, input_end: usize) -> Self { + let bt_log = ms.c_params.chain_log - 1; + let dict_limit = ms.window.dict_limit; + let dict_end = src.dict_pos_of(dict_limit); + let (ip_buf, ip, iend) = if curr >= dict_limit { + (src.buf, src.pos_of(curr), input_end) + } else { + (src.dict_buf, src.dict_pos_of(curr), dict_end) + }; + let window_valid = ms.window.low_limit; + let max_distance = 1u32 << ms.c_params.window_log; + Self { + bt_mask: (1u32 << bt_log) - 1, + dict_limit, + dict_end, + prefix_start: src.pos_of(dict_limit), + ip_buf, + ip, + iend, + window_low: if curr.wrapping_sub(window_valid) > max_distance { + curr - max_distance + } else { + window_valid + }, + } + } +} + /// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. fn insert_dubt1( ms: &mut MatchState, @@ -61,36 +103,20 @@ fn insert_dubt1( mut nb_compares: u32, bt_low: u32, ) -> Result<()> { - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; + let at = Unsorted::new(ms, src, curr, input_end); let mut common_length_smaller = 0usize; let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let (ip_buf, ip, iend) = if curr >= dict_limit { - (src.buf, src.pos_of(curr), input_end) - } else { - (src.dict_buf, src.dict_pos_of(curr), dict_end) - }; - let window_valid = ms.window.low_limit; - let max_distance = 1u32 << ms.c_params.window_log; - let window_low = if curr.wrapping_sub(window_valid) > max_distance { - curr - max_distance - } else { - window_valid - }; let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; + let mut smaller_ptr = 2 * (curr & at.bt_mask) as usize; let mut larger_ptr = smaller_ptr + 1; // The next sorted candidate. The other slot has the previous unsorted one. let mut match_index = bt.at(smaller_ptr)?; - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; + while nb_compares != 0 && match_index > at.window_low { + let next_ptr = 2 * (match_index & at.bt_mask) as usize; let mut match_length = common_length_smaller.min(common_length_larger); let in_prefix = - DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize; + DICT_MODE != EXT_DICT || match_index as usize + match_length >= at.dict_limit as usize; // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. let (mut match_buf, mut match_pos) = if in_prefix { (src.buf, src.pos_of(match_index)) @@ -98,36 +124,38 @@ fn insert_dubt1( (src.dict_buf, src.dict_pos_of(match_index)) }; - if in_prefix || curr < dict_limit { + if in_prefix || curr < at.dict_limit { // Both are in the same segment. match_length += count( - ip_buf, - ip + match_length, + at.ip_buf, + at.ip + match_length, match_buf, match_pos.wrapping_add(match_length), - iend, + at.iend, )?; } else { match_length += count_2segments( - ip_buf, - ip + match_length, + at.ip_buf, + at.ip + match_length, match_buf, match_pos + match_length, - iend, - dict_end, - prefix_start, + at.iend, + at.dict_end, + at.prefix_start, )?; - if match_index as usize + match_length >= dict_limit as usize { + if match_index as usize + match_length >= at.dict_limit as usize { (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); } } - if ip + match_length == iend { + if at.ip + match_length == at.iend { // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. break; } - if match_buf.at(match_pos.wrapping_add(match_length))? < ip_buf.at(ip + match_length)? { + if match_buf.at(match_pos.wrapping_add(match_length))? + < at.ip_buf.at(at.ip + match_length)? + { bt.put(smaller_ptr, match_index)?; common_length_smaller = match_length; if match_index <= bt_low { @@ -460,6 +488,35 @@ pub(crate) fn insert_and_find_first_index( insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) } +/// What [`hc_find_best_match`] works out about the position it searches for. +struct ChainSearch { + chain_mask: u32, + dict_limit: u32, + prefix_start: usize, + dict_end: usize, + curr: u32, + low_limit: u32, + min_chain: u32, +} + +impl ChainSearch { + #[inline(always)] + fn new(ms: &MatchState, src: &BlockSrc<'_>, ip: usize) -> Self { + let chain_size = 1u32 << ms.c_params.chain_log; + let dict_limit = ms.window.dict_limit; + let curr = src.index_of(ip); + Self { + chain_mask: chain_size - 1, + dict_limit, + prefix_start: src.pos_of(dict_limit), + dict_end: src.dict_pos_of(dict_limit), + curr, + low_limit: ms.get_lowest_match_index(curr, ms.c_params.window_log), + min_chain: curr.saturating_sub(chain_size), + } + } +} + /// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there /// is one. #[inline(never)] @@ -471,14 +528,7 @@ fn hc_find_best_match( offset_ptr: &mut u32, ) -> Result { let buf = src.buf; - let chain_size = 1u32 << ms.c_params.chain_log; - let chain_mask = chain_size - 1; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let min_chain = curr.saturating_sub(chain_size); + let search = ChainSearch::new(ms, src, ip); let mut nb_attempts = 1u32 << ms.c_params.search_log; let mut ml = 4 - 1; @@ -486,9 +536,9 @@ fn hc_find_best_match( insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; let chain_table = ms.chain_table.as_slice(); - while match_index >= low_limit && nb_attempts > 0 { + while match_index >= search.low_limit && nb_attempts > 0 { let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { + if DICT_MODE != EXT_DICT || match_index >= search.dict_limit { let match_pos = src.pos_of(match_index); // Whether it can be longer. if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { @@ -503,25 +553,25 @@ fn hc_find_best_match( src.dict_buf, match_pos + 4, i_limit, - dict_end, - prefix_start, + search.dict_end, + search.prefix_start, )? + 4; } } if current_ml > ml { ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + *offset_ptr = offset_to_offbase(search.curr.wrapping_sub(match_index)); if ip + current_ml == i_limit { // The best there can be. The next attempt would read too far. break; } } - if match_index <= min_chain { + if match_index <= search.min_chain { break; } - match_index = chain_table.at((match_index & chain_mask) as usize)?; + match_index = chain_table.at((match_index & search.chain_mask) as usize)?; nb_attempts -= 1; } @@ -532,7 +582,7 @@ fn hc_find_best_match( let dms_chain_mask = dms_chain_size - 1; let dms_lowest_index = dms.ms.window.dict_limit; let dms_size = dms_end_index(&dms); - let dms_index_delta = dict_limit.wrapping_sub(dms_size); + let dms_index_delta = search.dict_limit.wrapping_sub(dms_size); let dms_min_chain = dms_size.saturating_sub(dms_chain_size); let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; @@ -549,13 +599,15 @@ fn hc_find_best_match( match_pos + 4, i_limit, dms.end, - prefix_start, + search.prefix_start, )? + 4; } if current_ml > ml { ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); + let offset = search + .curr + .wrapping_sub(match_index.wrapping_add(dms_index_delta)); *offset_ptr = offset_to_offbase(offset); if ip + current_ml == i_limit { break; @@ -757,14 +809,19 @@ fn row_get_match_mask( tag: u8, head: u32, ) -> u64 { + const LOW_7_BITS: u64 = 0x7F7F_7F7F_7F7F_7F7F; + let tags = u64::from(tag) * 0x0101_0101_0101_0101; let mut matches = 0u64; - let (chunks, _) = tag_row.as_chunks::<16>(); + let (chunks, _) = tag_row.as_chunks::<8>(); for (n, chunk) in chunks.iter().enumerate() { - let mut chunk_matches = 0u16; - for (i, &entry) in chunk.iter().enumerate() { - chunk_matches |= u16::from(entry == tag) << i; - } - matches |= u64::from(chunk_matches) << (16 * n); + // A byte is 0 where the tags are the same. + let diff = u64::from_le_bytes(*chunk) ^ tags; + // The high bit of a byte is set if any of its bits is. No byte carries into the next. + let differs = ((diff & LOW_7_BITS) + LOW_7_BITS) | diff; + let same = (!differs & !LOW_7_BITS) >> 7; + // Bit `8 * i` goes to bit `56 + i`, and no two bits go to the same place. + let chunk_matches = same.wrapping_mul(0x0102_0408_1020_4080) >> 56; + matches |= chunk_matches << (8 * n); } match ROW_ENTRIES { 16 => u64::from((matches as u16).rotate_right(head)), @@ -895,13 +952,18 @@ fn row_find_best_match= dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; + let match_pos = match_index.wrapping_sub(dict_limit); + if near.read32(match_pos.wrapping_add(ml as u32 - 3)) == Some(tail) { + current_ml = near.count(near_ip, match_pos) as usize; } } else { let match_pos = src.dict_pos_of(match_index); @@ -925,6 +987,7 @@ fn row_find_best_match (usize, usize) { + fn feed(&mut self, data: &[u8], splits: &mut [usize; LDM_BATCH_SIZE]) -> Fed { let mut hash = self.rolling; let mask = self.stop_mask; let mut n = 0usize; @@ -144,10 +144,21 @@ impl RollingHashState { } } self.rolling = hash; - (n, num_splits) + Fed { + hashed: n, + num_splits, + } } } +/// What [`RollingHashState::feed`] did. +struct Fed { + /// How many bytes it went through. + hashed: usize, + /// How many splits it found. + num_splits: usize, +} + /// `XXH64`, with a seed of 0. #[inline] fn xxh64(data: &[u8]) -> u64 { @@ -318,7 +329,7 @@ pub(crate) fn fill_hash_table( let mut splits = [0usize; LDM_BATCH_SIZE]; while ip < iend { - let (hashed, num_splits) = hash_state.feed(prefix.buf.sub(ip..iend)?, &mut splits); + let Fed { hashed, num_splits } = hash_state.feed(prefix.buf.sub(ip..iend)?, &mut splits); for &split in splits.iter().take(num_splits) { if ip + split >= istart + min_match_length { let split = ip + split - min_match_length; @@ -402,7 +413,7 @@ fn generate_sequences_internal( ip += min_match_length; while ip < ilimit { - let (hashed, num_splits) = hash_state.feed(buf.sub(ip..ilimit)?, &mut splits); + let Fed { hashed, num_splits } = hash_state.feed(buf.sub(ip..ilimit)?, &mut splits); for (candidate, &split) in candidates.iter_mut().zip(&splits).take(num_splits) { let split = ip + split - min_match_length; diff --git a/src/zstd/compress/params.rs b/src/zstd/compress/params.rs index 2be5372af0a0..d062b759acb6 100644 --- a/src/zstd/compress/params.rs +++ b/src/zstd/compress/params.rs @@ -1,5 +1,7 @@ //! The parameters of `zstd_compress.c`: what can be asked for, and how it turns into what is applied. +use core::ops::RangeInclusive; + use super::internal::{ CompressionParameters, LdmParams, ParamSwitch, Strategy, ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, @@ -144,43 +146,42 @@ pub(crate) const NO_LDM_PARAMS: LdmParams = LdmParams { }; /// `ZSTD_cParam_getBounds` -fn bounds(param: i32) -> Result<(i32, i32)> { +fn bounds(param: i32) -> Result> { Ok(match param { - c_param::COMPRESSION_LEVEL => (ZSTD_MIN_CLEVEL, ZSTD_MAX_CLEVEL), - c_param::WINDOW_LOG => (ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX as i32), - c_param::HASH_LOG => (ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX), - c_param::CHAIN_LOG => (ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX), - c_param::SEARCH_LOG => (ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX), - c_param::MIN_MATCH => (ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX), - c_param::TARGET_LENGTH => (ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX), - c_param::STRATEGY => (Strategy::Fast as i32, Strategy::BtUltra2 as i32), - c_param::CONTENT_SIZE_FLAG | c_param::CHECKSUM_FLAG | c_param::DICT_ID_FLAG => (0, 1), - c_param::NB_WORKERS => (0, ZSTDMT_NBWORKERS_MAX), - c_param::JOB_SIZE => (0, ZSTDMT_JOBSIZE_MAX), - c_param::OVERLAP_LOG => (ZSTD_OVERLAPLOG_MIN, ZSTD_OVERLAPLOG_MAX), + c_param::COMPRESSION_LEVEL => ZSTD_MIN_CLEVEL..=ZSTD_MAX_CLEVEL, + c_param::WINDOW_LOG => ZSTD_WINDOWLOG_MIN..=ZSTD_WINDOWLOG_MAX as i32, + c_param::HASH_LOG => ZSTD_HASHLOG_MIN..=ZSTD_HASHLOG_MAX, + c_param::CHAIN_LOG => ZSTD_CHAINLOG_MIN..=ZSTD_CHAINLOG_MAX, + c_param::SEARCH_LOG => ZSTD_SEARCHLOG_MIN..=ZSTD_SEARCHLOG_MAX, + c_param::MIN_MATCH => ZSTD_MINMATCH_MIN..=ZSTD_MINMATCH_MAX, + c_param::TARGET_LENGTH => ZSTD_TARGETLENGTH_MIN..=ZSTD_TARGETLENGTH_MAX, + c_param::STRATEGY => Strategy::Fast as i32..=Strategy::BtUltra2 as i32, + c_param::CONTENT_SIZE_FLAG | c_param::CHECKSUM_FLAG | c_param::DICT_ID_FLAG => 0..=1, + c_param::NB_WORKERS => 0..=ZSTDMT_NBWORKERS_MAX, + c_param::JOB_SIZE => 0..=ZSTDMT_JOBSIZE_MAX, + c_param::OVERLAP_LOG => ZSTD_OVERLAPLOG_MIN..=ZSTD_OVERLAPLOG_MAX, c_param::ENABLE_LONG_DISTANCE_MATCHING => { - (ParamSwitch::Auto as i32, ParamSwitch::Disable as i32) + ParamSwitch::Auto as i32..=ParamSwitch::Disable as i32 } - c_param::LDM_HASH_LOG => (ZSTD_LDM_HASHLOG_MIN, ZSTD_LDM_HASHLOG_MAX), - c_param::LDM_MIN_MATCH => (ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), - c_param::LDM_BUCKET_SIZE_LOG => (ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), - c_param::LDM_HASH_RATE_LOG => (ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), - c_param::FORMAT => (0, 1), - c_param::TARGET_C_BLOCK_SIZE => (ZSTD_TARGETCBLOCKSIZE_MIN, ZSTD_TARGETCBLOCKSIZE_MAX), + c_param::LDM_HASH_LOG => ZSTD_LDM_HASHLOG_MIN..=ZSTD_LDM_HASHLOG_MAX, + c_param::LDM_MIN_MATCH => ZSTD_LDM_MINMATCH_MIN..=ZSTD_LDM_MINMATCH_MAX, + c_param::LDM_BUCKET_SIZE_LOG => ZSTD_LDM_BUCKETSIZELOG_MIN..=ZSTD_LDM_BUCKETSIZELOG_MAX, + c_param::LDM_HASH_RATE_LOG => ZSTD_LDM_HASHRATELOG_MIN..=ZSTD_LDM_HASHRATELOG_MAX, + c_param::FORMAT => 0..=1, + c_param::TARGET_C_BLOCK_SIZE => ZSTD_TARGETCBLOCKSIZE_MIN..=ZSTD_TARGETCBLOCKSIZE_MAX, _ => return Err(ErrorCode::ParameterUnsupported), }) } /// `ZSTD_cParam_clampBounds` fn clamp_bounds(param: i32, value: i32) -> Result { - let (lower, upper) = bounds(param)?; - Ok(value.clamp(lower, upper)) + let bounds = bounds(param)?; + Ok(value.clamp(*bounds.start(), *bounds.end())) } /// `BOUNDCHECK` fn bound_check(param: i32, value: i32) -> Result<()> { - let (lower, upper) = bounds(param)?; - if value < lower || value > upper { + if !bounds(param)?.contains(&value) { return Err(ErrorCode::ParameterOutOfBound); } Ok(()) diff --git a/src/zstd/compress/sequences.rs b/src/zstd/compress/sequences.rs index c3eb497cb86b..2f05b7d28137 100644 --- a/src/zstd/compress/sequences.rs +++ b/src/zstd/compress/sequences.rs @@ -232,10 +232,20 @@ pub(crate) fn build_ctable( } #[inline(always)] -fn bits_of(table: &[u8; N], code: u8) -> u32 { - u32::from(table.get(usize::from(code)).copied().unwrap_or(0)) +fn bits_of(table: &[u8; 64], code: u8) -> u32 { + u32::from(table.get(usize::from(code & 63)).copied().unwrap_or(0)) } +/// `table`, long enough for a masked code to be known to be in bounds. +const fn padded(table: &[u8; N]) -> [u8; 64] { + let mut padded = [0; 64]; + let (head, _) = padded.split_at_mut(N); + head.copy_from_slice(table); + padded +} +static LL_BITS_PADDED: [u8; 64] = padded(&LL_BITS); +static ML_BITS_PADDED: [u8; 64] = padded(&ML_BITS); + /// `ZSTD_encodeSequences`: the four slices are of the same length, which is not 0. pub(crate) fn encode_sequences( dst: &mut [u8], @@ -266,8 +276,14 @@ pub(crate) fn encode_sequences( let mut state_match_length = FseCState::new2(ctable_match_length, u32::from(ml_last)); let mut state_offset_bits = FseCState::new2(ctable_offset_bits, u32::from(of_last)); let mut state_lit_length = FseCState::new2(ctable_lit_length, u32::from(ll_last)); - block_stream.add_bits(u64::from(seq_last.lit_length), bits_of(&LL_BITS, ll_last)); - block_stream.add_bits(u64::from(seq_last.ml_base), bits_of(&ML_BITS, ml_last)); + block_stream.add_bits( + u64::from(seq_last.lit_length), + bits_of(&LL_BITS_PADDED, ll_last), + ); + block_stream.add_bits( + u64::from(seq_last.ml_base), + bits_of(&ML_BITS_PADDED, ml_last), + ); block_stream.add_bits(u64::from(seq_last.off_base), u32::from(of_last)); block_stream.flush_bits(); @@ -278,9 +294,9 @@ pub(crate) fn encode_sequences( .zip(ml_codes) .rev() { - let ll_bits = bits_of(&LL_BITS, ll_code); + let ll_bits = bits_of(&LL_BITS_PADDED, ll_code); let of_bits = u32::from(of_code); - let ml_bits = bits_of(&ML_BITS, ml_code); + let ml_bits = bits_of(&ML_BITS_PADDED, ml_code); state_offset_bits.encode_symbol(&mut block_stream, u32::from(of_code)); state_match_length.encode_symbol(&mut block_stream, u32::from(ml_code)); state_lit_length.encode_symbol(&mut block_stream, u32::from(ll_code)); diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index dabaefe5ca2c..293a380e0c9f 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -524,9 +524,10 @@ fn fill_dtable_x2_level2( 2 | 4 => length, _ => skip_size.next_multiple_of(8), }; + let first_symbol = base_seq; dtable .sub_mut(..filled)? - .fill(build_delt_x2(base_seq, consumed_bits, 0, 1)); + .fill(build_delt_x2(first_symbol, consumed_bits, 0, 1)); } for w in min_weight..max_weight1 { diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index e547eedcfbd4..bc932de17ddf 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -453,7 +453,10 @@ enum DStage { #[derive(Clone, Copy, PartialEq, Eq)] enum DStreamStage { Init, - LoadHeader, + /// `lh_size` bytes of the frame header are in `header_buffer`. + LoadHeader { + lh_size: usize, + }, Read, Load, Flush, @@ -497,7 +500,6 @@ struct DState { out_buff: Vec, out_start: usize, out_end: usize, - lh_size: usize, /// The last byte of a frame's input is held back until its output is flushed. hostage_byte: bool, no_forward_progress: u32, @@ -547,7 +549,6 @@ impl DCtx { out_buff: Vec::new(), out_start: 0, out_end: 0, - lh_size: 0, hostage_byte: false, no_forward_progress: 0, header_buffer: [0; ZSTD_FRAMEHEADERSIZE_MAX], @@ -1028,6 +1029,37 @@ impl DState { Ok(()) } + /// Buffers for the frame whose header is `f_params`. Those that are there stay, unless they are too small, or have + /// been much too large for many frames. + fn resize_buffers(&mut self) -> Result<()> { + let needed_in_buff_size = self.f_params.block_size_max.max(4); + let needed_out_buff_size = decoding_buffer_size( + self.f_params.window_size, + self.f_params.frame_content_size, + self.f_params.block_size_max, + ); + + // `ZSTD_DCtx_updateOversizedDuration` + if self.in_buff.len() + self.out_buff.len() + >= (needed_in_buff_size + needed_out_buff_size) * ZSTD_WORKSPACETOOLARGE_FACTOR + { + self.oversized_duration += 1; + } else { + self.oversized_duration = 0; + } + + let too_small = + self.in_buff.len() < needed_in_buff_size || self.out_buff.len() < needed_out_buff_size; + let too_large = self.oversized_duration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; + if too_small || too_large { + self.in_buff = Vec::new(); + self.out_buff = Vec::new(); + self.in_buff = try_zeroed(needed_in_buff_size)?; + self.out_buff = try_zeroed(needed_out_buff_size)?; + } + Ok(()) + } + /// `ZSTD_decompressStream` fn decompress_stream( &mut self, @@ -1051,32 +1083,33 @@ impl DState { 'work: loop { if self.stream_stage == DStreamStage::Init { - self.stream_stage = DStreamStage::LoadHeader; - self.lh_size = 0; + self.stream_stage = DStreamStage::LoadHeader { lh_size: 0 }; self.in_pos = 0; self.out_start = 0; self.out_end = 0; self.hostage_byte = false; } - if self.stream_stage == DStreamStage::LoadHeader { - match get_frame_header(self.header_buffer.sub(..self.lh_size)?)? { + if let DStreamStage::LoadHeader { lh_size } = self.stream_stage { + match get_frame_header(self.header_buffer.sub(..lh_size)?)? { HeaderStatus::Complete(header) => self.f_params = header, HeaderStatus::Needs(h_size) => { - let to_load = h_size - self.lh_size; + let to_load = h_size - lh_size; let remaining_input = iend - ip; if to_load > remaining_input { - self.header_buffer.write_at(self.lh_size, src.sub(ip..)?)?; - self.lh_size += remaining_input; + self.header_buffer.write_at(lh_size, src.sub(ip..)?)?; + let lh_size = lh_size + remaining_input; + self.stream_stage = DStreamStage::LoadHeader { lh_size }; input.pos = iend; // Refuse what cannot become a frame header. - get_frame_header(self.header_buffer.sub(..self.lh_size)?)?; - return Ok(FRAMEHEADERSIZE_MIN.max(h_size) - self.lh_size - + ZSTD_BLOCKHEADERSIZE); + get_frame_header(self.header_buffer.sub(..lh_size)?)?; + return Ok( + FRAMEHEADERSIZE_MIN.max(h_size) - lh_size + ZSTD_BLOCKHEADERSIZE + ); } self.header_buffer - .write_at(self.lh_size, src.span(ip, to_load)?)?; - self.lh_size = h_size; + .write_at(lh_size, src.span(ip, to_load)?)?; + self.stream_stage = DStreamStage::LoadHeader { lh_size: h_size }; ip += to_load; continue 'work; } @@ -1109,7 +1142,7 @@ impl DState { self.stage = DStage::SkipFrame; } else { let header_buffer = self.header_buffer; - self.decode_frame_header(header_buffer.sub(..self.lh_size)?)?; + self.decode_frame_header(header_buffer.sub(..lh_size)?)?; self.expected = ZSTD_BLOCKHEADERSIZE; self.stage = DStage::DecodeBlockHeader; } @@ -1122,34 +1155,7 @@ impl DState { return Err(ErrorCode::FrameParameterWindowTooLarge); } - { - let needed_in_buff_size = self.f_params.block_size_max.max(4); - let needed_out_buff_size = decoding_buffer_size( - self.f_params.window_size, - self.f_params.frame_content_size, - self.f_params.block_size_max, - ); - - // `ZSTD_DCtx_updateOversizedDuration` - if self.in_buff.len() + self.out_buff.len() - >= (needed_in_buff_size + needed_out_buff_size) - * ZSTD_WORKSPACETOOLARGE_FACTOR - { - self.oversized_duration += 1; - } else { - self.oversized_duration = 0; - } - - let too_small = self.in_buff.len() < needed_in_buff_size - || self.out_buff.len() < needed_out_buff_size; - let too_large = self.oversized_duration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; - if too_small || too_large { - self.in_buff = Vec::new(); - self.out_buff = Vec::new(); - self.in_buff = try_zeroed(needed_in_buff_size)?; - self.out_buff = try_zeroed(needed_out_buff_size)?; - } - } + self.resize_buffers()?; self.stream_stage = DStreamStage::Read; } diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index e21911b3742c..81817dbce674 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -1,6 +1,6 @@ //! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the //! same bytes in, in both directions, and the same errors. -#![feature(hint_prefetch)] +#![feature(hint_prefetch, vec_push_within_capacity)] #![forbid(unsafe_code)] // Frames come from anywhere and a panic aborts the process. #![deny( From 11a8d5b5d87e9eb380ab867b0f1f7805b20de432 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 14:45:43 +0000 Subject: [PATCH 015/199] zstd: shorter prologues in the generic match finders, contexts are Sync - fast, dfast: what does not depend on the minimum match length moves out of the functions that are compiled once for each (first position, least step, repeat offsets with an older segment, width of a tagged hash). - btlazy2: the slot of the position to sort and its first candidate are worked out with the rest of what is known about it. - Workers use mpmc channels, whose receivers are Sync, so that a context is Send + Sync and an unsafe impl around one has nothing to vouch for. - Removes what nothing reaches: indexing a dictionary for long distance matching (only for a prefix from the user) and the full load method when filling the table of a context (the C code asserts that it is not used). --- src/zstd/compress/cctx.rs | 24 ++----- src/zstd/compress/double_fast.rs | 14 ++-- src/zstd/compress/fast.rs | 119 ++++++++++++++----------------- src/zstd/compress/internal.rs | 10 +++ src/zstd/compress/lazy.rs | 24 ++++--- src/zstd/compress/ldm.rs | 33 --------- src/zstd/compress/mt.rs | 6 +- src/zstd/lib.rs | 9 ++- 8 files changed, 101 insertions(+), 138 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index ec2b6eacdbbd..8b21974b9a36 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -275,9 +275,10 @@ fn overflow_correct_if_needed(ms: &mut MatchState, params: &CCtxParams, ip: usiz } /// `ZSTD_loadDictionaryContent`: makes `src` the start of the window, and indexes it. +/// +/// The C code also indexes it for long distance matching if it is a prefix from the user, which there is none of here. fn load_dictionary_content( ms: &mut MatchState, - ls: Option<&mut LdmState>, params: &CCtxParams, bufs: &Buffers<'_>, mut src: Span, @@ -300,20 +301,6 @@ fn load_dictionary_content( ms.window .update(src.start_addr(), src.buf, src.len(), false); - if let Some(ls) = ls - && params.ldm_params.enable_ldm == ParamSwitch::Enable - { - ls.window - .update(src.start_addr(), src.buf, src.len(), false); - ls.loaded_dict_end = if params.force_window { - 0 - } else { - ls.window.index_of(src.end_addr()) - }; - let prefix = window_prefix(&ls.window, bufs); - ldm::fill_hash_table(ls, prefix, src.start, src.end, ¶ms.ldm_params)?; - } - { // More than the tables can hold is not worth indexing. let log = (params.c_params.hash_log + 3) @@ -477,7 +464,6 @@ fn load_c_entropy(bs: &mut CompressedBlockState, dict: &[u8]) -> Result { fn compress_insert_dictionary( bs: &mut CompressedBlockState, ms: &mut MatchState, - ls: Option<&mut LdmState>, params: &CCtxParams, bufs: &Buffers<'_>, dict: Option, @@ -495,7 +481,7 @@ fn compress_insert_dictionary( if dict_content_type == DictContentType::RawContent || read_le32(bytes, 0)? != ZSTD_MAGIC_DICTIONARY { - load_dictionary_content(ms, ls, params, bufs, dict, dtlm, tfp)?; + load_dictionary_content(ms, params, bufs, dict, dtlm, tfp)?; return Ok(0); } @@ -510,7 +496,7 @@ fn compress_insert_dictionary( start: dict.start + e_size, ..dict }; - load_dictionary_content(ms, None, params, bufs, content, dtlm, tfp)?; + load_dictionary_content(ms, params, bufs, content, dtlm, tfp)?; Ok(dict_id) } @@ -559,7 +545,6 @@ impl CDict { cdict.dict_id = compress_insert_dictionary( &mut cdict.c_block_state, &mut cdict.match_state, - None, ¶ms, &bufs, Some(Span { @@ -1091,7 +1076,6 @@ impl FrameCompressor { self.dict_id = compress_insert_dictionary( &mut self.prev_c_block, &mut self.match_state, - Some(&mut self.ldm_state), &self.applied_params, &input.bufs, dict, diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index a4b99966c902..2425f53e5e6b 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -18,8 +18,8 @@ fn fill_double_hash_table_for_cdict( end: usize, dtlm: DictTableLoadMethod, ) -> Result<()> { - let h_bits_l = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let h_bits_s = ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; + let h_bits_l = ms.tagged_hash_log(); + let h_bits_s = ms.tagged_chain_log(); let mls = ms.c_params.min_match; let hash_large = ms.hash_table.as_mut_slice(); let hash_small = ms.chain_table.as_mut_slice(); @@ -303,16 +303,16 @@ fn compress_block_double_fast_dict_match_state_generic( let h_bits_l = ms.c_params.hash_log; let h_bits_s = ms.c_params.chain_log; let [mut offset_1, mut offset_2, _] = *rep; + let hash_long = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); let dms = src.dms.as_ref().ok_or(BUG)?; let dict_hash_long = dms.ms.hash_table.as_slice(); let dict_hash_small = dms.ms.chain_table.as_slice(); - let dict_h_bits_l = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let dict_h_bits_s = dms.ms.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS; - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); + let dict_h_bits_l = dms.ms.tagged_hash_log(); + let dict_h_bits_s = dms.ms.tagged_chain_log(); let mut anchor = seg.istart; - let mut ip = seg.istart + usize::from(seg.dict_and_prefix_length() == 0); + let mut ip = seg.first(); 'search: while ip < seg.ilimit { let h2 = hash_ptr(seg.buf, ip, h_bits_l, 8)?; diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index e5bed54e5169..2880cd1cd612 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -1,10 +1,10 @@ //! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. use super::internal::{ - BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Near, Prefix, - REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, - compare_packed_tags, count, count_2segments, hash_of, hash_ptr, index_overlap_check, - offset_to_offbase, table_get, table_put, write_tagged_index, + BlockSrc, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, + MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, + ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, count, count_2segments, hash_of, hash_ptr, + index_overlap_check, offset_to_offbase, table_get, table_put, write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -101,12 +101,14 @@ impl<'a> TwoSegments<'a> { })) } - /// How many bytes matches can be in before the block. - pub(super) const fn dict_and_prefix_length(&self) -> u32 { - self.istart + /// The first position to look at, with a dictionary: not `istart` if there is nothing before it to match. + pub(super) const fn first(&self) -> usize { + let dict_and_prefix_length = self + .istart .wrapping_sub(self.prefix_start) .wrapping_add(self.dict_end) - .wrapping_sub(self.dict_start) as u32 + .wrapping_sub(self.dict_start) as u32; + self.istart + (dict_and_prefix_length == 0) as usize } /// The position in `dict_buf` of an index of the other segment. @@ -182,6 +184,12 @@ impl SavedRep { offset_saved2, } } + + fn with_ext_dict(rep: &[u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, seg: &TwoSegments<'_>) -> Self { + let max_rep = src.index_of(seg.istart).wrapping_sub(seg.dict_start_index); + // As far as `max_rep` is too far too. + Self::new(rep, max_rep.saturating_sub(1)) + } } /// The repeat offsets to leave at the end of a block, at the start of which those that reached too far were saved and @@ -220,7 +228,7 @@ fn fill_hash_table_for_cdict( end: usize, dtlm: DictTableLoadMethod, ) -> Result<()> { - let h_bits = ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; + let h_bits = ms.tagged_hash_log(); let mls = ms.c_params.min_match; let hash_table = ms.hash_table.as_mut_slice(); let buf = prefix.buf; @@ -246,13 +254,8 @@ fn fill_hash_table_for_cdict( Ok(()) } -/// `ZSTD_fillHashTableForCCtx` -fn fill_hash_table_for_cctx( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { +/// `ZSTD_fillHashTableForCCtx`, which is only for `ZSTD_dtlm_fast`. +fn fill_hash_table_for_cctx(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) -> Result<()> { let h_bits = ms.c_params.hash_log; let mls = ms.c_params.min_match; let hash_table = ms.hash_table.as_mut_slice(); @@ -265,15 +268,6 @@ fn fill_hash_table_for_cctx( while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { let curr = prefix.index_of(ip); hash_table.put(hash_ptr(buf, ip, h_bits, mls)?, curr)?; - if dtlm == DictTableLoadMethod::Full { - // The positions in between only go where there is nothing yet. - for p in 1..FAST_HASH_FILL_STEP { - let hash = hash_ptr(buf, ip + p as usize, h_bits, mls)?; - if hash_table.at(hash)? == 0 { - hash_table.put(hash, curr + p)?; - } - } - } ip += FAST_HASH_FILL_STEP as usize; } Ok(()) @@ -289,7 +283,7 @@ pub(crate) fn fill_hash_table( ) -> Result<()> { match tfp { TableFillPurpose::ForCDict => fill_hash_table_for_cdict(ms, prefix, end, dtlm), - TableFillPurpose::ForCCtx => fill_hash_table_for_cctx(ms, prefix, end, dtlm), + TableFillPurpose::ForCCtx => fill_hash_table_for_cctx(ms, prefix, end), } } @@ -306,6 +300,11 @@ fn match4_found( Ok(match_idx >= idx_low_limit && read_le32(buf, current)? == read_le32(buf, match_pos)?) } +/// `targetLength + !targetLength`: how far `fast` goes from one position to the next, before it speeds up. +const fn least_step(c_params: &CompressionParameters) -> u32 { + c_params.target_length + (c_params.target_length == 0) as u32 +} + /// What `fast` and `dfast` work out before they look for matches in a block, when there is no dictionary and the window /// is one segment. pub(super) struct NoDictBlock<'a> { @@ -388,21 +387,13 @@ fn compress_block_fast_no_dict_generic( let Some(block) = NoDictBlock::new(ms, rep, src)? else { return Ok(src.end - src.start); }; - let NoDictBlock { - near, - base_index, - candidates, - prefix_lowest_index: prefix_start_index, - prefix_lowest: prefix_start, - ilimit, - offset_1: mut rep_offset1, - offset_2: mut rep_offset2, - .. - } = block; + let near = block.near; + let candidates = block.candidates; + let mut rep_offset1 = block.offset_1; + let mut rep_offset2 = block.offset_2; let hlog = ms.c_params.hash_log; - let target_length = ms.c_params.target_length; // The least step is 2, as two positions are looked at per turn. - let step_size = target_length + u32::from(target_length == 0) + 1; + let step_size = least_step(&ms.c_params) + 1; let mut anchor = block.istart; let mut ip0 = block.first; let mut seq_store = SeqSink::new(seq_store); @@ -415,7 +406,7 @@ fn compress_block_fast_no_dict_generic( let mut ip1 = ip0 + 1; let mut ip2 = ip0 + step; let mut ip3 = ip2 + 1; - if ip3 as usize >= ilimit { + if ip3 as usize >= block.ilimit { break 'start; } @@ -432,7 +423,7 @@ fn compress_block_fast_no_dict_generic( let bytes2 = near.read64(ip2).ok_or(BUG)?; let rval = near.read32(ip2.wrapping_sub(rep_offset1)); - current0 = base_index.wrapping_add(ip0); + current0 = block.base_index.wrapping_add(ip0); table_put(hash_table, hash0, current0); if (rval == Some(bytes2 as u32)) & (rep_offset1 > 0) { @@ -441,13 +432,14 @@ fn compress_block_fast_no_dict_generic( let back = u32::from(near.read8(ip0 - 1) == near.read8(match0.wrapping_sub(1))); ip0 -= back; // The hash of ip1 is there, and ip1 is before the end of the match. - table_put(hash_table, hash1, base_index.wrapping_add(ip1)); + table_put(hash_table, hash1, block.base_index.wrapping_add(ip1)); break Some((match0.wrapping_sub(back), 4 + back)); } - if candidates.read32(match_idx.wrapping_sub(prefix_start_index)) == Some(bytes0 as u32) + if candidates.read32(match_idx.wrapping_sub(block.prefix_lowest_index)) + == Some(bytes0 as u32) { - table_put(hash_table, hash1, base_index.wrapping_add(ip1)); + table_put(hash_table, hash1, block.base_index.wrapping_add(ip1)); break None; } @@ -461,14 +453,15 @@ fn compress_block_fast_no_dict_generic( ip1 = ip2; ip2 = ip3; - current0 = base_index.wrapping_add(ip0); + current0 = block.base_index.wrapping_add(ip0); table_put(hash_table, hash0, current0); - if candidates.read32(match_idx.wrapping_sub(prefix_start_index)) == Some(bytes0 as u32) + if candidates.read32(match_idx.wrapping_sub(block.prefix_lowest_index)) + == Some(bytes0 as u32) { // ip1 is certain to be before the end of the match only if the step is small. if step <= 4 { - table_put(hash_table, hash1, base_index.wrapping_add(ip1)); + table_put(hash_table, hash1, block.base_index.wrapping_add(ip1)); } break None; } @@ -488,7 +481,7 @@ fn compress_block_fast_no_dict_generic( step += 1; next_step += K_STEP_INCR; } - if ip3 as usize >= ilimit { + if ip3 as usize >= block.ilimit { break 'start; } }; @@ -496,10 +489,10 @@ fn compress_block_fast_no_dict_generic( let (offcode, match0, mut m_length) = match rep_match { Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), None => { - let match0 = match_idx - base_index; + let match0 = match_idx - block.base_index; rep_offset2 = rep_offset1; rep_offset1 = ip0 - match0; - let back = near.count_backwards(ip0, anchor, match0, prefix_start); + let back = near.count_backwards(ip0, anchor, match0, block.prefix_lowest); ip0 -= back; (offset_to_offbase(rep_offset1), match0 - back, 4 + back) } @@ -510,17 +503,17 @@ fn compress_block_fast_no_dict_generic( ip0 += m_length; anchor = ip0; - if ip0 as usize <= ilimit { - let to_insert = near.read64(current0 + 2 - base_index).ok_or(BUG)?; + if ip0 as usize <= block.ilimit { + let to_insert = near.read64(current0 + 2 - block.base_index).ok_or(BUG)?; table_put(hash_table, hash_of::(to_insert, hlog), current0 + 2); let before2 = near.read64(ip0 - 2).ok_or(BUG)?; table_put( hash_table, hash_of::(before2, hlog), - base_index.wrapping_add(ip0 - 2), + block.base_index.wrapping_add(ip0 - 2), ); - while rep_offset2 > 0 && ip0 as usize <= ilimit { + while rep_offset2 > 0 && ip0 as usize <= block.ilimit { let bytes = near.read64(ip0).ok_or(BUG)?; if near.read32(ip0.wrapping_sub(rep_offset2)) != Some(bytes as u32) { break; @@ -530,7 +523,7 @@ fn compress_block_fast_no_dict_generic( table_put( hash_table, hash_of::(bytes, hlog), - base_index.wrapping_add(ip0), + block.base_index.wrapping_add(ip0), ); ip0 += r_length; seq_store.store_seq(0, near, anchor, REPCODE1_TO_OFFBASE, r_length)?; @@ -575,16 +568,15 @@ fn compress_block_fast_dict_match_state_generic( const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; let hlog = ms.c_params.hash_log; - let target_length = ms.c_params.target_length; - let step_size = (target_length + u32::from(target_length == 0)) as usize; + let step_size = least_step(&ms.c_params) as usize; let [mut offset_1, mut offset_2, _] = *rep; + let hash_table = ms.hash_table.as_mut_slice(); let dms = src.dms.as_ref().ok_or(BUG)?; let dict_hash_table = dms.ms.hash_table.as_slice(); - let dict_h_bits = dms.ms.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS; - let hash_table = ms.hash_table.as_mut_slice(); + let dict_h_bits = dms.ms.tagged_hash_log(); let mut anchor = seg.istart; - let mut ip0 = seg.istart + usize::from(seg.dict_and_prefix_length() == 0); + let mut ip0 = seg.first(); let mut ip1 = seg.istart + step_size; 'search: while ip1 <= seg.ilimit { @@ -791,18 +783,15 @@ fn compress_block_fast_ext_dict_generic( const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); let hlog = ms.c_params.hash_log; - let target_length = ms.c_params.target_length; - let step_size = (target_length + u32::from(target_length == 0) + 1) as usize; + let step_size = least_step(&ms.c_params) as usize + 1; let mut anchor = seg.istart; let mut ip0 = seg.istart; - let max_rep = src.index_of(ip0).wrapping_sub(seg.dict_start_index); - // As far as `max_rep` is too far too. let SavedRep { mut offset_1, mut offset_2, offset_saved1, offset_saved2, - } = SavedRep::new(rep, max_rep.saturating_sub(1)); + } = SavedRep::with_ext_dict(rep, src, seg); // Whether there is a match for the position `ip` at `idx`, which is from the table. let match_found = |ip: usize, idx: u32| -> Result { diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index a732e979a2de..308f45076f37 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1268,6 +1268,16 @@ impl MatchState { } } + /// How many bits to hash to for `hash_table` when it is a dictionary's, where entries have tags. + pub(crate) const fn tagged_hash_log(&self) -> u32 { + self.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS + } + + /// The same for `chain_table`, as `dfast` uses it. + pub(crate) const fn tagged_chain_log(&self) -> u32 { + self.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS + } + /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { if self.window.has_ext_dict() { diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index de614529d6e3..0696bd6dc8ff 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -55,6 +55,10 @@ fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> /// What [`insert_dubt1`] works out about the position it sorts. struct Unsorted<'a> { bt_mask: u32, + /// Where its two slots are in the table. + slot: usize, + /// The next sorted candidate, which is in the first slot. The other one has the previous unsorted candidate. + next_sorted: u32, dict_limit: u32, dict_end: usize, prefix_start: usize, @@ -66,8 +70,9 @@ struct Unsorted<'a> { } impl<'a> Unsorted<'a> { - fn new(ms: &MatchState, src: &BlockSrc<'a>, curr: u32, input_end: usize) -> Self { - let bt_log = ms.c_params.chain_log - 1; + fn new(ms: &MatchState, src: &BlockSrc<'a>, curr: u32, input_end: usize) -> Result { + let bt_mask = (1u32 << (ms.c_params.chain_log - 1)) - 1; + let slot = 2 * (curr & bt_mask) as usize; let dict_limit = ms.window.dict_limit; let dict_end = src.dict_pos_of(dict_limit); let (ip_buf, ip, iend) = if curr >= dict_limit { @@ -77,8 +82,10 @@ impl<'a> Unsorted<'a> { }; let window_valid = ms.window.low_limit; let max_distance = 1u32 << ms.c_params.window_log; - Self { - bt_mask: (1u32 << bt_log) - 1, + Ok(Self { + bt_mask, + slot, + next_sorted: ms.chain_table.at(slot)?, dict_limit, dict_end, prefix_start: src.pos_of(dict_limit), @@ -90,7 +97,7 @@ impl<'a> Unsorted<'a> { } else { window_valid }, - } + }) } } @@ -103,14 +110,13 @@ fn insert_dubt1( mut nb_compares: u32, bt_low: u32, ) -> Result<()> { - let at = Unsorted::new(ms, src, curr, input_end); + let at = Unsorted::new(ms, src, curr, input_end)?; let mut common_length_smaller = 0usize; let mut common_length_larger = 0usize; let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & at.bt_mask) as usize; + let mut smaller_ptr = at.slot; let mut larger_ptr = smaller_ptr + 1; - // The next sorted candidate. The other slot has the previous unsorted one. - let mut match_index = bt.at(smaller_ptr)?; + let mut match_index = at.next_sorted; while nb_compares != 0 && match_index > at.window_low { let next_ptr = 2 * (match_index & at.bt_mask) as usize; diff --git a/src/zstd/compress/ldm.rs b/src/zstd/compress/ldm.rs index 30ef5ed6853f..b1b17a6b677d 100644 --- a/src/zstd/compress/ldm.rs +++ b/src/zstd/compress/ldm.rs @@ -314,39 +314,6 @@ fn fill_fast_tables(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) -> Resu } } -/// `ZSTD_ldm_fillHashTable`: indexes the content of a dictionary, which is `prefix.buf` from `ip` to `iend`. -pub(crate) fn fill_hash_table( - ldm_state: &mut LdmState, - prefix: Prefix<'_>, - mut ip: usize, - iend: usize, - params: &LdmParams, -) -> Result<()> { - let min_match_length = params.min_match_length as usize; - let bucket_mask = bucket_mask(params); - let istart = ip; - let mut hash_state = RollingHashState::new(params); - let mut splits = [0usize; LDM_BATCH_SIZE]; - - while ip < iend { - let Fed { hashed, num_splits } = hash_state.feed(prefix.buf.sub(ip..iend)?, &mut splits); - for &split in splits.iter().take(num_splits) { - if ip + split >= istart + min_match_length { - let split = ip + split - min_match_length; - let xxhash = xxh64(prefix.buf.span(split, min_match_length)?); - let hash = xxhash as u32 & bucket_mask; - let entry = LdmEntry { - offset: prefix.index_of(split), - checksum: (xxhash >> 32) as u32, - }; - insert_entry(ldm_state, hash as usize, entry, params.bucket_size_log)?; - } - } - ip += hashed; - } - Ok(()) -} - /// `ZSTD_ldm_limitTableUpdate`: if the tables of `ms` are far behind `anchor`, skips most of the way. fn limit_table_update(ms: &mut MatchState, src: &BlockSrc<'_>, anchor: usize) { let curr = src.index_of(anchor); diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 352b25aaff99..9eeb40ee4387 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -9,7 +9,7 @@ use std::collections::VecDeque; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::{Receiver, Sender, channel}; +use std::sync::mpmc::{Receiver, Sender, TryRecvError, channel}; use std::thread::JoinHandle; use super::cctx::{ @@ -350,8 +350,8 @@ impl PendingJob { } else { match receiver.try_recv() { Ok(progress) => Some(progress), - Err(std::sync::mpsc::TryRecvError::Empty) => return, - Err(std::sync::mpsc::TryRecvError::Disconnected) => None, + Err(TryRecvError::Empty) => return, + Err(TryRecvError::Disconnected) => None, } }; match received { diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 81817dbce674..0c3995e8ac8e 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -1,6 +1,6 @@ //! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the //! same bytes in, in both directions, and the same errors. -#![feature(hint_prefetch, vec_push_within_capacity)] +#![feature(hint_prefetch, mpmc_channel, vec_push_within_capacity)] #![forbid(unsafe_code)] // Frames come from anywhere and a panic aborts the process. #![deny( @@ -27,6 +27,13 @@ pub use decompress::{DCtx, DDict, FrameExtent, frame_extent, get_frame_content_s use decompress::{decompress, find_decompressed_size}; pub use error::ErrorCode; +// Callers share contexts between threads behind their own `unsafe impl`s, which should have nothing to vouch for here. +const _: () = { + const fn shareable() {} + shareable::(); + shareable::(); +}; + /// `ZSTD_inBuffer` pub struct InBuffer<'a> { pub src: &'a [u8], From 9c7f6e0062af7ce51c0759b460a9e1b9e574de65 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 15:14:54 +0000 Subject: [PATCH 016/199] zstd: a job with workers makes room for the long distance matches of its own input The buffer was sized, and zeroed, for a whole section (jobSize / ldmMinMatch entries of 12 bytes) for every job, so a flush of 1 KB cost as much as that: 20x the time of the C library at level 22, and gigabytes with a large jobSize. The C library allocates as much but only touches what it uses. The limit that the search stops at is still that of a whole section, as it decides whether the input gets indexed at all. --- src/zstd/compress/mt.rs | 7 +++++-- test/js/bun/util/zstd.test.ts | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 9eeb40ee4387..771b482a22e7 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -138,8 +138,10 @@ struct Job { params: CCtxParams, cdict: Option>, full_frame_size: u64, - /// How many long distance matches to make room for. + /// How many long distance matches the C code makes room for, which is as many as a whole section can have. max_nb_seq: usize, + /// How many the input of this job can have. + nb_seq_bound: usize, /// From the previous job. serial_in: Receiver>, /// To the next one. @@ -174,7 +176,7 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - let generated = if cancelled.load(Ordering::SeqCst) { Err(ErrorCode::Generic) } else { - reset_struct_table(&mut raw_seq_store.seq, job.max_nb_seq, false) + reset_struct_table(&mut raw_seq_store.seq, job.nb_seq_bound, false) .and_then(|()| serial.gen_sequences(&mut raw_seq_store, src, job.round_pos)) }; // The receiver is gone if the frame is given up. @@ -640,6 +642,7 @@ impl MtCtx { cdict: if first_job { self.cdict.clone() } else { None }, full_frame_size: self.frame_content_size, max_nb_seq: ldm::get_max_nb_seq(ldm_params, self.target_section_size), + nb_seq_bound: ldm::get_max_nb_seq(ldm_params, src_size), serial_in, serial_out, progress, diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 4646f452d4a1..03c32f3ce389 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1347,4 +1347,28 @@ describe("the output is that of zstd 1.5.7", () => { expect(zlib.zstdDecompressSync(compressed).equals(corpus)).toBe(true); } }); + + it("in a stream with workers, where a flush does not cost as much as a whole job", async () => { + // A job makes room for one long distance match for each ldmMinMatch bytes. For a whole job that is 192 MB here. + const stream = zlib.createZstdCompress({ + params: { + [c.ZSTD_c_nbWorkers]: 1, + [c.ZSTD_c_jobSize]: 64 << 20, + [c.ZSTD_c_enableLongDistanceMatching]: 1, + [c.ZSTD_c_ldmMinMatch]: 4, + }, + }); + const chunks: Buffer[] = []; + stream.on("data", chunk => chunks.push(chunk)); + const { promise, resolve, reject } = Promise.withResolvers(); + stream.on("end", resolve).on("error", reject); + const input = corpus.subarray(0, 200_000); + for (let at = 0; at < input.length; at += 1000) { + stream.write(input.subarray(at, at + 1000)); + stream.flush(); + } + stream.end(); + await promise; + expect(zlib.zstdDecompressSync(Buffer.concat(chunks)).equals(input)).toBe(true); + }); }); From 078890fbce448ec2f60718d2132e88962adaa8cf Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 16:11:26 +0000 Subject: [PATCH 017/199] zstd: price the offset of a match once, not for each of its lengths The optimal parser prices every length of every match it finds. The price is the sum of a part for the offset, which is now worked out once for a match, and a part for the length, which is now worked out when the code of the length changes. The positions that a match reaches are walked as a slice. With targetLength at its maximum and repetitive input, where this loop is nearly all there is, level 19 took twice the time of the C library and now takes two thirds of it. Other input is within 2% of what it was. --- src/zstd/compress/opt.rs | 125 +++++++++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 46 deletions(-) diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index a7a6022a3a45..5f5a5e5a5151 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -283,20 +283,16 @@ fn lit_length_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Resu .wrapping_sub(weight(opt_ptr.lit_length_freq.at(ll_code)?, opt_level))) } -/// `ZSTD_getMatchPrice`: the price of the offset and of the match length of a sequence. `off_base` is as in -/// [`SeqStore::store_seq`]. Below level 2, it favours small offsets, which are faster to decompress. +/// `ZSTD_getMatchPrice`, the price of the offset and of the match length of a sequence, in two parts to add up: all the +/// lengths of a match are priced, with the same offset. +/// +/// This is the part for the offset. `off_base` is as in [`SeqStore::store_seq`]. Below level 2, it favours small +/// offsets, which are faster to decompress. #[inline(always)] -fn get_match_price( - off_base: u32, - match_length: u32, - opt_ptr: &OptState, - opt_level: i32, -) -> Result { +fn offset_price(off_base: u32, opt_ptr: &OptState, opt_level: i32) -> Result { let off_code = highbit32(off_base); - let ml_base = match_length.wrapping_sub(MINMATCH); - if opt_ptr.price_type == OptPrice::Predef { - return Ok(weight(ml_base, opt_level).wrapping_add((16 + off_code) * BITCOST_MULTIPLIER)); + return Ok((16 + off_code) * BITCOST_MULTIPLIER); } let off_code_weight = weight(opt_ptr.off_code_freq.at(off_code as usize)?, opt_level); @@ -308,21 +304,54 @@ fn get_match_price( if opt_level < 2 && off_code >= 20 { price = price.wrapping_add((off_code - 19) * 2 * BITCOST_MULTIPLIER); } - - let ml_code = ml_code(ml_base) as usize; - let ml_code_weight = weight(opt_ptr.match_length_freq.at(ml_code)?, opt_level); - price = price - .wrapping_add(u32::from(ML_BITS.at(ml_code)?) * BITCOST_MULTIPLIER) - .wrapping_add( - opt_ptr - .match_length_sum_base_price - .wrapping_sub(ml_code_weight), - ); - // Matches cost a bit more, to have fewer sequences, which is faster to decompress. Ok(price.wrapping_add(BITCOST_MULTIPLIER / 5)) } +/// The part of `ZSTD_getMatchPrice` for the length. It keeps the price of the last code, which many lengths share. +struct MatchLengthPrice<'a> { + opt_ptr: &'a OptState, + opt_level: i32, + last: Option, +} + +#[derive(Clone, Copy)] +struct CodePrice { + ml_code: u32, + price: u32, +} + +impl<'a> MatchLengthPrice<'a> { + const fn new(opt_ptr: &'a OptState, opt_level: i32) -> Self { + Self { + opt_ptr, + opt_level, + last: None, + } + } + + #[inline(always)] + fn of(&mut self, match_length: u32) -> Result { + let ml_base = match_length.wrapping_sub(MINMATCH); + if self.opt_ptr.price_type == OptPrice::Predef { + return Ok(weight(ml_base, self.opt_level)); + } + + let ml_code = ml_code(ml_base); + if let Some(last) = self.last + && last.ml_code == ml_code + { + return Ok(last.price); + } + let freq = self.opt_ptr.match_length_freq.at(ml_code as usize)?; + let price = (u32::from(ML_BITS.at(ml_code as usize)?) * BITCOST_MULTIPLIER) + .wrapping_add(self.opt_ptr.match_length_sum_base_price) + .wrapping_sub(weight(freq, self.opt_level)); + self.last = Some(CodePrice { ml_code, price }); + Ok(price) + } +} + /// `ZSTD_updateStats` #[inline(always)] fn update_stats( @@ -1387,10 +1416,11 @@ fn parse_block( stretch.litlen = litlen.wrapping_add(pos); pos += 1; } + let mut ml_price = MatchLengthPrice::new(&ms.opt, opt_level); for candidate in matches { + let off_price = offset_price(candidate.off, &ms.opt, opt_level)?; while pos <= candidate.len { - let match_price = - get_match_price(candidate.off, pos, &ms.opt, opt_level)? as i32; + let match_price = off_price.wrapping_add(ml_price.of(pos)?) as i32; let sequence_price = first_price.wrapping_add(match_price); let stretch = opt.at_mut(pos as usize)?; stretch.mlen = pos; @@ -1530,28 +1560,31 @@ fn parse_block( // Set the prices that the matches found here give. let mut start_ml = min_match; + let mut ml_price = MatchLengthPrice::new(&ms.opt, opt_level); for candidate in matches { - let mut mlen = candidate.len; - while mlen >= start_ml { - let pos = cur + mlen; - let price = base_price.wrapping_add(get_match_price( - candidate.off, - mlen, - &ms.opt, - opt_level, - )? - as i32); - - if pos > last_pos || price < opt.at(pos as usize)?.price { - while last_pos < pos { - // Positions that nothing leads to yet. - last_pos += 1; - let stretch = opt.at_mut(last_pos as usize)?; - stretch.price = ZSTD_MAX_PRICE; - // Not 0, which would be the end of a match. - stretch.litlen = 1; - } - let stretch = opt.at_mut(pos as usize)?; + if candidate.len < start_ml { + start_ml = candidate.len + 1; + continue; + } + let off_price = offset_price(candidate.off, &ms.opt, opt_level)?; + let furthest = cur + candidate.len; + // The first position that nothing leads to yet is taken at any price. + let mut new = furthest > last_pos; + while last_pos < furthest { + last_pos += 1; + let stretch = opt.at_mut(last_pos as usize)?; + stretch.price = ZSTD_MAX_PRICE; + // Not 0, which would be the end of a match. + stretch.litlen = 1; + } + + // From the longest length to the shortest. + let reached = opt.sub_mut((cur + start_ml) as usize..=furthest as usize)?; + let lengths = (start_ml..=candidate.len).rev(); + for (stretch, mlen) in reached.iter_mut().rev().zip(lengths) { + let match_price = off_price.wrapping_add(ml_price.of(mlen)?) as i32; + let price = base_price.wrapping_add(match_price); + if new || price < stretch.price { stretch.mlen = mlen; stretch.off = candidate.off; stretch.litlen = 0; @@ -1560,7 +1593,7 @@ fn parse_block( // Shorter lengths are not tried. break; } - mlen -= 1; + new = false; } start_ml = candidate.len + 1; } From b37c7d489db9e29d8577a4a9a4dd660e735f323a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:15:11 +0000 Subject: [PATCH 018/199] zstd: a worker does not clear its tables for a job larger than those before A job is only told its own size, and the model of the C workspace took a job that was a little larger than any before it for a reason to make a new workspace, which clears the tables: 640 MB at level 22, for each flush of a stream whose flushes grow. The C library has the same rule but hardly gets there, as its small jobs all go to the context that had the first job of the frame. A worker's context now counts with blocks of any size from the start. --- src/zstd/compress/cctx.rs | 20 ++++++++++++++++- src/zstd/compress/mt.rs | 2 +- test/js/bun/util/zstd.test.ts | 42 +++++++++++++++++++++++------------ 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 8b21974b9a36..e4c1da17d745 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -783,6 +783,8 @@ pub(crate) struct FrameCompressor { pub(crate) dict_id: u32, pub(crate) dict_content_size: usize, workspace: Workspace, + /// See [`Self::for_worker`]. + room_for_any_block: bool, pub(crate) block_size_max: usize, pub(crate) pledged_src_size_plus_one: u64, pub(crate) consumed_src_size: u64, @@ -817,6 +819,7 @@ impl FrameCompressor { dict_id: 0, dict_content_size: 0, workspace: Workspace::new(), + room_for_any_block: false, block_size_max: 0, pledged_src_size_plus_one: 0, consumed_src_size: 0, @@ -834,6 +837,17 @@ impl FrameCompressor { } } + /// For the jobs of one worker. The size of a job is all it is told about the size of the input, and a workspace that + /// is too small for a job a little larger than those before would be replaced, which clears the tables. The C code + /// hardly gets there, as small jobs all go to the context that had the first job of the frame, which did not know + /// the size. Which context a job goes to is a matter of timing there, so it cannot show in the output. + pub(crate) fn for_worker() -> Self { + Self { + room_for_any_block: true, + ..Self::new() + } + } + /// `ZSTD_resetCCtx_internal` fn reset_internal( &mut self, @@ -882,7 +896,11 @@ impl FrameCompressor { use_row_match_finder: params.use_row_match_finder, buff_in_size: sizes.buff_in_size, buff_out_size: sizes.buff_out_size, - block_size, + block_size: if self.room_for_any_block { + ZSTD_BLOCKSIZE_MAX.min(1 << params.c_params.window_log) + } else { + block_size + }, }) { needs_index_reset = IndexResetPolicy::Reset; } diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 771b482a22e7..07f6a09a8619 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -286,7 +286,7 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - fn worker_main(jobs: &Receiver, cancelled: &AtomicBool) { let mut state = WorkerState { - cctx: FrameCompressor::new(), + cctx: FrameCompressor::for_worker(), seq: Vec::new(), dst: Vec::new(), }; diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 03c32f3ce389..08e315d2765e 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1348,27 +1348,41 @@ describe("the output is that of zstd 1.5.7", () => { } }); - it("in a stream with workers, where a flush does not cost as much as a whole job", async () => { - // A job makes room for one long distance match for each ldmMinMatch bytes. For a whole job that is 192 MB here. - const stream = zlib.createZstdCompress({ - params: { - [c.ZSTD_c_nbWorkers]: 1, - [c.ZSTD_c_jobSize]: 64 << 20, - [c.ZSTD_c_enableLongDistanceMatching]: 1, - [c.ZSTD_c_ldmMinMatch]: 4, - }, - }); + async function flushed(params: Record, sizes: number[]) { + const stream = zlib.createZstdCompress({ params }); const chunks: Buffer[] = []; stream.on("data", chunk => chunks.push(chunk)); const { promise, resolve, reject } = Promise.withResolvers(); stream.on("end", resolve).on("error", reject); - const input = corpus.subarray(0, 200_000); - for (let at = 0; at < input.length; at += 1000) { - stream.write(input.subarray(at, at + 1000)); + let at = 0; + for (const size of sizes) { + stream.write(corpus.subarray(at, at + size)); stream.flush(); + at += size; } stream.end(); await promise; - expect(zlib.zstdDecompressSync(Buffer.concat(chunks)).equals(input)).toBe(true); + expect(zlib.zstdDecompressSync(Buffer.concat(chunks)).equals(corpus.subarray(0, at))).toBe(true); + } + + it("in a stream with workers, where a flush does not cost as much as a whole job", async () => { + // A job makes room for one long distance match for each ldmMinMatch bytes. For a whole job that is 192 MB here. + await flushed( + { + [c.ZSTD_c_nbWorkers]: 1, + [c.ZSTD_c_jobSize]: 64 << 20, + [c.ZSTD_c_enableLongDistanceMatching]: 1, + [c.ZSTD_c_ldmMinMatch]: 4, + }, + Array(200).fill(1000), + ); + }); + + it("in a stream with workers, where a larger job than those before does not clear the tables", async () => { + // The table is 256 MB. + await flushed( + { [c.ZSTD_c_nbWorkers]: 4, [c.ZSTD_c_strategy]: c.ZSTD_fast, [c.ZSTD_c_hashLog]: 26 }, + Array.from({ length: 300 }, (_, i) => i + 1), + ); }); }); From 20bd84845763b659fe66c8eb2c488925544d32a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:22:28 +0000 Subject: [PATCH 019/199] zstd: find where a match ends 16 bytes at a time, and where it starts 8 at a time Where two words differ takes no branch, so most matches are measured without one that is hard to predict: 15 to 20% fewer branch misses from level 1 to 9. --- src/zstd/compress/internal.rs | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 308f45076f37..0ae2205bd39e 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -697,14 +697,15 @@ impl<'a> Near<'a> { #[inline(always)] pub(crate) fn count(self, pos: u32, match_pos: u32) -> u32 { let mut len = 0; + // Where 16 bytes differ takes no branch, and few matches are longer. while let (Some(bytes), Some(match_bytes)) = - (self.read64(pos + len), self.read64(match_pos + len)) + (self.chunk16(pos + len), self.chunk16(match_pos + len)) { - let diff = bytes ^ match_bytes; + let diff = u128::from_le_bytes(*bytes) ^ u128::from_le_bytes(*match_bytes); if diff != 0 { return len + (diff.trailing_zeros() >> 3); } - len += 8; + len += 16; } while let Some(byte) = self.read8(pos + len) && self.read8(match_pos + len) == Some(byte) @@ -723,9 +724,24 @@ impl<'a> Near<'a> { match_pos: u32, match_low: u32, ) -> u32 { + let most = (pos - anchor).min(match_pos - match_low); let mut len = 0; - while pos - len > anchor - && match_pos - len > match_low + while len < most + && let (Some(bytes), Some(match_bytes)) = ( + self.read64((pos - len).wrapping_sub(8)), + self.read64((match_pos - len).wrapping_sub(8)), + ) + { + let diff = bytes ^ match_bytes; + if diff != 0 { + return (len + (diff.leading_zeros() >> 3)).min(most); + } + len += 8; + } + if len >= most { + return most; + } + while len < most && let Some(byte) = self.read8(pos - len - 1) && self.read8(match_pos - len - 1) == Some(byte) { From 7bbf2a0e7e18c3f2469a8c0f075a376ad57d0ecc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:29:00 +0000 Subject: [PATCH 020/199] zstd: fast reads the input once per turn and indexes both positions of a turn at once --- src/zstd/compress/fast.rs | 201 +++++++++++++++------------------- src/zstd/compress/internal.rs | 25 ++++- 2 files changed, 113 insertions(+), 113 deletions(-) diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 2880cd1cd612..c8d25a5ba7ca 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -4,7 +4,7 @@ use super::internal::{ BlockSrc, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, count, count_2segments, hash_of, hash_ptr, - index_overlap_check, offset_to_offbase, table_get, table_put, write_tagged_index, + index_overlap_check, le64_at, offset_to_offbase, table_put, table_swap, write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -374,8 +374,9 @@ impl<'a> NoDictBlock<'a> { /// `ZSTD_compressBlock_fast_noDict_generic` /// -/// Each turn of the inner loop looks at two positions, `ip0` and `ip1`, for a match, and at `ip2` for a repeat. The -/// hash of a position is computed, and the table is read, one position ahead of the comparison. +/// Each turn of the inner loop looks at two positions, `ip0` and the next one, for a match, and at `ip2` for a repeat. +/// The C code computes hashes and reads the table one position ahead. Here that is left to the processor: a turn only +/// takes the position and the 8 bytes there from the one before. fn compress_block_fast_no_dict_generic( ms: &mut MatchState, seq_store: &mut SeqStore, @@ -398,137 +399,113 @@ fn compress_block_fast_no_dict_generic( let mut ip0 = block.first; let mut seq_store = SeqSink::new(seq_store); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; + // The 8 bytes at `ip0`. + let Some(mut bytes0) = near.read64(ip0) else { + return Ok(src.end - src.start); + }; 'start: loop { let mut step = step_size; let mut next_step = ip0 + K_STEP_INCR; - - let mut ip1 = ip0 + 1; let mut ip2 = ip0 + step; - let mut ip3 = ip2 + 1; - if ip3 as usize >= block.ilimit { - break 'start; - } - - // The 8 bytes at `ip0` and at `ip1`. - let mut bytes0 = near.read64(ip0).ok_or(BUG)?; - let mut bytes1 = near.read64(ip1).ok_or(BUG)?; - let mut hash0 = hash_of::(bytes0, hlog); - let mut hash1 = hash_of::(bytes1, hlog); - let mut match_idx = table_get(hash_table, hash0); - let mut current0; - // Ends with the start and the length so far of a repeat match, or with `None` for a match at `match_idx`. - let rep_match = loop { - let bytes2 = near.read64(ip2).ok_or(BUG)?; - let rval = near.read32(ip2.wrapping_sub(rep_offset1)); - - current0 = block.base_index.wrapping_add(ip0); - table_put(hash_table, hash0, current0); - - if (rval == Some(bytes2 as u32)) & (rep_offset1 > 0) { + // Ends with where 4 bytes are the same, at `ip0` and at `match0`. + let (match0, back, offcode, current0) = loop { + // There is a position after `ip2` to look at, and that one is before the last that can be. + let Some(ahead) = near.window::<{ HASH_READ_SIZE + 2 }>(ip2) else { + break 'start; + }; + let bytes2 = le64_at::<0, _>(ahead); + // As far as they are hashed or compared. + let bytes1 = bytes0 >> 8; + let current0 = block.base_index.wrapping_add(ip0); + let idx0 = table_swap(hash_table, hash_of::(bytes0, hlog), current0); + // The next position is indexed whatever is found, as it is before the end of any match. + let idx1 = table_swap(hash_table, hash_of::(bytes1, hlog), current0 + 1); + + if rep_offset1 > 0 && near.read32(ip2.wrapping_sub(rep_offset1)) == Some(bytes2 as u32) + { ip0 = ip2; let match0 = ip0 - rep_offset1; let back = u32::from(near.read8(ip0 - 1) == near.read8(match0.wrapping_sub(1))); - ip0 -= back; - // The hash of ip1 is there, and ip1 is before the end of the match. - table_put(hash_table, hash1, block.base_index.wrapping_add(ip1)); - break Some((match0.wrapping_sub(back), 4 + back)); + break (match0, back, REPCODE1_TO_OFFBASE, current0); } - if candidates.read32(match_idx.wrapping_sub(block.prefix_lowest_index)) + let (idx, current0) = if candidates.read32(idx0.wrapping_sub(block.prefix_lowest_index)) == Some(bytes0 as u32) { - table_put(hash_table, hash1, block.base_index.wrapping_add(ip1)); - break None; - } - - match_idx = table_get(hash_table, hash1); - hash0 = hash1; - bytes0 = bytes1; - bytes1 = bytes2; - hash1 = hash_of::(bytes1, hlog); - - ip0 = ip1; - ip1 = ip2; - ip2 = ip3; - - current0 = block.base_index.wrapping_add(ip0); - table_put(hash_table, hash0, current0); - - if candidates.read32(match_idx.wrapping_sub(block.prefix_lowest_index)) - == Some(bytes0 as u32) + (idx0, current0) + } else if candidates.read32(idx1.wrapping_sub(block.prefix_lowest_index)) + == Some(bytes1 as u32) { - // ip1 is certain to be before the end of the match only if the step is small. + ip0 += 1; + // `ip2` is certain to be before the end of the match only if the step is small. if step <= 4 { - table_put(hash_table, hash1, block.base_index.wrapping_add(ip1)); + table_put( + hash_table, + hash_of::(bytes2, hlog), + block.base_index.wrapping_add(ip2), + ); } - break None; - } - - match_idx = table_get(hash_table, hash1); - hash0 = hash1; - bytes0 = bytes1; - bytes1 = near.read64(ip2).ok_or(BUG)?; - hash1 = hash_of::(bytes1, hlog); - - ip0 = ip1; - ip1 = ip2; - ip2 = ip0 + step; - ip3 = ip1 + step; - - if ip2 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - if ip3 as usize >= block.ilimit { - break 'start; - } - }; - - let (offcode, match0, mut m_length) = match rep_match { - Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), - None => { - let match0 = match_idx - block.base_index; - rep_offset2 = rep_offset1; - rep_offset1 = ip0 - match0; - let back = near.count_backwards(ip0, anchor, match0, block.prefix_lowest); - ip0 -= back; - (offset_to_offbase(rep_offset1), match0 - back, 4 + back) - } + (idx1, current0 + 1) + } else { + ip0 = ip2; + bytes0 = bytes2; + ip2 += step; + if ip2 >= next_step { + step += 1; + next_step += K_STEP_INCR; + } + continue; + }; + + let match0 = idx - block.base_index; + rep_offset2 = rep_offset1; + rep_offset1 = ip0 - match0; + let back = near.count_backwards(ip0, anchor, match0, block.prefix_lowest); + break (match0, back, offset_to_offbase(rep_offset1), current0); }; - m_length += near.count(ip0 + m_length, match0 + m_length); - seq_store.store_seq(ip0 - anchor, near, anchor, offcode, m_length)?; - ip0 += m_length; - anchor = ip0; + let end = ip0 + 4 + near.count(ip0 + 4, match0 + 4); + let match_start = ip0 - back; + seq_store.store_seq( + match_start - anchor, + near, + anchor, + offcode, + end - match_start, + )?; + ip0 = end; + anchor = end; - if ip0 as usize <= block.ilimit { - let to_insert = near.read64(current0 + 2 - block.base_index).ok_or(BUG)?; - table_put(hash_table, hash_of::(to_insert, hlog), current0 + 2); - let before2 = near.read64(ip0 - 2).ok_or(BUG)?; + // `ip0` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { + break 'start; + }; + let to_insert = near.read64(current0 + 2 - block.base_index).ok_or(BUG)?; + table_put(hash_table, hash_of::(to_insert, hlog), current0 + 2); + table_put( + hash_table, + hash_of::(le64_at::<0, _>(around), hlog), + block.base_index.wrapping_add(ip0 - 2), + ); + bytes0 = le64_at::<2, _>(around); + + while rep_offset2 > 0 && near.read32(ip0.wrapping_sub(rep_offset2)) == Some(bytes0 as u32) { + let r_length = near.count(ip0 + 4, (ip0 + 4).wrapping_sub(rep_offset2)) + 4; + core::mem::swap(&mut rep_offset1, &mut rep_offset2); table_put( hash_table, - hash_of::(before2, hlog), - block.base_index.wrapping_add(ip0 - 2), + hash_of::(bytes0, hlog), + block.base_index.wrapping_add(ip0), ); - - while rep_offset2 > 0 && ip0 as usize <= block.ilimit { - let bytes = near.read64(ip0).ok_or(BUG)?; - if near.read32(ip0.wrapping_sub(rep_offset2)) != Some(bytes as u32) { - break; - } - let r_length = near.count(ip0 + 4, (ip0 + 4).wrapping_sub(rep_offset2)) + 4; - core::mem::swap(&mut rep_offset1, &mut rep_offset2); - table_put( - hash_table, - hash_of::(bytes, hlog), - block.base_index.wrapping_add(ip0), - ); - ip0 += r_length; - seq_store.store_seq(0, near, anchor, REPCODE1_TO_OFFBASE, r_length)?; - anchor = ip0; - } + ip0 += r_length; + seq_store.store_seq(0, near, anchor, REPCODE1_TO_OFFBASE, r_length)?; + anchor = ip0; + let Some(bytes) = near.read64(ip0) else { + break 'start; + }; + bytes0 = bytes; } } diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 0ae2205bd39e..cc0e0315d01c 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -684,8 +684,14 @@ impl<'a> Near<'a> { #[inline(always)] fn chunk16(self, pos: u32) -> Option<&'a [u8; 16]> { + self.window(pos) + } + + /// The `N` bytes at `pos`, which can then be read with no further check. + #[inline(always)] + pub(crate) fn window(self, pos: u32) -> Option<&'a [u8; N]> { let pos = pos as usize; - self.0.get(pos..pos + 16)?.first_chunk() + self.0.get(pos..pos + N)?.first_chunk() } #[inline(always)] @@ -767,6 +773,23 @@ pub(crate) fn table_put(table: &mut [u32], hash: usize, index: u32) { } } +/// `table[hash] = index`, likewise. Returns what was there. +#[inline(always)] +pub(crate) fn table_swap(table: &mut [u32], hash: usize, index: u32) -> u32 { + let mask = table.len().wrapping_sub(1); + match table.get_mut(hash & mask) { + Some(cell) => core::mem::replace(cell, index), + None => 0, + } +} + +/// The 8 bytes of `window` from `AT` on. +#[inline(always)] +pub(crate) fn le64_at(window: &[u8; N]) -> u64 { + let bytes = window.get(AT..).and_then(<[u8]>::first_chunk); + bytes.map_or(0, |bytes| u64::from_le_bytes(*bytes)) +} + /// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. #[inline(always)] pub(crate) fn count( From 9ca1713a54ae8f18ee6f90c6b1e5f982432df097 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:47:32 +0000 Subject: [PATCH 021/199] zstd: fast keeps what it needs once per match out of the way of the search --- src/zstd/compress/fast.rs | 360 +++++++++++++++++++++------------- src/zstd/compress/internal.rs | 41 ++-- 2 files changed, 251 insertions(+), 150 deletions(-) diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index c8d25a5ba7ca..6abc85b4b431 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -4,7 +4,8 @@ use super::internal::{ BlockSrc, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, count, count_2segments, hash_of, hash_ptr, - index_overlap_check, le64_at, offset_to_offbase, table_put, table_swap, write_tagged_index, + index_overlap_check, le32_at, le64_at, offset_to_offbase, table_put, table_swap, + write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -372,151 +373,204 @@ impl<'a> NoDictBlock<'a> { } } -/// `ZSTD_compressBlock_fast_noDict_generic` -/// -/// Each turn of the inner loop looks at two positions, `ip0` and the next one, for a match, and at `ip2` for a repeat. -/// The C code computes hashes and reads the table one position ahead. Here that is left to the processor: a turn only -/// takes the position and the 8 bytes there from the one before. -fn compress_block_fast_no_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: u32 = 1 << (K_SEARCH_STRENGTH - 1); +/// Where `fast` and `dfast` put the matches they find in a block, when there is no dictionary and the window is one +/// segment. All of it is read and written once per match, so that the search for the next one has the registers. +pub(super) struct MatchSink<'a, 'b> { + seq_store: SeqSink<'b>, + /// The input from the lowest index a match can be at. Positions are in this. + near: Near<'a>, + /// The same with what is before it, as far as a repeat offset can reach. + older: Near<'a>, + /// How much that is. + older_len: u32, + /// The end of the last match. + anchor: u32, + failed: bool, +} - let Some(block) = NoDictBlock::new(ms, rep, src)? else { - return Ok(src.end - src.start); - }; - let near = block.near; - let candidates = block.candidates; - let mut rep_offset1 = block.offset_1; - let mut rep_offset2 = block.offset_2; - let hlog = ms.c_params.hash_log; - // The least step is 2, as two positions are looked at per turn. - let step_size = least_step(&ms.c_params) + 1; - let mut anchor = block.istart; - let mut ip0 = block.first; - let mut seq_store = SeqSink::new(seq_store); - let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; - // The 8 bytes at `ip0`. - let Some(mut bytes0) = near.read64(ip0) else { - return Ok(src.end - src.start); - }; +impl MatchSink<'_, '_> { + /// The 4 bytes that are `offset` before `pos`. + #[inline(always)] + fn read32_before(&self, pos: u32, offset: u32) -> Option { + match self.near.read32(pos.wrapping_sub(offset)) { + Some(bytes) => Some(bytes), + None => { + core::hint::cold_path(); + self.older + .read32((pos + self.older_len).wrapping_sub(offset)) + } + } + } - 'start: loop { - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - let mut ip2 = ip0 + step; + /// How long what is at `pos` is the same as what is `offset` before. + #[inline(always)] + fn count_before(&self, pos: u32, offset: u32) -> u32 { + if offset <= pos { + self.near.count(pos, pos - offset) + } else { + core::hint::cold_path(); + let pos = pos + self.older_len; + self.older.count(pos, pos.wrapping_sub(offset)) + } + } - // Ends with where 4 bytes are the same, at `ip0` and at `match0`. - let (match0, back, offcode, current0) = loop { - // There is a position after `ip2` to look at, and that one is before the last that can be. - let Some(ahead) = near.window::<{ HASH_READ_SIZE + 2 }>(ip2) else { - break 'start; - }; - let bytes2 = le64_at::<0, _>(ahead); - // As far as they are hashed or compared. - let bytes1 = bytes0 >> 8; - let current0 = block.base_index.wrapping_add(ip0); - let idx0 = table_swap(hash_table, hash_of::(bytes0, hlog), current0); - // The next position is indexed whatever is found, as it is before the end of any match. - let idx1 = table_swap(hash_table, hash_of::(bytes1, hlog), current0 + 1); - - if rep_offset1 > 0 && near.read32(ip2.wrapping_sub(rep_offset1)) == Some(bytes2 as u32) - { - ip0 = ip2; - let match0 = ip0 - rep_offset1; - let back = u32::from(near.read8(ip0 - 1) == near.read8(match0.wrapping_sub(1))); - break (match0, back, REPCODE1_TO_OFFBASE, current0); - } + /// Stores the match from `ip` to `end`, with the literals before it. + #[inline(always)] + fn store(&mut self, ip: u32, end: u32, off_base: u32) { + let anchor = self.anchor; + self.anchor = end; + if (self.seq_store) + .store_seq(ip - anchor, self.near, anchor, off_base, end - ip) + .is_err() + { + core::hint::cold_path(); + self.failed = true; + } + } - let (idx, current0) = if candidates.read32(idx0.wrapping_sub(block.prefix_lowest_index)) - == Some(bytes0 as u32) - { - (idx0, current0) - } else if candidates.read32(idx1.wrapping_sub(block.prefix_lowest_index)) - == Some(bytes1 as u32) - { - ip0 += 1; - // `ip2` is certain to be before the end of the match only if the step is small. - if step <= 4 { - table_put( - hash_table, - hash_of::(bytes2, hlog), - block.base_index.wrapping_add(ip2), - ); + /// Whether the bytes before `ip` and before what is `offset` before it are the same. + #[inline(always)] + fn same_byte_before(&self, ip: u32, offset: u32) -> bool { + let ip = ip + self.older_len; + self.older.read8(ip - 1) == self.older.read8((ip - 1).wrapping_sub(offset)) + } +} + +/// What `fast` looks for matches with, when there is no dictionary and the window is one segment. +struct Finder { + /// The index of the first position. + base_index: u32, + step_size: u32, + hlog: u32, + /// Where to start. + first: u32, + rep_offset1: u32, + rep_offset2: u32, +} + +impl Finder { + /// `ZSTD_compressBlock_fast_noDict_generic` + /// + /// Each turn of the inner loop looks at two positions, that of index `current0` and the next one, for a match, and + /// at `ip2` for a repeat. The C code computes hashes and reads the table one position ahead. Here that is left to + /// the processor: a turn only takes the position and the 8 bytes there, `bytes0`, from the one before. + #[inline(never)] + fn find_matches( + &mut self, + hash_table: &mut [u32], + sink: &mut MatchSink<'_, '_>, + ) { + const K_STEP_INCR: u32 = 1 << (K_SEARCH_STRENGTH - 1); + let Self { + base_index, + step_size, + hlog, + first: mut ip0, + mut rep_offset1, + mut rep_offset2, + } = *self; + let near = sink.near; + if hash_table.is_empty() { + return; + } + let Some(mut bytes0) = near.read64(ip0) else { + return; + }; + + 'block: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + let mut ip2 = ip0 + step; + let mut current0 = base_index.wrapping_add(ip0); + + // Ends with where 4 bytes are the same as earlier. + let (ip, back, off_base) = loop { + // There is a position after `ip2` to look at, and that one is before the last that can be. + let Some(ahead) = near.window::<{ HASH_READ_SIZE + 2 }>(ip2) else { + break 'block; + }; + let idx0 = table_swap(hash_table, hash_of::(bytes0, hlog), current0); + + if sink.read32_before(ip2, rep_offset1) == Some(le32_at::<0, _>(ahead)) + && rep_offset1 > 0 + { + table_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); + let back = u32::from(sink.same_byte_before(ip2, rep_offset1)); + break (ip2, back, REPCODE1_TO_OFFBASE); } - (idx1, current0 + 1) - } else { - ip0 = ip2; - bytes0 = bytes2; - ip2 += step; - if ip2 >= next_step { - step += 1; - next_step += K_STEP_INCR; + let mut match0 = idx0.wrapping_sub(base_index); + if near.read32(match0) == Some(bytes0 as u32) { + table_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); + } else { + // As far as they are hashed or compared. + let bytes1 = bytes0 >> 8; + current0 += 1; + let idx1 = table_swap(hash_table, hash_of::(bytes1, hlog), current0); + match0 = idx1.wrapping_sub(base_index); + if near.read32(match0) != Some(bytes1 as u32) { + current0 = base_index.wrapping_add(ip2); + bytes0 = le64_at::<0, _>(ahead); + ip2 += step; + if ip2 >= next_step { + core::hint::cold_path(); + step += 1; + next_step += K_STEP_INCR; + } + continue; + } + // `ip2` is certain to be before the end of the match only if the step is small. + if step <= 4 { + table_put( + hash_table, + hash_of::(le64_at::<0, _>(ahead), hlog), + base_index.wrapping_add(ip2), + ); + } } - continue; + let ip = current0.wrapping_sub(base_index); + rep_offset2 = rep_offset1; + rep_offset1 = ip - match0; + let back = near.count_backwards(ip, sink.anchor, match0, 0); + break (ip, back, offset_to_offbase(rep_offset1)); }; - let match0 = idx - block.base_index; - rep_offset2 = rep_offset1; - rep_offset1 = ip0 - match0; - let back = near.count_backwards(ip0, anchor, match0, block.prefix_lowest); - break (match0, back, offset_to_offbase(rep_offset1), current0); - }; + ip0 = ip + 4 + sink.count_before(ip + 4, rep_offset1); + sink.store(ip - back, ip0, off_base); - let end = ip0 + 4 + near.count(ip0 + 4, match0 + 4); - let match_start = ip0 - back; - seq_store.store_seq( - match_start - anchor, - near, - anchor, - offcode, - end - match_start, - )?; - ip0 = end; - anchor = end; - - // `ip0` is not past the last position that can be looked at. - let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { - break 'start; - }; - let to_insert = near.read64(current0 + 2 - block.base_index).ok_or(BUG)?; - table_put(hash_table, hash_of::(to_insert, hlog), current0 + 2); - table_put( - hash_table, - hash_of::(le64_at::<0, _>(around), hlog), - block.base_index.wrapping_add(ip0 - 2), - ); - bytes0 = le64_at::<2, _>(around); - - while rep_offset2 > 0 && near.read32(ip0.wrapping_sub(rep_offset2)) == Some(bytes0 as u32) { - let r_length = near.count(ip0 + 4, (ip0 + 4).wrapping_sub(rep_offset2)) + 4; - core::mem::swap(&mut rep_offset1, &mut rep_offset2); + // `ip0` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { + break 'block; + }; + if let Some(to_insert) = near.read64((current0 + 2).wrapping_sub(base_index)) { + table_put(hash_table, hash_of::(to_insert, hlog), current0 + 2); + } table_put( hash_table, - hash_of::(bytes0, hlog), - block.base_index.wrapping_add(ip0), + hash_of::(le64_at::<0, _>(around), hlog), + base_index.wrapping_add(ip0 - 2), ); - ip0 += r_length; - seq_store.store_seq(0, near, anchor, REPCODE1_TO_OFFBASE, r_length)?; - anchor = ip0; - let Some(bytes) = near.read64(ip0) else { - break 'start; - }; - bytes0 = bytes; + bytes0 = le64_at::<2, _>(around); + + while rep_offset2 > 0 && sink.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) { + let end = ip0 + 4 + sink.count_before(ip0 + 4, rep_offset2); + core::mem::swap(&mut rep_offset1, &mut rep_offset2); + table_put( + hash_table, + hash_of::(bytes0, hlog), + base_index.wrapping_add(ip0), + ); + sink.store(ip0, end, REPCODE1_TO_OFFBASE); + ip0 = end; + let Some(bytes) = near.read64(ip0) else { + break 'block; + }; + bytes0 = bytes; + } } - } - restore_rep( - rep, - rep_offset1, - rep_offset2, - block.offset_saved1, - block.offset_saved2, - ); - Ok((near.len() - anchor) as usize) + self.rep_offset1 = rep_offset1; + self.rep_offset2 = rep_offset2; + } } /// `ZSTD_compressBlock_fast` @@ -526,12 +580,46 @@ pub(crate) fn compress_block_fast( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { + let Some(block) = NoDictBlock::new(ms, rep, src)? else { + return Ok(src.end - src.start); + }; + let hlog = ms.c_params.hash_log; + let mut finder = Finder { + base_index: block.prefix_lowest_index, + // The least step is 2, as two positions are looked at per turn. + step_size: least_step(&ms.c_params) + 1, + hlog, + first: block.first - block.prefix_lowest, + rep_offset1: block.offset_1, + rep_offset2: block.offset_2, + }; + let mut sink = MatchSink { + seq_store: SeqSink::new(seq_store), + near: block.candidates, + older: block.near, + older_len: block.prefix_lowest, + anchor: block.istart - block.prefix_lowest, + failed: false, + }; + let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; match ms.c_params.min_match { - 5 => compress_block_fast_no_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_fast_no_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_fast_no_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_fast_no_dict_generic::<4>(ms, seq_store, rep, src), + 5 => finder.find_matches::<5>(hash_table, &mut sink), + 6 => finder.find_matches::<6>(hash_table, &mut sink), + 7 => finder.find_matches::<7>(hash_table, &mut sink), + _ => finder.find_matches::<4>(hash_table, &mut sink), } + if sink.failed { + return Err(BUG); + } + + restore_rep( + rep, + finder.rep_offset1, + finder.rep_offset2, + block.offset_saved1, + block.offset_saved2, + ); + Ok((sink.near.len() - sink.anchor) as usize) } /// `ZSTD_compressBlock_fast_dictMatchState_generic` diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index cc0e0315d01c..7a964942b44c 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -7,6 +7,8 @@ use crate::common::{ ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; use crate::error::{ErrorCode, Result}; +use core::hint::{Locality, prefetch_read}; + use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64, write_le24}; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; @@ -694,6 +696,12 @@ impl<'a> Near<'a> { self.0.get(pos..pos + N)?.first_chunk() } + /// A hint that `pos`, which may be anything, is about to be read. + #[inline(always)] + pub(crate) fn prefetch(self, pos: u32) { + prefetch_read(self.0.as_ptr().wrapping_add(pos as usize), Locality::L1); + } + #[inline(always)] pub(crate) fn read8(self, pos: u32) -> Option { self.0.get(pos as usize).copied() @@ -731,22 +739,20 @@ impl<'a> Near<'a> { match_low: u32, ) -> u32 { let most = (pos - anchor).min(match_pos - match_low); - let mut len = 0; - while len < most - && let (Some(bytes), Some(match_bytes)) = ( - self.read64((pos - len).wrapping_sub(8)), - self.read64((match_pos - len).wrapping_sub(8)), - ) + // Whether there is anything to compare is as good as random: it takes no branch to find out. + if let (Some(bytes), Some(match_bytes)) = ( + self.read64(pos.wrapping_sub(8)), + self.read64(match_pos.wrapping_sub(8)), + ) && bytes != match_bytes { - let diff = bytes ^ match_bytes; - if diff != 0 { - return (len + (diff.leading_zeros() >> 3)).min(most); - } - len += 8; - } - if len >= most { - return most; + return ((bytes ^ match_bytes).leading_zeros() >> 3).min(most); } + self.count_backwards_bytewise(pos, match_pos, most) + } + + #[cold] + fn count_backwards_bytewise(self, pos: u32, match_pos: u32, most: u32) -> u32 { + let mut len = 0; while len < most && let Some(byte) = self.read8(pos - len - 1) && self.read8(match_pos - len - 1) == Some(byte) @@ -783,6 +789,13 @@ pub(crate) fn table_swap(table: &mut [u32], hash: usize, index: u32) -> u32 { } } +/// The 4 bytes of `window` from `AT` on. +#[inline(always)] +pub(crate) fn le32_at(window: &[u8; N]) -> u32 { + let bytes = window.get(AT..).and_then(<[u8]>::first_chunk); + bytes.map_or(0, |bytes| u32::from_le_bytes(*bytes)) +} + /// The 8 bytes of `window` from `AT` on. #[inline(always)] pub(crate) fn le64_at(window: &[u8; N]) -> u64 { From 2fd7bd8daf0e5d1a2df119f82f6a5915d907882a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:49:58 +0000 Subject: [PATCH 022/199] zstd: the codes of a sequence come with its extra bits and are counted as they are made, and a sequence is written at once --- src/zstd/compress/block_entropy.rs | 407 ++++++++++---------- src/zstd/compress/fse.rs | 7 + src/zstd/compress/internal.rs | 14 +- src/zstd/compress/sequences.rs | 575 ++++++++++++++++++++++++----- src/zstd/compress/superblock.rs | 108 +++--- 5 files changed, 743 insertions(+), 368 deletions(-) diff --git a/src/zstd/compress/block_entropy.rs b/src/zstd/compress/block_entropy.rs index 440376512a31..2fc24fad51a6 100644 --- a/src/zstd/compress/block_entropy.rs +++ b/src/zstd/compress/block_entropy.rs @@ -4,47 +4,38 @@ use super::fse::{FseCTable, FseRepeat}; use super::huf::{self, HUF_OPTIMAL_DEPTH_THRESHOLD, HufFlags, HufRepeat}; use super::internal::{ EntropyCTables, EntropyCTablesMetadata, FseCTables, FseCTablesMetadata, HufCTables, - HufCTablesMetadata, LongLengthType, SeqChunk, SeqStore, Strategy, ll_code, min_gain, ml_code, + HufCTablesMetadata, LongLengthType, SeqChunk, SeqStore, Strategy, min_gain, }; use super::literals::compress_literals; use super::params::CCtxParams; use super::sequences::{ - build_ctable, cross_entropy_cost, encode_sequences, fse_bit_cost, select_encoding_type, + self, CodeCounts, SeqCodes, SeqHistograms, build_ctable, cross_entropy_cost, encode_sequences, + fse_bit_cost, select_encoding_type, summarize, }; use super::{hist, huf::HufCTable}; use crate::common::{ DEFAULT_MAX_OFF, HUF_SYMBOLVALUE_MAX, LIT_HUF_LOG, LL_BITS, LL_DEFAULT_NORM, - LL_DEFAULT_NORM_LOG, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MINMATCH, - ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OF_DEFAULT_NORM, - OF_DEFAULT_NORM_LOG, OFF_FSE_LOG, SymbolEncodingType, ZSTD_BLOCKHEADERSIZE, + LL_DEFAULT_NORM_LOG, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MINMATCH, ML_BITS, ML_DEFAULT_NORM, + ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, OFF_FSE_LOG, + SymbolEncodingType, ZSTD_BLOCKHEADERSIZE, }; use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, highbit32, write_le16}; +use crate::util::{SliceExt, write_le16}; /// `ZSTD_literalsCompressionIsDisabled` pub(crate) fn literals_compression_is_disabled(cctx_params: &CCtxParams) -> bool { cctx_params.c_params.strat() == Strategy::Fast && cctx_params.c_params.target_length > 0 } -/// `ZSTD_seqToCodes` -pub(crate) fn seq_to_codes(store: &mut SeqStore, chunk: &SeqChunk) -> Result<()> { +/// `ZSTD_seqToCodes`, and the distributions of the codes. +fn seq_to_codes(store: &mut SeqStore, chunk: &SeqChunk) -> Result { let range = chunk.seq_start..chunk.seq_end; - let sequences = store.sequences.sub(range.clone())?; - let ll_codes = store.ll_code.sub_mut(range.clone())?; - let of_codes = store.of_code.sub_mut(range.clone())?; - let ml_codes = store.ml_code.sub_mut(range)?; - for (((seq, ll), of), ml) in sequences.iter().zip(ll_codes).zip(of_codes).zip(ml_codes) { - *ll = ll_code(u32::from(seq.lit_length)) as u8; - *of = highbit32(seq.off_base) as u8; - *ml = ml_code(u32::from(seq.ml_base)) as u8; - } - let long_length_pos = chunk.seq_start + chunk.long_length_pos as usize; - match chunk.long_length_type { - LongLengthType::None => {} - LongLengthType::LiteralLength => store.ll_code.put(long_length_pos, MAX_LL as u8)?, - LongLengthType::MatchLength => store.ml_code.put(long_length_pos, MAX_ML as u8)?, - } - Ok(()) + Ok(sequences::seq_to_codes( + store.codes.sub_mut(range.clone())?, + store.sequences.sub(range)?, + chunk.long_length_type, + chunk.long_length_pos as usize, + )) } /// `ZSTD_symbolEncodingTypeStats_t` @@ -58,142 +49,160 @@ struct SymbolEncodingTypeStats { last_count_size: usize, } -/// `ZSTD_buildSequencesStatistics`: makes the tables for the sequences of `chunk`, of which there is at least one, and -/// writes their headers to `dst`. +/// What is particular to lengths or to offsets in how a table is made. +struct CodeKind<'a> { + fse_log: u32, + default_norm: &'a [i16], + default_norm_log: u32, + default_max: u32, +} + +/// The table of one kind of code in the previous block. +#[derive(Clone, Copy)] +struct PrevTable<'a> { + table: &'a FseCTable, + repeat_mode: FseRepeat, +} + +/// The header of a table. +struct TableHeader { + ty: SymbolEncodingType, + size: usize, +} + +/// Chooses how to encode `nb_seq` codes, the last of which is `last_code` and of which `count` is the distribution, +/// makes the table and writes its header. +fn build_table( + kind: &CodeKind<'_>, + prev: PrevTable<'_>, + next_table: &mut FseCTable, + next_repeat_mode: &mut FseRepeat, + count: &mut CodeCounts, + nb_seq: usize, + last_code: u8, + dst: &mut [u8], + strategy: Strategy, +) -> Result { + let summary = summarize(count); + let max = summary.max_symbol_value; + *next_repeat_mode = prev.repeat_mode; + let ty = select_encoding_type( + next_repeat_mode, + count, + max, + summary.largest, + nb_seq, + kind.fse_log, + prev.table, + kind.default_norm, + kind.default_norm_log, + // The predefined table has no code for the largest offsets. + max <= kind.default_max, + strategy, + )?; + let size = build_ctable( + dst, + next_table, + kind.fse_log, + ty, + count, + max, + nb_seq, + last_code, + kind.default_norm, + kind.default_norm_log, + kind.default_max, + prev.table, + )?; + Ok(TableHeader { ty, size }) +} + +/// `ZSTD_buildSequencesStatistics`: makes the tables for `nb_seq` sequences, which is not 0, the last of which has the +/// codes `last_codes`, and writes their headers to `dst`. fn build_sequences_statistics( - store: &mut SeqStore, - chunk: &SeqChunk, + histograms: &mut SeqHistograms, + nb_seq: usize, + last_codes: SeqCodes, prev_entropy: &FseCTables, next_entropy: &mut FseCTables, dst: &mut [u8], strategy: Strategy, ) -> Result { - let mut count = [0u32; MAX_SEQ as usize + 1]; - let nb_seq = chunk.nb_seq(); - let range = chunk.seq_start..chunk.seq_end; let mut op = 0usize; let mut last_count_size = 0; - - seq_to_codes(store, chunk)?; - let of_code_table = store.of_code.sub(range.clone())?; - let ll_code_table = store.ll_code.sub(range.clone())?; - let ml_code_table = store.ml_code.sub(range)?; - - let ll_type = { - let mut max = MAX_LL; - let most_frequent = hist::count_fast_wksp(&mut count, &mut max, ll_code_table)?; - next_entropy.litlength_repeat_mode = prev_entropy.litlength_repeat_mode; - let ll_type = select_encoding_type( - &mut next_entropy.litlength_repeat_mode, - &count, - max, - most_frequent, - nb_seq, - LL_FSE_LOG, - &prev_entropy.litlength_c_table, - &LL_DEFAULT_NORM, - LL_DEFAULT_NORM_LOG, - true, - strategy, - )?; - let count_size = build_ctable( - dst.sub_mut(op..)?, - &mut next_entropy.litlength_c_table, - LL_FSE_LOG, - ll_type, - &mut count, - max, - ll_code_table, - &LL_DEFAULT_NORM, - LL_DEFAULT_NORM_LOG, - MAX_LL, - &prev_entropy.litlength_c_table, - )?; - if ll_type == SymbolEncodingType::Compressed { - last_count_size = count_size; + let mut written = |header: &TableHeader| { + if header.ty == SymbolEncodingType::Compressed { + last_count_size = header.size; } - op += count_size; - ll_type - }; - let off_type = { - let mut max = MAX_OFF; - let most_frequent = hist::count_fast_wksp(&mut count, &mut max, of_code_table)?; - // The predefined table has no code for the largest offsets. - let default_allowed = max <= DEFAULT_MAX_OFF; - next_entropy.offcode_repeat_mode = prev_entropy.offcode_repeat_mode; - let off_type = select_encoding_type( - &mut next_entropy.offcode_repeat_mode, - &count, - max, - most_frequent, - nb_seq, - OFF_FSE_LOG, - &prev_entropy.offcode_c_table, - &OF_DEFAULT_NORM, - OF_DEFAULT_NORM_LOG, - default_allowed, - strategy, - )?; - let count_size = build_ctable( - dst.sub_mut(op..)?, - &mut next_entropy.offcode_c_table, - OFF_FSE_LOG, - off_type, - &mut count, - max, - of_code_table, - &OF_DEFAULT_NORM, - OF_DEFAULT_NORM_LOG, - DEFAULT_MAX_OFF, - &prev_entropy.offcode_c_table, - )?; - if off_type == SymbolEncodingType::Compressed { - last_count_size = count_size; - } - op += count_size; - off_type - }; - let ml_type = { - let mut max = MAX_ML; - let most_frequent = hist::count_fast_wksp(&mut count, &mut max, ml_code_table)?; - next_entropy.matchlength_repeat_mode = prev_entropy.matchlength_repeat_mode; - let ml_type = select_encoding_type( - &mut next_entropy.matchlength_repeat_mode, - &count, - max, - most_frequent, - nb_seq, - ML_FSE_LOG, - &prev_entropy.matchlength_c_table, - &ML_DEFAULT_NORM, - ML_DEFAULT_NORM_LOG, - true, - strategy, - )?; - let count_size = build_ctable( - dst.sub_mut(op..)?, - &mut next_entropy.matchlength_c_table, - ML_FSE_LOG, - ml_type, - &mut count, - max, - ml_code_table, - &ML_DEFAULT_NORM, - ML_DEFAULT_NORM_LOG, - MAX_ML, - &prev_entropy.matchlength_c_table, - )?; - if ml_type == SymbolEncodingType::Compressed { - last_count_size = count_size; - } - op += count_size; - ml_type + op += header.size; + op }; + + let ll = build_table( + &CodeKind { + fse_log: LL_FSE_LOG, + default_norm: &LL_DEFAULT_NORM, + default_norm_log: LL_DEFAULT_NORM_LOG, + default_max: MAX_LL, + }, + PrevTable { + table: &prev_entropy.litlength_c_table, + repeat_mode: prev_entropy.litlength_repeat_mode, + }, + &mut next_entropy.litlength_c_table, + &mut next_entropy.litlength_repeat_mode, + &mut histograms.ll, + nb_seq, + last_codes.ll as u8, + dst, + strategy, + )?; + let op_of = written(&ll); + let of = build_table( + &CodeKind { + fse_log: OFF_FSE_LOG, + default_norm: &OF_DEFAULT_NORM, + default_norm_log: OF_DEFAULT_NORM_LOG, + default_max: DEFAULT_MAX_OFF, + }, + PrevTable { + table: &prev_entropy.offcode_c_table, + repeat_mode: prev_entropy.offcode_repeat_mode, + }, + &mut next_entropy.offcode_c_table, + &mut next_entropy.offcode_repeat_mode, + &mut histograms.of, + nb_seq, + last_codes.of as u8, + dst.sub_mut(op_of..)?, + strategy, + )?; + let op_ml = written(&of); + let ml = build_table( + &CodeKind { + fse_log: ML_FSE_LOG, + default_norm: &ML_DEFAULT_NORM, + default_norm_log: ML_DEFAULT_NORM_LOG, + default_max: MAX_ML, + }, + PrevTable { + table: &prev_entropy.matchlength_c_table, + repeat_mode: prev_entropy.matchlength_repeat_mode, + }, + &mut next_entropy.matchlength_c_table, + &mut next_entropy.matchlength_repeat_mode, + &mut histograms.ml, + nb_seq, + last_codes.ml as u8, + dst.sub_mut(op_ml..)?, + strategy, + )?; + let size = written(&ml); Ok(SymbolEncodingTypeStats { - ll_type, - off_type, - ml_type, - size: op, + ll_type: ll.ty, + off_type: of.ty, + ml_type: ml.ty, + size, last_count_size, }) } @@ -254,9 +263,11 @@ fn entropy_compress_seq_store_internal( let last_count_size = { let seq_head = op; op += 1; + let mut histograms = seq_to_codes(store, chunk)?; let stats = build_sequences_statistics( - store, - chunk, + &mut histograms, + nb_seq, + store.codes.at(chunk.seq_end - 1)?, &prev_entropy.fse, &mut next_entropy.fse, dst.sub_mut(op..)?, @@ -275,12 +286,8 @@ fn entropy_compress_seq_store_internal( let range = chunk.seq_start..chunk.seq_end; let bitstream_size = encode_sequences( dst.sub_mut(op..)?, - &next_entropy.fse.matchlength_c_table, - store.ml_code.sub(range.clone())?, - &next_entropy.fse.offcode_c_table, - store.of_code.sub(range.clone())?, - &next_entropy.fse.litlength_c_table, - store.ll_code.sub(range.clone())?, + &next_entropy.fse, + store.codes.sub(range.clone())?, store.sequences.sub(range)?, )?; op += bitstream_size; @@ -417,7 +424,7 @@ fn build_block_entropy_stats_literals( } /// `ZSTD_buildBlockEntropyStats_sequences`: makes the tables for the sequences of `chunk`, and their description in -/// `fse_metadata`. Returns the size of the description. +/// `fse_metadata`. Returns the distributions of the codes. fn build_block_entropy_stats_sequences( store: &mut SeqStore, chunk: &SeqChunk, @@ -425,11 +432,13 @@ fn build_block_entropy_stats_sequences( next_entropy: &mut FseCTables, cctx_params: &CCtxParams, fse_metadata: &mut FseCTablesMetadata, -) -> Result { +) -> Result { + let mut histograms = seq_to_codes(store, chunk)?; let stats = if chunk.nb_seq() != 0 { build_sequences_statistics( - store, - chunk, + &mut histograms, + chunk.nb_seq(), + store.codes.at(chunk.seq_end - 1)?, prev_entropy, next_entropy, &mut fse_metadata.fse_tables_buffer, @@ -451,11 +460,12 @@ fn build_block_entropy_stats_sequences( fse_metadata.of_type = stats.off_type; fse_metadata.ml_type = stats.ml_type; fse_metadata.last_count_size = stats.last_count_size; - Ok(stats.size) + fse_metadata.fse_tables_size = stats.size; + Ok(histograms) } /// `ZSTD_buildBlockEntropyStats`: makes the tables for `chunk` in `next_entropy`, and what it takes to write them in -/// `entropy_metadata`. +/// `entropy_metadata`. Returns the distributions of the codes of the sequences. pub(crate) fn build_block_entropy_stats( store: &mut SeqStore, chunk: &SeqChunk, @@ -463,7 +473,7 @@ pub(crate) fn build_block_entropy_stats( next_entropy: &mut EntropyCTables, cctx_params: &CCtxParams, entropy_metadata: &mut EntropyCTablesMetadata, -) -> Result<()> { +) -> Result { let huf_flags = HufFlags { optimal_depth: cctx_params.c_params.strat() >= HUF_OPTIMAL_DEPTH_THRESHOLD, ..HufFlags::default() @@ -476,15 +486,14 @@ pub(crate) fn build_block_entropy_stats( literals_compression_is_disabled(cctx_params), huf_flags, )?; - entropy_metadata.fse_metadata.fse_tables_size = build_block_entropy_stats_sequences( + build_block_entropy_stats_sequences( store, chunk, &prev_entropy.fse, &mut next_entropy.fse, cctx_params, &mut entropy_metadata.fse_metadata, - )?; - Ok(()) + ) } /// `ZSTD_estimateBlockSize_literal` @@ -522,56 +531,56 @@ fn estimate_block_size_literal( } } -/// `ZSTD_estimateBlockSize_symbolType` +/// `ZSTD_estimateBlockSize_symbolType`: `count` is the distribution of `nb_seq` codes. fn estimate_block_size_symbol_type( ty: SymbolEncodingType, - code_table: &[u8], - max_code: u32, + count: &CodeCounts, + nb_seq: usize, fse_c_table: &FseCTable, additional_bits: Option<&[u8]>, default_norm: &[i16], default_norm_log: u32, ) -> Result { - let mut count = [0u32; MAX_SEQ as usize + 1]; - let mut max = max_code; - hist::count_fast_wksp(&mut count, &mut max, code_table)?; + let max = summarize(count).max_symbol_value; let estimate_in_bits = match ty { - SymbolEncodingType::Basic => { - cross_entropy_cost(default_norm, default_norm_log, &count, max) - } + SymbolEncodingType::Basic => cross_entropy_cost(default_norm, default_norm_log, count, max), SymbolEncodingType::Rle => Ok(0), SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { - fse_bit_cost(fse_c_table, &count, max) + fse_bit_cost(fse_c_table, count, max) } }; - let Ok(mut estimate_in_bits) = estimate_in_bits else { - return Ok(code_table.len() * 10); + let Ok(estimate_in_bits) = estimate_in_bits else { + return Ok(nb_seq * 10); }; - for &code in code_table { - estimate_in_bits += usize::from(match additional_bits { - Some(bits) => bits.at(usize::from(code))?, - None => code, - }); + Ok((estimate_in_bits + additional_bits_of(count, additional_bits)) >> 3) +} + +/// How many bits follow the codes counted in `count`: as many as the code itself says, or as `additional_bits` does. +pub(crate) fn additional_bits_of(count: &CodeCounts, additional_bits: Option<&[u8]>) -> usize { + match additional_bits { + Some(bits) => (count.iter().zip(bits)) + .map(|(&count, &bits)| count as usize * usize::from(bits)) + .sum(), + None => (count.iter().enumerate()) + .map(|(code, &count)| count as usize * code) + .sum(), } - Ok(estimate_in_bits >> 3) } /// `ZSTD_estimateBlockSize_sequences` fn estimate_block_size_sequences( - of_code_table: &[u8], - ll_code_table: &[u8], - ml_code_table: &[u8], + histograms: &SeqHistograms, + nb_seq: usize, fse_tables: &FseCTables, fse_metadata: &FseCTablesMetadata, write_entropy: bool, ) -> Result { - let nb_seq = of_code_table.len(); let sequences_section_header_size = 1 + 1 + usize::from(nb_seq >= 128) + usize::from(nb_seq >= LONGNBSEQ); let mut c_seq_size_estimate = estimate_block_size_symbol_type( fse_metadata.of_type, - of_code_table, - MAX_OFF, + &histograms.of, + nb_seq, &fse_tables.offcode_c_table, None, &OF_DEFAULT_NORM, @@ -579,8 +588,8 @@ fn estimate_block_size_sequences( )?; c_seq_size_estimate += estimate_block_size_symbol_type( fse_metadata.ll_type, - ll_code_table, - MAX_LL, + &histograms.ll, + nb_seq, &fse_tables.litlength_c_table, Some(&LL_BITS), &LL_DEFAULT_NORM, @@ -588,8 +597,8 @@ fn estimate_block_size_sequences( )?; c_seq_size_estimate += estimate_block_size_symbol_type( fse_metadata.ml_type, - ml_code_table, - MAX_ML, + &histograms.ml, + nb_seq, &fse_tables.matchlength_c_table, Some(&ML_BITS), &ML_DEFAULT_NORM, @@ -611,7 +620,7 @@ pub(crate) fn build_entropy_statistics_and_estimate_sub_block_size( cctx_params: &CCtxParams, entropy_metadata: &mut EntropyCTablesMetadata, ) -> Result { - build_block_entropy_stats( + let histograms = build_block_entropy_stats( store, chunk, prev_entropy, @@ -619,7 +628,6 @@ pub(crate) fn build_entropy_statistics_and_estimate_sub_block_size( cctx_params, entropy_metadata, )?; - let range = chunk.seq_start..chunk.seq_end; let literals_size = estimate_block_size_literal( store.lit_buffer.sub(chunk.lit_start..chunk.lit_end)?, &next_entropy.huf, @@ -627,9 +635,8 @@ pub(crate) fn build_entropy_statistics_and_estimate_sub_block_size( entropy_metadata.huf_metadata.h_type == SymbolEncodingType::Compressed, ); let seq_size = estimate_block_size_sequences( - store.of_code.sub(range.clone())?, - store.ll_code.sub(range.clone())?, - store.ml_code.sub(range)?, + &histograms, + chunk.nb_seq(), &next_entropy.fse, &entropy_metadata.fse_metadata, true, diff --git a/src/zstd/compress/fse.rs b/src/zstd/compress/fse.rs index 999f24140c31..87094b119d80 100644 --- a/src/zstd/compress/fse.rs +++ b/src/zstd/compress/fse.rs @@ -174,6 +174,13 @@ impl FseCTable { .put(usize::from(symbol_value), FseSymbolTransform::default()) } + /// `FSE_initCState2`: the state that it costs the fewest bits to start from, given that the first symbol is `symbol`, + /// which the table has a code for. + #[inline] + pub(crate) fn first_state(&self, symbol: u8) -> u32 { + FseCState::new2(self, u32::from(symbol)).value as u32 + } + /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log /// if the table has no code for it. #[inline(always)] diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 0ae2205bd39e..ebf0ab1c2072 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -2,6 +2,7 @@ use super::fse::{FseCTable, FseRepeat}; use super::huf::{HufCTable, HufRepeat}; +use super::sequences::SeqCodes; use crate::common::{ BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, @@ -241,9 +242,8 @@ pub(crate) struct SeqStore { pub(crate) lit_buffer: Vec, /// How much of `lit_buffer` is literals. pub(crate) lit_len: usize, - pub(crate) ll_code: Vec, - pub(crate) ml_code: Vec, - pub(crate) of_code: Vec, + /// Room for the codes of `max_nb_seq` sequences. + pub(crate) codes: Vec, pub(crate) max_nb_seq: usize, pub(crate) max_nb_lit: usize, pub(crate) long_length_type: LongLengthType, @@ -319,9 +319,7 @@ impl SeqStore { sequences: Vec::new(), lit_buffer: Vec::new(), lit_len: 0, - ll_code: Vec::new(), - ml_code: Vec::new(), - of_code: Vec::new(), + codes: Vec::new(), max_nb_seq: 0, max_nb_lit: 0, long_length_type: LongLengthType::None, @@ -340,9 +338,7 @@ impl SeqStore { } grow(&mut self.sequences, max_nb_seq)?; grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; - grow(&mut self.ll_code, max_nb_seq)?; - grow(&mut self.ml_code, max_nb_seq)?; - grow(&mut self.of_code, max_nb_seq)?; + grow(&mut self.codes, max_nb_seq)?; self.max_nb_seq = max_nb_seq; self.max_nb_lit = max_nb_lit; self.reset(); diff --git a/src/zstd/compress/sequences.rs b/src/zstd/compress/sequences.rs index 2f05b7d28137..dacd66203f01 100644 --- a/src/zstd/compress/sequences.rs +++ b/src/zstd/compress/sequences.rs @@ -1,13 +1,13 @@ //! `zstd_compress_sequences.c`: how the sequences of a block are encoded. -use super::fse::{self, FSE_NCOUNTBOUND, FseCState, FseCTable, FseRepeat}; -use super::internal::{SeqDef, Strategy}; -use crate::common::bitstream::BitCStream; +use super::fse::{self, FSE_NCOUNTBOUND, FseCTable, FseRepeat}; +use super::internal::{FseCTables, LongLengthType, SeqDef, Strategy}; use crate::common::{ - LL_BITS, LL_FSE_LOG, MAX_SEQ, ML_BITS, ML_FSE_LOG, OFF_FSE_LOG, SymbolEncodingType, + LL_BITS, LL_FSE_LOG, MAX_LL, MAX_ML, MAX_SEQ, ML_BITS, ML_FSE_LOG, OFF_FSE_LOG, + SymbolEncodingType, }; use crate::error::{ErrorCode, Result}; -use crate::util::SliceExt; +use crate::util::{SliceExt, highbit32}; /// `-log2(x / 256)` in units of 1/256 of a bit, for `x` from 1 to 255. The entry for 0 is not used. #[rustfmt::skip] @@ -173,8 +173,8 @@ pub(crate) fn select_encoding_type( Ok(SymbolEncodingType::Compressed) } -/// `ZSTD_buildCTable`: makes the table for the codes `code_table`, of which `count` is the distribution, and writes -/// what the decoder needs to make the same one. Returns how many bytes that takes. +/// `ZSTD_buildCTable`: makes the table for `nb_seq` codes, the last of which is `last_code` and of which `count` is the +/// distribution, and writes what the decoder needs to make the same one. Returns how many bytes that takes. pub(crate) fn build_ctable( dst: &mut [u8], next_ctable: &mut FseCTable, @@ -182,7 +182,8 @@ pub(crate) fn build_ctable( ty: SymbolEncodingType, count: &mut [u32], max: u32, - code_table: &[u8], + nb_seq: usize, + last_code: u8, default_norm: &[i16], default_norm_log: u32, default_max: u32, @@ -194,7 +195,7 @@ pub(crate) fn build_ctable( let Some(first) = dst.first_mut() else { return Err(ErrorCode::DstSizeTooSmall); }; - *first = code_table.at(0)?; + *first = max as u8; Ok(1) } SymbolEncodingType::Repeat => { @@ -207,23 +208,22 @@ pub(crate) fn build_ctable( } SymbolEncodingType::Compressed => { let mut norm = [0i16; MAX_SEQ as usize + 1]; - let nb_seq = code_table.len(); - let mut nb_seq_1 = nb_seq; let table_log = fse::optimal_table_log(fse_log, nb_seq, max); // The last code is the first state, which takes no transition. - let last_count = count.at_mut(usize::from(code_table.at(nb_seq.wrapping_sub(1))?))?; - if *last_count > 1 { - *last_count -= 1; - nb_seq_1 -= 1; - } - fse::normalize_count( + let last_count = count.at_mut(usize::from(last_code))?; + let without_last = usize::from(*last_count > 1); + *last_count -= without_last as u32; + let nb_seq_1 = nb_seq - without_last; + let normalized = fse::normalize_count( &mut norm, table_log, count, nb_seq_1, max, use_low_prob_count(nb_seq_1), - )?; + ); + *count.at_mut(usize::from(last_code))? += without_last as u32; + normalized?; let ncount_size = fse::write_ncount(dst, &norm, max, table_log)?; next_ctable.build(&norm, max, table_log)?; Ok(ncount_size) @@ -231,93 +231,482 @@ pub(crate) fn build_ctable( } } +/// The code of a length or of an offset. As the index into a table of 64, it takes no check. +#[rustfmt::skip] +#[derive(Clone, Copy, Default)] +#[repr(u8)] +pub(crate) enum Code { + #[default] + C0, C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, C12, C13, C14, C15, + C16, C17, C18, C19, C20, C21, C22, C23, C24, C25, C26, C27, C28, C29, C30, C31, + C32, C33, C34, C35, C36, C37, C38, C39, C40, C41, C42, C43, C44, C45, C46, C47, + C48, C49, C50, C51, C52, C53, C54, C55, C56, C57, C58, C59, C60, C61, C62, C63, +} +use Code::*; + +impl Code { + #[rustfmt::skip] + const ALL: [Self; 64] = [ + C0, C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, C12, C13, C14, C15, + C16, C17, C18, C19, C20, C21, C22, C23, C24, C25, C26, C27, C28, C29, C30, C31, + C32, C33, C34, C35, C36, C37, C38, C39, C40, C41, C42, C43, C44, C45, C46, C47, + C48, C49, C50, C51, C52, C53, C54, C55, C56, C57, C58, C59, C60, C61, C62, C63, + ]; + + /// `code` is less than 64. + #[inline(always)] + fn new(code: u32) -> Self { + (Self::ALL.get(code as usize & 63).copied()).unwrap_or_default() + } +} + +/// The code of a length, and the bits that follow the code in the stream. +#[derive(Clone, Copy)] +struct LengthCode { + code: Code, + nb_bits: u32, + bits: u32, +} + +/// [`LengthCode`] for the lengths below `N`. +struct LengthCodes { + code: [Code; N], + nb_bits: [u8; N], + bits: [u8; N], +} + +impl LengthCodes { + #[inline(always)] + fn get(&self, length: u16) -> Option { + let index = usize::from(length); + Some(LengthCode { + code: *self.code.get(index)?, + nb_bits: u32::from(*self.nb_bits.get(index)?), + bits: u32::from(*self.bits.get(index)?), + }) + } + + /// From `N` on, a code is for the lengths of one binary logarithm, to which `delta` is added. + fn get_any(&self, length: u16, delta: u32) -> LengthCode { + self.get(length).unwrap_or_else(|| { + let log = highbit32(u32::from(length)); + LengthCode { + code: Code::new(log + delta), + nb_bits: log, + bits: u32::from(length) & !(1 << log), + } + }) + } +} + +/// `ZSTD_LLcode` +#[rustfmt::skip] +static LL_CODES: LengthCodes<64> = LengthCodes { + code: [ + C0, C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, C12, C13, C14, C15, + C16, C16, C17, C17, C18, C18, C19, C19, C20, C20, C20, C20, C21, C21, C21, C21, + C22, C22, C22, C22, C22, C22, C22, C22, C23, C23, C23, C23, C23, C23, C23, C23, + C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, + ], + nb_bits: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + ], + bits: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + ], +}; + +/// `ZSTD_MLcode` +#[rustfmt::skip] +static ML_CODES: LengthCodes<128> = LengthCodes { + code: [ + C0, C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, C12, C13, C14, C15, + C16, C17, C18, C19, C20, C21, C22, C23, C24, C25, C26, C27, C28, C29, C30, C31, + C32, C32, C33, C33, C34, C34, C35, C35, C36, C36, C36, C36, C37, C37, C37, C37, + C38, C38, C38, C38, C38, C38, C38, C38, C39, C39, C39, C39, C39, C39, C39, C39, + C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, + C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, + C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, + C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, + ], + nb_bits: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + ], + bits: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ], +}; + +const LL_DELTA_CODE: u32 = 19; +const ML_DELTA_CODE: u32 = 36; + +/// The codes of a sequence, and the bits that follow them. +#[derive(Clone, Copy, Default)] +pub(crate) struct SeqCodes { + /// The bits for the literal length, above them those for the match length, above them those for the offset, and + /// above them a bit set. If there are too many for the sequence to be written at once, only the bit set. + extra_bits: u32, + pub(crate) ll: Code, + pub(crate) of: Code, + pub(crate) ml: Code, + /// How many `extra_bits` are, or [`Self::TOO_MANY`]. + nb_extra_bits: u8, +} + +impl SeqCodes { + /// With more than this on top of the bits that leave the three states, a sequence may not fit the container. + const MAX_EXTRA_BITS: u32 = 64 - 7 - (LL_FSE_LOG + ML_FSE_LOG + OFF_FSE_LOG) - 1; + const TOO_MANY: u8 = 0x80; + + /// `None` if there are too many extra bits. + #[inline(always)] + fn new(ll: LengthCode, ml: LengthCode, off_base: u32) -> Option { + let of = highbit32(off_base); + let nb_extra_bits = ll.nb_bits + ml.nb_bits + of; + if nb_extra_bits > Self::MAX_EXTRA_BITS { + return None; + } + // The highest bit of `off_base`, which the code stands for, is the bit set above the others. + let above_ll = (off_base << (ml.nb_bits & 31)) | ml.bits; + Some(Self { + extra_bits: (above_ll << (ll.nb_bits & 31)) | ll.bits, + ll: ll.code, + of: Code::new(of), + ml: ml.code, + nb_extra_bits: nb_extra_bits as u8, + }) + } + + fn new_any(ll: LengthCode, ml: LengthCode, off_base: u32) -> Self { + Self::new(ll, ml, off_base).unwrap_or_else(|| Self { + extra_bits: 1, + ll: ll.code, + of: Code::new(highbit32(off_base)), + ml: ml.code, + nb_extra_bits: Self::TOO_MANY, + }) + } + + /// `*self = codes`, without putting the fields together first. + #[inline(always)] + fn set(&mut self, codes: Self) { + self.extra_bits = codes.extra_bits; + self.ll = codes.ll; + self.of = codes.of; + self.ml = codes.ml; + self.nb_extra_bits = codes.nb_extra_bits; + } +} + +/// How many times each code occurs. +pub(crate) type CodeCounts = [u32; 64]; + #[inline(always)] -fn bits_of(table: &[u8; 64], code: u8) -> u32 { - u32::from(table.get(usize::from(code & 63)).copied().unwrap_or(0)) +fn count_code(counts: &mut CodeCounts, code: Code, delta: u32) { + if let Some(count) = counts.get_mut(code as usize) { + *count = count.wrapping_add(delta); + } } -/// `table`, long enough for a masked code to be known to be in bounds. -const fn padded(table: &[u8; N]) -> [u8; 64] { - let mut padded = [0; 64]; - let (head, _) = padded.split_at_mut(N); - head.copy_from_slice(table); - padded +/// What `HIST_count` tells about counts. +#[derive(Clone, Copy)] +pub(crate) struct CountsSummary { + pub(crate) max_symbol_value: u32, + pub(crate) largest: usize, } -static LL_BITS_PADDED: [u8; 64] = padded(&LL_BITS); -static ML_BITS_PADDED: [u8; 64] = padded(&ML_BITS); -/// `ZSTD_encodeSequences`: the four slices are of the same length, which is not 0. -pub(crate) fn encode_sequences( - dst: &mut [u8], - ctable_match_length: &FseCTable, - ml_code_table: &[u8], - ctable_offset_bits: &FseCTable, - of_code_table: &[u8], - ctable_lit_length: &FseCTable, - ll_code_table: &[u8], +pub(crate) fn summarize(counts: &CodeCounts) -> CountsSummary { + CountsSummary { + max_symbol_value: counts.iter().rposition(|&c| c != 0).unwrap_or(0) as u32, + largest: counts.iter().copied().max().unwrap_or(0) as usize, + } +} + +/// The distributions of the codes of some sequences. +pub(crate) struct SeqHistograms { + pub(crate) ll: CodeCounts, + pub(crate) of: CodeCounts, + pub(crate) ml: CodeCounts, +} + +impl SeqHistograms { + #[inline(always)] + fn add(&mut self, codes: SeqCodes) { + count_code(&mut self.ll, codes.ll, 1); + count_code(&mut self.of, codes.of, 1); + count_code(&mut self.ml, codes.ml, 1); + } + + /// For a sequence that takes more than the tables of [`seq_to_codes`]. + #[cold] + #[inline(never)] + fn add_unusual(&mut self, seq: SeqDef) -> SeqCodes { + let codes = SeqCodes::new_any( + LL_CODES.get_any(seq.lit_length, LL_DELTA_CODE), + ML_CODES.get_any(seq.ml_base, ML_DELTA_CODE), + seq.off_base, + ); + self.add(codes); + codes + } +} + +/// `ZSTD_seqToCodes`, and the distributions of the codes. `codes` is as long as `sequences`. One length of the sequence +/// `long_length_pos`, if there is such a sequence, is 0x10000 more than it says, unless `long_length_type` is `None`. +pub(crate) fn seq_to_codes( + codes: &mut [SeqCodes], sequences: &[SeqDef], -) -> Result { - let mut block_stream = BitCStream::new(dst).map_err(|_| ErrorCode::DstSizeTooSmall)?; - let ( - Some((&ml_last, ml_codes)), - Some((&of_last, of_codes)), - Some((&ll_last, ll_codes)), - Some((seq_last, sequences)), - ) = ( - ml_code_table.split_last(), - of_code_table.split_last(), - ll_code_table.split_last(), - sequences.split_last(), - ) - else { - return Err(ErrorCode::Generic); + long_length_type: LongLengthType, + long_length_pos: usize, +) -> SeqHistograms { + let mut histograms = SeqHistograms { + ll: [0; 64], + of: [0; 64], + ml: [0; 64], }; - - let mut state_match_length = FseCState::new2(ctable_match_length, u32::from(ml_last)); - let mut state_offset_bits = FseCState::new2(ctable_offset_bits, u32::from(of_last)); - let mut state_lit_length = FseCState::new2(ctable_lit_length, u32::from(ll_last)); - block_stream.add_bits( - u64::from(seq_last.lit_length), - bits_of(&LL_BITS_PADDED, ll_last), - ); - block_stream.add_bits( - u64::from(seq_last.ml_base), - bits_of(&ML_BITS_PADDED, ml_last), - ); - block_stream.add_bits(u64::from(seq_last.off_base), u32::from(of_last)); - block_stream.flush_bits(); - - for (((seq, &ll_code), &of_code), &ml_code) in sequences - .iter() - .zip(ll_codes) - .zip(of_codes) - .zip(ml_codes) - .rev() + for (codes, seq) in codes.iter_mut().zip(sequences) { + if let (Some(ll), Some(ml)) = (LL_CODES.get(seq.lit_length), ML_CODES.get(seq.ml_base)) + && let Some(usual) = SeqCodes::new(ll, ml, seq.off_base) + { + codes.set(usual); + histograms.add(usual); + } else { + *codes = histograms.add_unusual(*seq); + } + } + if long_length_type != LongLengthType::None + && let (Some(codes), Some(seq)) = ( + codes.get_mut(long_length_pos), + sequences.get(long_length_pos), + ) { - let ll_bits = bits_of(&LL_BITS_PADDED, ll_code); - let of_bits = u32::from(of_code); - let ml_bits = bits_of(&ML_BITS_PADDED, ml_code); - state_offset_bits.encode_symbol(&mut block_stream, u32::from(of_code)); - state_match_length.encode_symbol(&mut block_stream, u32::from(ml_code)); - state_lit_length.encode_symbol(&mut block_stream, u32::from(ll_code)); - if of_bits + ml_bits + ll_bits >= 64 - 7 - (LL_FSE_LOG + ML_FSE_LOG + OFF_FSE_LOG) { - block_stream.flush_bits(); + let mut ll = LL_CODES.get_any(seq.lit_length, LL_DELTA_CODE); + let mut ml = ML_CODES.get_any(seq.ml_base, ML_DELTA_CODE); + let (length, counts, code, bits) = if long_length_type == LongLengthType::LiteralLength { + (&mut ll, &mut histograms.ll, MAX_LL, seq.lit_length) + } else { + (&mut ml, &mut histograms.ml, MAX_ML, seq.ml_base) + }; + count_code(counts, length.code, 1u32.wrapping_neg()); + *length = LengthCode { + code: Code::new(code), + nb_bits: 16, + bits: u32::from(bits), + }; + count_code(counts, length.code, 1); + *codes = SeqCodes::new_any(ll, ml, seq.off_base); + } + histograms +} + +/// Bits on their way to the stream: the first to be written are the lowest. +#[derive(Clone, Copy)] +struct SeqBits { + bits: u64, + nb_bits: u32, +} + +impl SeqBits { + /// `FSE_encodeSymbol`: the bits that leave the state go below the others. + #[inline(always)] + fn encode_below(&mut self, table: &FseCTable, state: &mut u32, code: Code) { + let transform = (table.symbol_tt.get(code as usize).copied()).unwrap_or_default(); + let nb_bits = state.wrapping_add(transform.delta_nb_bits) >> 16; + let kept = state.wrapping_shr(nb_bits); + // Taking `kept` from what comes above the state cancels the bits that stay in it. + self.bits = (self.bits.wrapping_sub(u64::from(kept))) + .wrapping_shl(nb_bits) + .wrapping_add(u64::from(*state)); + self.nb_bits += nb_bits; + let next = kept.wrapping_add_signed(transform.delta_find_state) as usize; + let state_table = &table.state_table; + *state = u32::from((state_table.get(next & (state_table.len() - 1)).copied()).unwrap_or(0)); + } +} + +/// The states of the three encoders. +#[derive(Clone, Copy)] +struct SeqStates { + lit_length: u32, + match_length: u32, + offset_bits: u32, +} + +/// `BIT_CStream_t`, without a container: what a byte lacks is added to it in the output. +struct SeqStream<'a> { + /// In the byte that `bit_pos` is in, that bit is set, and those above it are not. + dst: &'a mut [u8], + bit_pos: usize, +} + +impl SeqStream<'_> { + /// `BIT_addBits` and `BIT_flushBits`: `seq` makes at most 56 bits, with a bit set above them. `None` if the output is + /// too small, which is found out first so that there is less to keep while `seq` is at work. + #[inline(always)] + fn write_with(&mut self, seq: impl FnOnce() -> SeqBits) -> Option<()> { + let pos = self.bit_pos >> 3; + if pos > self.dst.len().checked_sub(size_of::())? { + return None; } - block_stream.add_bits(u64::from(seq.lit_length), ll_bits); - block_stream.add_bits(u64::from(seq.ml_base), ml_bits); - if of_bits + ml_bits + ll_bits > 56 { - block_stream.flush_bits(); + let out: &mut [u8; 8] = self.dst.get_mut(pos..)?.first_chunk_mut()?; + let seq = seq(); + let [partial, ..] = *out; + // The lowest of the new bits replaces the bit set. + let bits = seq + .bits + .wrapping_sub(1) + .wrapping_shl(self.bit_pos as u32 & 7); + *out = bits.wrapping_add(u64::from(partial)).to_le_bytes(); + self.bit_pos += seq.nb_bits as usize; + Some(()) + } + + #[inline(always)] + fn write(&mut self, bits: u32, nb_bits: u32) -> Option<()> { + let above = 1 << (nb_bits & 31); + self.write_with(|| SeqBits { + bits: (u64::from(bits) & (above - 1)) | above, + nb_bits, + }) + } +} + +/// Where [`encode_run`] leaves things. +struct Encoded { + states: SeqStates, + bit_pos: usize, + /// How many sequences are left. + nb_left: usize, +} + +/// From the last of `codes` towards the first. It stops after one whose extra bits are too many, with +/// [`SeqCodes::TOO_MANY`] bits too many in the position. `None` if the output is too small. +/// +/// On its own, so that nothing else competes for the registers. +#[inline(never)] +fn encode_run( + dst: &mut [u8], + bit_pos: usize, + tables: &FseCTables, + states: SeqStates, + codes: &[SeqCodes], +) -> Option { + let mut stream = SeqStream { dst, bit_pos }; + let mut states = SeqStates { + lit_length: states.lit_length, + match_length: states.match_length, + offset_bits: states.offset_bits, + }; + let mut codes = codes.iter(); + while let Some(seq_codes) = codes.next_back() { + stream.write_with(|| { + let mut seq = SeqBits { + bits: u64::from(seq_codes.extra_bits), + nb_bits: u32::from(seq_codes.nb_extra_bits), + }; + seq.encode_below( + &tables.litlength_c_table, + &mut states.lit_length, + seq_codes.ll, + ); + seq.encode_below( + &tables.matchlength_c_table, + &mut states.match_length, + seq_codes.ml, + ); + seq.encode_below( + &tables.offcode_c_table, + &mut states.offset_bits, + seq_codes.of, + ); + seq + })?; + if seq_codes.nb_extra_bits == SeqCodes::TOO_MANY { + break; } - block_stream.add_bits(u64::from(seq.off_base), of_bits); - block_stream.flush_bits(); } + Some(Encoded { + states, + bit_pos: stream.bit_pos, + nb_left: codes.len(), + }) +} - state_match_length.flush(&mut block_stream); - state_offset_bits.flush(&mut block_stream); - state_lit_length.flush(&mut block_stream); +/// Returns the size of the stream in bits, or `None` if the output is too small. +fn encode_sequences_into( + dst: &mut [u8], + tables: &FseCTables, + codes: &[SeqCodes], + sequences: &[SeqDef], +) -> Option { + *dst.first_mut()? = 1; + let mut stream = SeqStream { dst, bit_pos: 0 }; + let last_codes = codes.last()?; + let mut states = SeqStates { + lit_length: (tables.litlength_c_table).first_state(last_codes.ll as u8), + match_length: (tables.matchlength_c_table).first_state(last_codes.ml as u8), + offset_bits: tables.offcode_c_table.first_state(last_codes.of as u8), + }; + let mut nb_left = codes.len() - 1; + loop { + // The extra bits of the last sequence, which has nothing else to write, or of the one that the run stopped at. + let (seq_codes, seq) = (codes.get(nb_left)?, sequences.get(nb_left)?); + let ll_bits = LL_BITS.get(seq_codes.ll as usize).copied()?; + let ml_bits = ML_BITS.get(seq_codes.ml as usize).copied()?; + stream.write(u32::from(seq.lit_length), u32::from(ll_bits))?; + stream.write(u32::from(seq.ml_base), u32::from(ml_bits))?; + stream.write(seq.off_base, seq_codes.of as u32)?; - match block_stream.close() { - 0 => Err(ErrorCode::DstSizeTooSmall), - stream_size => Ok(stream_size), + let before = nb_left; + let run = encode_run( + stream.dst, + stream.bit_pos, + tables, + states, + codes.get(..nb_left)?, + )?; + states = run.states; + stream.bit_pos = run.bit_pos; + nb_left = run.nb_left; + if nb_left == before || codes.get(nb_left)?.nb_extra_bits != SeqCodes::TOO_MANY { + break; + } + stream.bit_pos -= usize::from(SeqCodes::TOO_MANY); } + + // `FSE_flushCState` + stream.write(states.match_length, tables.matchlength_c_table.table_log)?; + stream.write(states.offset_bits, tables.offcode_c_table.table_log)?; + stream.write(states.lit_length, tables.litlength_c_table.table_log)?; + // `BIT_closeCStream`: a bit set, which is there. + Some(stream.bit_pos + 1) +} + +/// `ZSTD_encodeSequences`: `codes` are those of `sequences`, of which there is at least one. +pub(crate) fn encode_sequences( + dst: &mut [u8], + tables: &FseCTables, + codes: &[SeqCodes], + sequences: &[SeqDef], +) -> Result { + let capacity = dst.len(); + encode_sequences_into(dst, tables, codes, sequences) + // The stream has to end before the last position that a container can be written to. + .filter(|nb_bits| (nb_bits >> 3) + size_of::() < capacity) + .map(|nb_bits| nb_bits.div_ceil(8)) + .ok_or(ErrorCode::DstSizeTooSmall) } diff --git a/src/zstd/compress/superblock.rs b/src/zstd/compress/superblock.rs index cffc902e2352..91a2b24966d3 100644 --- a/src/zstd/compress/superblock.rs +++ b/src/zstd/compress/superblock.rs @@ -1,6 +1,6 @@ //! `zstd_compress_superblock.c`: a block written as several smaller ones, of about the size asked for. -use super::block_entropy::{build_block_entropy_stats, write_nb_seq}; +use super::block_entropy::{additional_bits_of, build_block_entropy_stats, write_nb_seq}; use super::fse::FseCTable; use super::internal::{ CompressedBlockState, EntropyCTables, EntropyCTablesMetadata, FseCTables, FseCTablesMetadata, @@ -8,12 +8,15 @@ use super::internal::{ }; use super::literals::{compress_rle_literals_block, no_compress_literals}; use super::params::CCtxParams; -use super::sequences::{cross_entropy_cost, encode_sequences, fse_bit_cost}; +use super::sequences::{ + CodeCounts, SeqCodes, SeqHistograms, cross_entropy_cost, encode_sequences, fse_bit_cost, + summarize, +}; use super::{hist, huf}; use crate::common::{ BlockType, DEFAULT_MAX_OFF, HUF_SYMBOLVALUE_MAX, LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, - MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MINMATCH, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, - OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, SymbolEncodingType, ZSTD_BLOCKHEADERSIZE, + MAX_LL, MAX_ML, MINMATCH, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, + OF_DEFAULT_NORM_LOG, SymbolEncodingType, ZSTD_BLOCKHEADERSIZE, }; use crate::error::{ErrorCode, Result}; use crate::util::{SliceExt, write_le24, write_le32}; @@ -116,21 +119,13 @@ fn seq_decompressed_size( Ok(match_length_sum + lit_size) } -/// The codes of some sequences. -#[derive(Clone, Copy)] -struct Codes<'a> { - ll: &'a [u8], - ml: &'a [u8], - of: &'a [u8], -} - /// `ZSTD_compressSubBlock_sequences`: the sequences section of a sub-block. Returns its size, which is 0 if it could /// not be compressed, and whether it holds the descriptions of the tables. fn compress_sub_block_sequences( fse_tables: &FseCTables, fse_metadata: &FseCTablesMetadata, sequences: &[SeqDef], - codes: Codes<'_>, + codes: &[SeqCodes], dst: &mut [u8], write_entropy: bool, ) -> Result<(usize, bool)> { @@ -166,16 +161,7 @@ fn compress_sub_block_sequences( )?; } - let bitstream_size = encode_sequences( - dst.sub_mut(op..)?, - &fse_tables.matchlength_c_table, - codes.ml, - &fse_tables.offcode_c_table, - codes.of, - &fse_tables.litlength_c_table, - codes.ll, - sequences, - )?; + let bitstream_size = encode_sequences(dst.sub_mut(op..)?, fse_tables, codes, sequences)?; op += bitstream_size; // Decoders up to 1.3.4 fail on a table header with fewer than 4 bytes after it, if write_entropy @@ -205,7 +191,7 @@ fn compress_sub_block( entropy_metadata: &EntropyCTablesMetadata, sequences: &[SeqDef], literals: &[u8], - codes: Codes<'_>, + codes: &[SeqCodes], dst: &mut [u8], write_lit_entropy: bool, write_seq_entropy: bool, @@ -281,87 +267,80 @@ fn estimate_sub_block_size_literal( } } -/// `ZSTD_estimateSubBlockSize_symbolType` +/// `ZSTD_estimateSubBlockSize_symbolType`: `count` is the distribution of `nb_seq` codes. fn estimate_sub_block_size_symbol_type( ty: SymbolEncodingType, - code_table: &[u8], - max_code: u32, + count: &CodeCounts, + nb_seq: usize, fse_c_table: &FseCTable, additional_bits: Option<&[u8]>, default_norm: &[i16], default_norm_log: u32, default_max: u32, -) -> Result { - let mut count = [0u32; MAX_SEQ as usize + 1]; - let mut max = max_code; - hist::count_fast_wksp(&mut count, &mut max, code_table)?; +) -> usize { + let max = summarize(count).max_symbol_value; let estimate_in_bits = match ty { SymbolEncodingType::Basic if max <= default_max => { - cross_entropy_cost(default_norm, default_norm_log, &count, max) + cross_entropy_cost(default_norm, default_norm_log, count, max) } SymbolEncodingType::Basic => Err(ErrorCode::Generic), SymbolEncodingType::Rle => Ok(0), SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { - fse_bit_cost(fse_c_table, &count, max) + fse_bit_cost(fse_c_table, count, max) } }; - let Ok(mut estimate_in_bits) = estimate_in_bits else { - return Ok(code_table.len() * 10); + let Ok(estimate_in_bits) = estimate_in_bits else { + return nb_seq * 10; }; - for &code in code_table { - estimate_in_bits += usize::from(match additional_bits { - Some(bits) => bits.at(usize::from(code))?, - None => code, - }); - } - Ok(estimate_in_bits / 8) + (estimate_in_bits + additional_bits_of(count, additional_bits)) / 8 } /// `ZSTD_estimateSubBlockSize_sequences` fn estimate_sub_block_size_sequences( - codes: Codes<'_>, + histograms: &SeqHistograms, + nb_seq: usize, fse_tables: &FseCTables, fse_metadata: &FseCTablesMetadata, write_entropy: bool, -) -> Result { +) -> usize { const SEQUENCES_SECTION_HEADER_SIZE: usize = 3; - if codes.of.is_empty() { - return Ok(SEQUENCES_SECTION_HEADER_SIZE); + if nb_seq == 0 { + return SEQUENCES_SECTION_HEADER_SIZE; } let mut c_seq_size_estimate = estimate_sub_block_size_symbol_type( fse_metadata.of_type, - codes.of, - MAX_OFF, + &histograms.of, + nb_seq, &fse_tables.offcode_c_table, None, &OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, DEFAULT_MAX_OFF, - )?; + ); c_seq_size_estimate += estimate_sub_block_size_symbol_type( fse_metadata.ll_type, - codes.ll, - MAX_LL, + &histograms.ll, + nb_seq, &fse_tables.litlength_c_table, Some(&LL_BITS), &LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, MAX_LL, - )?; + ); c_seq_size_estimate += estimate_sub_block_size_symbol_type( fse_metadata.ml_type, - codes.ml, - MAX_ML, + &histograms.ml, + nb_seq, &fse_tables.matchlength_c_table, Some(&ML_BITS), &ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, MAX_ML, - )?; + ); if write_entropy { c_seq_size_estimate += fse_metadata.fse_tables_size; } - Ok(c_seq_size_estimate + SEQUENCES_SECTION_HEADER_SIZE) + c_seq_size_estimate + SEQUENCES_SECTION_HEADER_SIZE } /// `ZSTD_needSequenceEntropyTables` @@ -421,6 +400,7 @@ fn size_block_sequences( /// `ZSTD_compressSubBlock_multi` fn compress_sub_block_multi( seq_store: &SeqStore, + histograms: &SeqHistograms, prev_c_block: &CompressedBlockState, next_c_block: &mut CompressedBlockState, entropy_metadata: &EntropyCTablesMetadata, @@ -441,13 +421,7 @@ fn compress_sub_block_multi( let mut write_lit_entropy = entropy_metadata.huf_metadata.h_type == SymbolEncodingType::Compressed; let mut write_seq_entropy = true; - let codes = |from: usize, count: usize| -> Result> { - Ok(Codes { - ll: seq_store.ll_code.span(from, count)?, - ml: seq_store.ml_code.span(from, count)?, - of: seq_store.of_code.span(from, count)?, - }) - }; + let codes = |from: usize, count: usize| seq_store.codes.span(from, count); if nb_seqs > 0 { let est_lit_size = estimate_sub_block_size_literal( @@ -457,11 +431,12 @@ fn compress_sub_block_multi( write_lit_entropy, ); let est_block_size = estimate_sub_block_size_sequences( - codes(0, nb_seqs)?, + histograms, + nb_seqs, &next_c_block.entropy.fse, &entropy_metadata.fse_metadata, write_seq_entropy, - )? + est_lit_size + ) + est_lit_size + ZSTD_BLOCKHEADERSIZE; let avg_lit_cost = (est_lit_size * BYTESCALE) .checked_div(nb_literals) @@ -583,7 +558,7 @@ pub(crate) fn compress_super_block( ) -> Result { let mut entropy_metadata = EntropyCTablesMetadata::new(); let whole = seq_store.whole(); - build_block_entropy_stats( + let histograms = build_block_entropy_stats( seq_store, &whole, &prev_c_block.entropy, @@ -593,6 +568,7 @@ pub(crate) fn compress_super_block( )?; compress_sub_block_multi( seq_store, + &histograms, prev_c_block, next_c_block, &entropy_metadata, From d43d2f4e2036a881a7295ac3de4630730e863e57 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:54:17 +0000 Subject: [PATCH 023/199] zstd: dfast keeps what it needs once per match out of the way of the search, and takes no branch on an index that is too low --- src/zstd/compress/double_fast.rs | 331 +++++++++++++++---------------- src/zstd/compress/fast.rs | 47 +++-- src/zstd/compress/internal.rs | 15 ++ 3 files changed, 208 insertions(+), 185 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 2425f53e5e6b..d24daa4ec656 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -1,11 +1,13 @@ //! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. -use super::fast::{FAST_HASH_FILL_STEP, NoDictBlock, TwoSegments, count_backwards, restore_rep}; +use super::fast::{ + FAST_HASH_FILL_STEP, MatchSink, NoDictBlock, TwoSegments, count_backwards, restore_rep, +}; use super::internal::{ BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, - REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, count, count_2segments, hash_of, hash_ptr, hash8, index_overlap_check, - offset_to_offbase, table_get, table_put, write_tagged_index, + le64_at, offset_to_offbase, table_get, table_put, table_swap, write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -103,193 +105,205 @@ pub(crate) fn fill_double_hash_table( } } -/// `ZSTD_compressBlock_doubleFast_noDict_generic` -fn compress_block_double_fast_no_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - const K_STEP_INCR: u32 = 1 << K_SEARCH_STRENGTH; - - let Some(block) = NoDictBlock::new(ms, rep, src)? else { - return Ok(src.end - src.start); - }; - let NoDictBlock { - near, - base_index, - candidates, - prefix_lowest_index, - prefix_lowest, - ilimit, - mut offset_1, - mut offset_2, - .. - } = block; - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let mut anchor = block.istart; - let mut ip = block.first; - let mut seq_store = SeqSink::new(seq_store); - let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; - let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; +/// What `dfast` looks for matches with, when there is no dictionary and the window is one segment. +struct Finder { + /// The index of the first position. + base_index: u32, + h_bits_l: u32, + h_bits_s: u32, + /// Where to start. + first: u32, + offset_1: u32, + offset_2: u32, +} - // One turn per match. - 'start: loop { - let mut step = 1; - let mut next_step = ip + K_STEP_INCR; - let mut ip1 = ip + step; - if ip1 as usize > ilimit { - break 'start; +impl Finder { + /// `ZSTD_compressBlock_doubleFast_noDict_generic` + /// + /// The C code computes the long hash and reads its table one position ahead. Here that is left to the processor: a + /// turn of the inner loop only takes the position and the 8 bytes there, `bytes0`, from the one before. + #[inline(never)] + fn find_matches( + &mut self, + hash_long: &mut [u32], + hash_small: &mut [u32], + sink: &mut MatchSink<'_, '_>, + ) { + const K_STEP_INCR: u32 = 1 << K_SEARCH_STRENGTH; + let Self { + base_index, + h_bits_l, + h_bits_s, + first: mut ip, + mut offset_1, + mut offset_2, + } = *self; + let near = sink.near; + if hash_long.is_empty() || hash_small.is_empty() { + return; } + let Some(mut bytes0) = near.read64(ip) else { + return; + }; - // The 8 bytes at `ip`. - let mut bytes0 = near.read64(ip).ok_or(BUG)?; - let mut hl0 = hash8(bytes0, h_bits_l, 0); - let mut idxl0 = table_get(hash_long, hl0); - let mut curr; - - let m_length = 'match_stored: { - // One turn per position. Ends with the offset and the length of a match, which `ip` is moved to the start - // of. - let (offset, m_length, hl1) = loop { - let hs0 = hash_of::(bytes0, h_bits_s); - let idxs0 = table_get(hash_small, hs0); + // One turn per match. + 'block: loop { + let mut step = 1; + let mut next_step = ip + K_STEP_INCR; + let mut ip1 = ip + step; + let mut curr; + + // One turn per position. + let end = loop { + let Some(bytes1) = near.read64(ip1) else { + break 'block; + }; curr = base_index.wrapping_add(ip); - table_put(hash_long, hl0, curr); - table_put(hash_small, hs0, curr); + let idxl0 = table_swap(hash_long, hash8(bytes0, h_bits_l, 0), curr); + let idxs0 = table_swap(hash_small, hash_of::(bytes0, h_bits_s), curr); - if offset_1 > 0 - && near.read32((ip + 1).wrapping_sub(offset_1)) == Some((bytes0 >> 8) as u32) + if sink.read32_before(ip + 1, offset_1) == Some((bytes0 >> 8) as u32) + && offset_1 > 0 { ip += 1; - let m_length = near.count(ip + 4, (ip + 4).wrapping_sub(offset_1)) + 4; - seq_store.store_seq( - ip - anchor, - near, - anchor, - REPCODE1_TO_OFFBASE, - m_length, - )?; - break 'match_stored m_length; + let end = ip + 4 + sink.count_before(ip + 4, offset_1); + sink.store(ip, end, REPCODE1_TO_OFFBASE); + break end; } - let bytes1 = near.read64(ip1).ok_or(BUG)?; - let hl1 = hash8(bytes1, h_bits_l, 0); - - if candidates.read64(idxl0.wrapping_sub(prefix_lowest_index)) == Some(bytes0) { - let matchl0 = idxl0 - base_index; - let m_length = near.count(ip + 8, matchl0 + 8) + 8; - let offset = ip - matchl0; - let back = near.count_backwards(ip, anchor, matchl0, prefix_lowest); - ip -= back; - break (offset, m_length + back, hl1); - } - - let idxl1 = table_get(hash_long, hl1); - - if candidates.read32(idxs0.wrapping_sub(prefix_lowest_index)) == Some(bytes0 as u32) - { + // An index that is too low leads past the end. Whether one is takes no branch to find out: what is + // read instead is unlikely to be the same. + let matchl0 = idxl0.wrapping_sub(base_index); + let matchs0 = idxs0.wrapping_sub(base_index); + let (match0, end) = if near.diff64(matchl0, bytes0) == 0 { + (matchl0, ip + 8 + near.count(ip + 8, matchl0 + 8)) + } else if near.diff64(matchs0, bytes0) as u32 == 0 { // A short match: a longer one at the next position is preferred. - let mut matchs0 = idxs0 - base_index; - let mut m_length = near.count(ip + 4, matchs0 + 4) + 4; - let mut offset = ip - matchs0; + let end = ip + 4 + near.count(ip + 4, matchs0 + 4); + let idxl1 = table_get(hash_long, hash8(bytes1, h_bits_l, 0)); let matchl1 = idxl1.wrapping_sub(base_index); - if idxl1 > prefix_lowest_index && near.read64(matchl1) == Some(bytes1) { - let l1len = near.count(ip1 + 8, matchl1 + 8) + 8; - if l1len > m_length { + if matchl1 > 0 && near.read64(matchl1) == Some(bytes1) { + let end1 = ip1 + 8 + near.count(ip1 + 8, matchl1 + 8); + if end1 - ip1 > end - ip { ip = ip1; - m_length = l1len; - offset = ip - matchl1; - matchs0 = matchl1; + (matchl1, end1) + } else { + (matchs0, end) } + } else { + (matchs0, end) } - let back = near.count_backwards(ip, anchor, matchs0, prefix_lowest); - ip -= back; - break (offset, m_length + back, hl1); - } + } else { + if ip1 >= next_step { + core::hint::cold_path(); + step += 1; + next_step += K_STEP_INCR; + } + ip = ip1; + ip1 += step; + bytes0 = bytes1; + continue; + }; - if ip1 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - ip = ip1; - ip1 += step; - - bytes0 = bytes1; - hl0 = hl1; - idxl0 = idxl1; - if ip1 as usize > ilimit { - break 'start; + offset_2 = offset_1; + offset_1 = ip - match0; + // With a larger step, ip1 may be past the end of the match. + if step < 4 { + table_put( + hash_long, + hash8(bytes1, h_bits_l, 0), + base_index.wrapping_add(ip1), + ); } + let back = near.count_backwards(ip, sink.anchor, match0, 0); + sink.store(ip - back, end, offset_to_offbase(offset_1)); + break end; }; + ip = end; - offset_2 = offset_1; - offset_1 = offset; - // With a larger step, ip1 may be past the end of the match. - if step < 4 { - table_put(hash_long, hl1, base_index.wrapping_add(ip1)); - } - seq_store.store_seq( - ip - anchor, - near, - anchor, - offset_to_offbase(offset), - m_length, - )?; - m_length - }; - - ip += m_length; - anchor = ip; - - if ip as usize <= ilimit { + // `ip` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip - 2) else { + break 'block; + }; let index_to_insert = curr + 2; - let to_insert = near.read64(index_to_insert - base_index).ok_or(BUG)?; - let before2 = near.read64(ip - 2).ok_or(BUG)?; - let before1 = near.read64(ip - 1).ok_or(BUG)?; - table_put(hash_long, hash8(to_insert, h_bits_l, 0), index_to_insert); + if let Some(to_insert) = near.read64(index_to_insert.wrapping_sub(base_index)) { + table_put(hash_long, hash8(to_insert, h_bits_l, 0), index_to_insert); + table_put( + hash_small, + hash_of::(to_insert, h_bits_s), + index_to_insert, + ); + } table_put( hash_long, - hash8(before2, h_bits_l, 0), + hash8(le64_at::<0, _>(around), h_bits_l, 0), base_index.wrapping_add(ip - 2), ); table_put( hash_small, - hash_of::(to_insert, h_bits_s), - index_to_insert, - ); - table_put( - hash_small, - hash_of::(before1, h_bits_s), + hash_of::(le64_at::<1, _>(around), h_bits_s), base_index.wrapping_add(ip - 1), ); + bytes0 = le64_at::<2, _>(around); - while ip as usize <= ilimit && offset_2 > 0 { - let bytes = near.read64(ip).ok_or(BUG)?; - if near.read32(ip.wrapping_sub(offset_2)) != Some(bytes as u32) { - break; - } - let r_length = near.count(ip + 4, (ip + 4).wrapping_sub(offset_2)) + 4; + while offset_2 > 0 && sink.read32_before(ip, offset_2) == Some(bytes0 as u32) { + let end = ip + 4 + sink.count_before(ip + 4, offset_2); core::mem::swap(&mut offset_1, &mut offset_2); let index = base_index.wrapping_add(ip); - table_put(hash_small, hash_of::(bytes, h_bits_s), index); - table_put(hash_long, hash8(bytes, h_bits_l, 0), index); - seq_store.store_seq(0, near, anchor, REPCODE1_TO_OFFBASE, r_length)?; - ip += r_length; - anchor = ip; + table_put(hash_small, hash_of::(bytes0, h_bits_s), index); + table_put(hash_long, hash8(bytes0, h_bits_l, 0), index); + sink.store(ip, end, REPCODE1_TO_OFFBASE); + ip = end; + let Some(bytes) = near.read64(ip) else { + break 'block; + }; + bytes0 = bytes; } } + + self.offset_1 = offset_1; + self.offset_2 = offset_2; } +} +/// `ZSTD_compressBlock_doubleFast` +pub(crate) fn compress_block_double_fast( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let Some(block) = NoDictBlock::new(ms, rep, src)? else { + return Ok(src.end - src.start); + }; + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let mut finder = Finder { + base_index: block.prefix_lowest_index, + h_bits_l, + h_bits_s, + first: block.first - block.prefix_lowest, + offset_1: block.offset_1, + offset_2: block.offset_2, + }; + let mut sink = MatchSink::new(seq_store, &block); + let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; + let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; + match ms.c_params.min_match { + 5 => finder.find_matches::<5>(hash_long, hash_small, &mut sink), + 6 => finder.find_matches::<6>(hash_long, hash_small, &mut sink), + 7 => finder.find_matches::<7>(hash_long, hash_small, &mut sink), + _ => finder.find_matches::<4>(hash_long, hash_small, &mut sink), + } + let last_literals = sink.finish()?; restore_rep( rep, - offset_1, - offset_2, + finder.offset_1, + finder.offset_2, block.offset_saved1, block.offset_saved2, ); - Ok((near.len() - anchor) as usize) + Ok(last_literals) } /// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` @@ -552,21 +566,6 @@ fn compress_block_double_fast_dict_match_state_generic( Ok(seg.iend - anchor) } -/// `ZSTD_compressBlock_doubleFast` -pub(crate) fn compress_block_double_fast( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - match ms.c_params.min_match { - 5 => compress_block_double_fast_no_dict_generic::<5>(ms, seq_store, rep, src), - 6 => compress_block_double_fast_no_dict_generic::<6>(ms, seq_store, rep, src), - 7 => compress_block_double_fast_no_dict_generic::<7>(ms, seq_store, rep, src), - _ => compress_block_double_fast_no_dict_generic::<4>(ms, seq_store, rep, src), - } -} - /// `ZSTD_compressBlock_doubleFast_dictMatchState` pub(crate) fn compress_block_double_fast_dict_match_state( ms: &mut MatchState, diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 6abc85b4b431..e57d2a622e0f 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -378,20 +378,39 @@ impl<'a> NoDictBlock<'a> { pub(super) struct MatchSink<'a, 'b> { seq_store: SeqSink<'b>, /// The input from the lowest index a match can be at. Positions are in this. - near: Near<'a>, + pub(super) near: Near<'a>, /// The same with what is before it, as far as a repeat offset can reach. older: Near<'a>, /// How much that is. older_len: u32, /// The end of the last match. - anchor: u32, + pub(super) anchor: u32, failed: bool, } -impl MatchSink<'_, '_> { +impl<'a, 'b> MatchSink<'a, 'b> { + pub(super) fn new(seq_store: &'b mut SeqStore, block: &NoDictBlock<'a>) -> Self { + Self { + seq_store: SeqSink::new(seq_store), + near: block.candidates, + older: block.near, + older_len: block.prefix_lowest, + anchor: block.istart - block.prefix_lowest, + failed: false, + } + } + + /// How many bytes there are after the last match. + pub(super) fn finish(self) -> Result { + if self.failed { + return Err(BUG); + } + Ok((self.near.len() - self.anchor) as usize) + } + /// The 4 bytes that are `offset` before `pos`. #[inline(always)] - fn read32_before(&self, pos: u32, offset: u32) -> Option { + pub(super) fn read32_before(&self, pos: u32, offset: u32) -> Option { match self.near.read32(pos.wrapping_sub(offset)) { Some(bytes) => Some(bytes), None => { @@ -404,7 +423,7 @@ impl MatchSink<'_, '_> { /// How long what is at `pos` is the same as what is `offset` before. #[inline(always)] - fn count_before(&self, pos: u32, offset: u32) -> u32 { + pub(super) fn count_before(&self, pos: u32, offset: u32) -> u32 { if offset <= pos { self.near.count(pos, pos - offset) } else { @@ -416,7 +435,7 @@ impl MatchSink<'_, '_> { /// Stores the match from `ip` to `end`, with the literals before it. #[inline(always)] - fn store(&mut self, ip: u32, end: u32, off_base: u32) { + pub(super) fn store(&mut self, ip: u32, end: u32, off_base: u32) { let anchor = self.anchor; self.anchor = end; if (self.seq_store) @@ -593,14 +612,7 @@ pub(crate) fn compress_block_fast( rep_offset1: block.offset_1, rep_offset2: block.offset_2, }; - let mut sink = MatchSink { - seq_store: SeqSink::new(seq_store), - near: block.candidates, - older: block.near, - older_len: block.prefix_lowest, - anchor: block.istart - block.prefix_lowest, - failed: false, - }; + let mut sink = MatchSink::new(seq_store, &block); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; match ms.c_params.min_match { 5 => finder.find_matches::<5>(hash_table, &mut sink), @@ -608,10 +620,7 @@ pub(crate) fn compress_block_fast( 7 => finder.find_matches::<7>(hash_table, &mut sink), _ => finder.find_matches::<4>(hash_table, &mut sink), } - if sink.failed { - return Err(BUG); - } - + let last_literals = sink.finish()?; restore_rep( rep, finder.rep_offset1, @@ -619,7 +628,7 @@ pub(crate) fn compress_block_fast( block.offset_saved1, block.offset_saved2, ); - Ok((sink.near.len() - sink.anchor) as usize) + Ok(last_literals) } /// `ZSTD_compressBlock_fast_dictMatchState_generic` diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 7a964942b44c..371126b8cb3d 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -684,6 +684,21 @@ impl<'a> Near<'a> { Some(u64::from_le_bytes(*bytes)) } + /// What differs between `bytes` and the 8 bytes at `pos`: 0 if nothing does. Not 0 in the lower half if `pos` is too + /// far to read them, which takes no branch to find out. + #[inline(always)] + pub(crate) fn diff64(self, pos: u32, bytes: u64) -> u64 { + let Some(last) = self.0.len().checked_sub(8) else { + return 1; + }; + let pos = pos as usize; + let at = pos.min(last); + match self.0.get(at..at + 8).and_then(<[u8]>::first_chunk) { + Some(there) => (u64::from_le_bytes(*there) ^ bytes) | u64::from(pos > last), + None => 1, + } + } + #[inline(always)] fn chunk16(self, pos: u32) -> Option<&'a [u8; 16]> { self.window(pos) From a9409b1aa372fa3ad9997741c9bc7a0fa285be22 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:54:45 +0000 Subject: [PATCH 024/199] zstd: the tables of the predefined distributions are not made again for each block --- src/zstd/compress/block_entropy.rs | 53 +++---------- src/zstd/compress/fse.rs | 44 +++++++++++ src/zstd/compress/sequences.rs | 119 ++++++++++++++++++++++++++--- 3 files changed, 163 insertions(+), 53 deletions(-) diff --git a/src/zstd/compress/block_entropy.rs b/src/zstd/compress/block_entropy.rs index 2fc24fad51a6..540d08778878 100644 --- a/src/zstd/compress/block_entropy.rs +++ b/src/zstd/compress/block_entropy.rs @@ -9,14 +9,13 @@ use super::internal::{ use super::literals::compress_literals; use super::params::CCtxParams; use super::sequences::{ - self, CodeCounts, SeqCodes, SeqHistograms, build_ctable, cross_entropy_cost, encode_sequences, - fse_bit_cost, select_encoding_type, summarize, + self, CodeCounts, CodeKind, LL_KIND, ML_KIND, OF_KIND, SeqCodes, SeqHistograms, build_ctable, + cross_entropy_cost, encode_sequences, fse_bit_cost, select_encoding_type, summarize, }; use super::{hist, huf::HufCTable}; use crate::common::{ - DEFAULT_MAX_OFF, HUF_SYMBOLVALUE_MAX, LIT_HUF_LOG, LL_BITS, LL_DEFAULT_NORM, - LL_DEFAULT_NORM_LOG, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MINMATCH, ML_BITS, ML_DEFAULT_NORM, - ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, OFF_FSE_LOG, + HUF_SYMBOLVALUE_MAX, LIT_HUF_LOG, LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, LONGNBSEQ, + MINMATCH, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, SymbolEncodingType, ZSTD_BLOCKHEADERSIZE, }; use crate::error::{ErrorCode, Result}; @@ -49,14 +48,6 @@ struct SymbolEncodingTypeStats { last_count_size: usize, } -/// What is particular to lengths or to offsets in how a table is made. -struct CodeKind<'a> { - fse_log: u32, - default_norm: &'a [i16], - default_norm_log: u32, - default_max: u32, -} - /// The table of one kind of code in the previous block. #[derive(Clone, Copy)] struct PrevTable<'a> { @@ -73,7 +64,7 @@ struct TableHeader { /// Chooses how to encode `nb_seq` codes, the last of which is `last_code` and of which `count` is the distribution, /// makes the table and writes its header. fn build_table( - kind: &CodeKind<'_>, + kind: &CodeKind, prev: PrevTable<'_>, next_table: &mut FseCTable, next_repeat_mode: &mut FseRepeat, @@ -101,18 +92,7 @@ fn build_table( strategy, )?; let size = build_ctable( - dst, - next_table, - kind.fse_log, - ty, - count, - max, - nb_seq, - last_code, - kind.default_norm, - kind.default_norm_log, - kind.default_max, - prev.table, + dst, next_table, kind, ty, count, max, nb_seq, last_code, prev.table, )?; Ok(TableHeader { ty, size }) } @@ -139,12 +119,7 @@ fn build_sequences_statistics( }; let ll = build_table( - &CodeKind { - fse_log: LL_FSE_LOG, - default_norm: &LL_DEFAULT_NORM, - default_norm_log: LL_DEFAULT_NORM_LOG, - default_max: MAX_LL, - }, + &LL_KIND, PrevTable { table: &prev_entropy.litlength_c_table, repeat_mode: prev_entropy.litlength_repeat_mode, @@ -159,12 +134,7 @@ fn build_sequences_statistics( )?; let op_of = written(&ll); let of = build_table( - &CodeKind { - fse_log: OFF_FSE_LOG, - default_norm: &OF_DEFAULT_NORM, - default_norm_log: OF_DEFAULT_NORM_LOG, - default_max: DEFAULT_MAX_OFF, - }, + &OF_KIND, PrevTable { table: &prev_entropy.offcode_c_table, repeat_mode: prev_entropy.offcode_repeat_mode, @@ -179,12 +149,7 @@ fn build_sequences_statistics( )?; let op_ml = written(&of); let ml = build_table( - &CodeKind { - fse_log: ML_FSE_LOG, - default_norm: &ML_DEFAULT_NORM, - default_norm_log: ML_DEFAULT_NORM_LOG, - default_max: MAX_ML, - }, + &ML_KIND, PrevTable { table: &prev_entropy.matchlength_c_table, repeat_mode: prev_entropy.matchlength_repeat_mode, diff --git a/src/zstd/compress/fse.rs b/src/zstd/compress/fse.rs index 87094b119d80..5d969fef0ea7 100644 --- a/src/zstd/compress/fse.rs +++ b/src/zstd/compress/fse.rs @@ -43,6 +43,40 @@ pub(crate) struct FseCTable { const SYMBOL_TT_SIZE: usize = (MAX_SEQ as usize + 1).next_power_of_two(); +/// A table for a distribution that is known in advance, of a table log of at most 6. +pub(crate) struct PredefinedCTable { + table_log: u32, + max_symbol_value: u32, + state_table: [u16; 64], + symbol_tt: [FseSymbolTransform; SYMBOL_TT_SIZE], +} + +impl PredefinedCTable { + pub(crate) const fn new( + table_log: u32, + states: [u16; STATES], + transforms: [FseSymbolTransform; SYMBOLS], + ) -> Self { + let mut table = Self { + table_log, + max_symbol_value: SYMBOLS as u32 - 1, + state_table: [0; 64], + symbol_tt: FseCTable::new().symbol_tt, + }; + table + .state_table + .split_at_mut(STATES) + .0 + .copy_from_slice(&states); + table + .symbol_tt + .split_at_mut(SYMBOLS) + .0 + .copy_from_slice(&transforms); + table + } +} + impl FseCTable { pub(crate) const fn new() -> Self { Self { @@ -165,6 +199,16 @@ impl FseCTable { Ok(()) } + /// What [`Self::build`] does for the distribution that `table` was made for. + pub(crate) fn set_predefined(&mut self, table: &PredefinedCTable) { + self.table_log = table.table_log; + self.max_symbol_value = table.max_symbol_value; + if let Some(states) = self.state_table.first_chunk_mut() { + *states = table.state_table; + } + self.symbol_tt = table.symbol_tt; + } + /// `FSE_buildCTable_rle`: a table that encodes `symbol_value`, and nothing else, in 0 bits. pub(crate) fn build_rle(&mut self, symbol_value: u8) -> Result<()> { self.table_log = 0; diff --git a/src/zstd/compress/sequences.rs b/src/zstd/compress/sequences.rs index dacd66203f01..1edbb720290b 100644 --- a/src/zstd/compress/sequences.rs +++ b/src/zstd/compress/sequences.rs @@ -1,10 +1,13 @@ //! `zstd_compress_sequences.c`: how the sequences of a block are encoded. -use super::fse::{self, FSE_NCOUNTBOUND, FseCTable, FseRepeat}; +use super::fse::{ + self, FSE_NCOUNTBOUND, FseCTable, FseRepeat, FseSymbolTransform, PredefinedCTable, +}; use super::internal::{FseCTables, LongLengthType, SeqDef, Strategy}; use crate::common::{ - LL_BITS, LL_FSE_LOG, MAX_LL, MAX_ML, MAX_SEQ, ML_BITS, ML_FSE_LOG, OFF_FSE_LOG, - SymbolEncodingType, + DEFAULT_MAX_OFF, LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, LL_FSE_LOG, MAX_LL, MAX_ML, + MAX_SEQ, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OF_DEFAULT_NORM, + OF_DEFAULT_NORM_LOG, OFF_FSE_LOG, SymbolEncodingType, }; use crate::error::{ErrorCode, Result}; use crate::util::{SliceExt, highbit32}; @@ -173,20 +176,118 @@ pub(crate) fn select_encoding_type( Ok(SymbolEncodingType::Compressed) } +const fn t(delta_find_state: i32, delta_nb_bits: u32) -> FseSymbolTransform { + FseSymbolTransform { + delta_find_state, + delta_nb_bits, + } +} + +/// What [`FseCTable::build`] makes of [`LL_DEFAULT_NORM`]. +#[rustfmt::skip] +static LL_DEFAULT_CTABLE: PredefinedCTable = PredefinedCTable::new( + LL_DEFAULT_NORM_LOG, + [ + 64, 65, 86, 107, 66, 87, 108, 88, 109, 67, 110, 68, 89, 90, 111, 69, + 112, 70, 91, 92, 113, 71, 114, 72, 93, 94, 115, 73, 116, 95, 74, 117, + 75, 96, 97, 118, 76, 119, 77, 98, 99, 120, 78, 121, 79, 100, 101, 122, + 80, 123, 81, 102, 103, 82, 104, 83, 105, 84, 106, 85, 127, 126, 125, 124, + ], + [ + t(-4, 327552), t( 1, 327584), t( 5, 393088), t( 7, 393088), t( 9, 393088), t(11, 393088), + t(13, 393088), t(15, 393088), t(17, 393088), t(19, 393088), t(21, 393088), t(23, 393088), + t(25, 393088), t(28, 393152), t(29, 393152), t(30, 393152), t(30, 393088), t(32, 393088), + t(34, 393088), t(36, 393088), t(38, 393088), t(40, 393088), t(42, 393088), t(44, 393088), + t(46, 393088), t(47, 327584), t(51, 393088), t(54, 393152), t(55, 393152), t(56, 393152), + t(57, 393152), t(58, 393152), t(59, 393152), t(60, 393152), t(61, 393152), t(62, 393152), + ], +); + +/// What [`FseCTable::build`] makes of [`OF_DEFAULT_NORM`]. +#[rustfmt::skip] +static OF_DEFAULT_CTABLE: PredefinedCTable = PredefinedCTable::new( + OF_DEFAULT_NORM_LOG, + [ + 32, 55, 46, 37, 51, 42, 33, 56, 38, 47, 43, 52, 34, 57, 48, 39, + 53, 44, 35, 58, 49, 40, 54, 45, 36, 50, 41, 63, 62, 61, 60, 59, + ], + [ + t(-1, 327648), t( 0, 327648), t( 1, 327648), t( 2, 327648), t( 3, 327648), t( 4, 327648), + t( 4, 327616), t( 6, 327616), t( 8, 327616), t(11, 327648), t(12, 327648), t(13, 327648), + t(14, 327648), t(15, 327648), t(16, 327648), t(17, 327648), t(18, 327648), t(19, 327648), + t(20, 327648), t(21, 327648), t(22, 327648), t(23, 327648), t(24, 327648), t(25, 327648), + t(26, 327648), t(27, 327648), t(28, 327648), t(29, 327648), t(30, 327648), + ], +); + +/// What [`FseCTable::build`] makes of [`ML_DEFAULT_NORM`]. +#[rustfmt::skip] +static ML_DEFAULT_CTABLE: PredefinedCTable = PredefinedCTable::new( + ML_DEFAULT_NORM_LOG, + [ + 64, 65, 86, 107, 108, 66, 87, 109, 67, 88, 89, 110, 68, 111, 69, 90, + 91, 112, 70, 113, 92, 71, 114, 93, 72, 115, 94, 73, 116, 95, 74, 117, + 96, 75, 118, 97, 76, 119, 98, 77, 120, 99, 78, 100, 79, 101, 80, 102, + 81, 103, 82, 104, 83, 105, 84, 106, 85, 127, 126, 125, 124, 123, 122, 121, + ], + [ + t(-1, 393152), t(-3, 327552), t( 2, 327584), t( 6, 393088), t( 8, 393088), t(10, 393088), + t(12, 393088), t(14, 393088), t(16, 393088), t(19, 393152), t(20, 393152), t(21, 393152), + t(22, 393152), t(23, 393152), t(24, 393152), t(25, 393152), t(26, 393152), t(27, 393152), + t(28, 393152), t(29, 393152), t(30, 393152), t(31, 393152), t(32, 393152), t(33, 393152), + t(34, 393152), t(35, 393152), t(36, 393152), t(37, 393152), t(38, 393152), t(39, 393152), + t(40, 393152), t(41, 393152), t(42, 393152), t(43, 393152), t(44, 393152), t(45, 393152), + t(46, 393152), t(47, 393152), t(48, 393152), t(49, 393152), t(50, 393152), t(51, 393152), + t(52, 393152), t(53, 393152), t(54, 393152), t(55, 393152), t(56, 393152), t(57, 393152), + t(58, 393152), t(59, 393152), t(60, 393152), t(61, 393152), t(62, 393152), + ], +); + +/// What is particular to literal lengths, to offsets or to match lengths in how a table is made. +pub(crate) struct CodeKind { + pub(crate) fse_log: u32, + pub(crate) default_norm: &'static [i16], + pub(crate) default_norm_log: u32, + /// The largest code that the predefined table has. + pub(crate) default_max: u32, + default_ctable: &'static PredefinedCTable, +} + +pub(crate) static LL_KIND: CodeKind = CodeKind { + fse_log: LL_FSE_LOG, + default_norm: &LL_DEFAULT_NORM, + default_norm_log: LL_DEFAULT_NORM_LOG, + default_max: MAX_LL, + default_ctable: &LL_DEFAULT_CTABLE, +}; + +pub(crate) static OF_KIND: CodeKind = CodeKind { + fse_log: OFF_FSE_LOG, + default_norm: &OF_DEFAULT_NORM, + default_norm_log: OF_DEFAULT_NORM_LOG, + default_max: DEFAULT_MAX_OFF, + default_ctable: &OF_DEFAULT_CTABLE, +}; + +pub(crate) static ML_KIND: CodeKind = CodeKind { + fse_log: ML_FSE_LOG, + default_norm: &ML_DEFAULT_NORM, + default_norm_log: ML_DEFAULT_NORM_LOG, + default_max: MAX_ML, + default_ctable: &ML_DEFAULT_CTABLE, +}; + /// `ZSTD_buildCTable`: makes the table for `nb_seq` codes, the last of which is `last_code` and of which `count` is the /// distribution, and writes what the decoder needs to make the same one. Returns how many bytes that takes. pub(crate) fn build_ctable( dst: &mut [u8], next_ctable: &mut FseCTable, - fse_log: u32, + kind: &CodeKind, ty: SymbolEncodingType, count: &mut [u32], max: u32, nb_seq: usize, last_code: u8, - default_norm: &[i16], - default_norm_log: u32, - default_max: u32, prev_ctable: &FseCTable, ) -> Result { match ty { @@ -203,12 +304,12 @@ pub(crate) fn build_ctable( Ok(0) } SymbolEncodingType::Basic => { - next_ctable.build(default_norm, default_max, default_norm_log)?; + next_ctable.set_predefined(kind.default_ctable); Ok(0) } SymbolEncodingType::Compressed => { let mut norm = [0i16; MAX_SEQ as usize + 1]; - let table_log = fse::optimal_table_log(fse_log, nb_seq, max); + let table_log = fse::optimal_table_log(kind.fse_log, nb_seq, max); // The last code is the first state, which takes no transition. let last_count = count.at_mut(usize::from(last_code))?; let without_last = usize::from(*last_count > 1); From 7d42e1bd57d5c891bf16e3351a1091ec975848aa Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:56:57 +0000 Subject: [PATCH 025/199] zstd: 8 equal bytes before a match take no detour if no more can be taken --- src/zstd/compress/internal.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 371126b8cb3d..153328170cb2 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -758,9 +758,11 @@ impl<'a> Near<'a> { if let (Some(bytes), Some(match_bytes)) = ( self.read64(pos.wrapping_sub(8)), self.read64(match_pos.wrapping_sub(8)), - ) && bytes != match_bytes - { - return ((bytes ^ match_bytes).leading_zeros() >> 3).min(most); + ) { + let same = (bytes ^ match_bytes).leading_zeros() >> 3; + if same < 8 || most <= 8 { + return same.min(most); + } } self.count_backwards_bytewise(pos, match_pos, most) } From c67b15a89ea03b66ae57827bd697b108ecb66759 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:58:28 +0000 Subject: [PATCH 026/199] zstd: a new context makes and clears less, and tables that cannot be used again are not copied --- src/zstd/compress/block_entropy.rs | 11 ++-- src/zstd/compress/cctx.rs | 89 ++++++++++++++++++------------ src/zstd/compress/internal.rs | 45 ++++++++++++--- src/zstd/compress/literals.rs | 6 +- src/zstd/compress/superblock.rs | 2 +- 5 files changed, 101 insertions(+), 52 deletions(-) diff --git a/src/zstd/compress/block_entropy.rs b/src/zstd/compress/block_entropy.rs index 540d08778878..619c0fb19658 100644 --- a/src/zstd/compress/block_entropy.rs +++ b/src/zstd/compress/block_entropy.rs @@ -29,6 +29,9 @@ pub(crate) fn literals_compression_is_disabled(cctx_params: &CCtxParams) -> bool /// `ZSTD_seqToCodes`, and the distributions of the codes. fn seq_to_codes(store: &mut SeqStore, chunk: &SeqChunk) -> Result { let range = chunk.seq_start..chunk.seq_end; + if store.codes.len() < store.sequences.len() { + (store.codes).resize(store.sequences.len(), SeqCodes::default()); + } Ok(sequences::seq_to_codes( store.codes.sub_mut(range.clone())?, store.sequences.sub(range)?, @@ -222,7 +225,7 @@ fn entropy_compress_seq_store_internal( } op += write_nb_seq(dst.sub_mut(op..)?, nb_seq)?; if nb_seq == 0 { - next_entropy.fse.clone_from(&prev_entropy.fse); + next_entropy.fse.copy_from(&prev_entropy.fse); return Ok(op); } let last_count_size = { @@ -314,7 +317,7 @@ fn build_block_entropy_stats_literals( let mut max_symbol_value = HUF_SYMBOLVALUE_MAX; let mut repeat = prev_huf.repeat_mode; - next_huf.clone_from(prev_huf); + next_huf.copy_from(prev_huf); if literals_compression_is_disabled { huf_metadata.h_type = SymbolEncodingType::Basic; @@ -373,13 +376,13 @@ fn build_block_entropy_stats_literals( && (old_c_size <= h_size.wrapping_add(new_c_size) || h_size.wrapping_add(12) >= src_size) { - next_huf.clone_from(prev_huf); + next_huf.copy_from(prev_huf); huf_metadata.h_type = SymbolEncodingType::Repeat; return Ok(0); } } if new_c_size.wrapping_add(h_size) >= src_size { - next_huf.clone_from(prev_huf); + next_huf.copy_from(prev_huf); huf_metadata.h_type = SymbolEncodingType::Basic; return Ok(0); } diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index e4c1da17d745..b21cbfb295b7 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -796,11 +796,23 @@ pub(crate) struct FrameCompressor { ldm_state: LdmState, ldm_sequences: RawSeqStore, extern_seq_store: RawSeqStore, - prev_c_block: Box, - next_c_block: Box, + /// `None` stands for [`INITIAL_BLOCK_STATE`]. + prev_c_block: Option>, + /// Made when the first block needs it. + next_c_block: Option>, match_state: MatchState, } +static INITIAL_BLOCK_STATE: CompressedBlockState = CompressedBlockState::new(); + +fn block_state(state: &Option>) -> &CompressedBlockState { + state.as_deref().unwrap_or(&INITIAL_BLOCK_STATE) +} + +fn block_state_mut(state: &mut Option>) -> &mut CompressedBlockState { + state.get_or_insert_with(|| Box::new(CompressedBlockState::new())) +} + const fn empty_raw_seq_store() -> RawSeqStore { RawSeqStore { seq: Vec::new(), @@ -831,8 +843,8 @@ impl FrameCompressor { ldm_state: LdmState::new(), ldm_sequences: empty_raw_seq_store(), extern_seq_store: empty_raw_seq_store(), - prev_c_block: Box::new(CompressedBlockState::new()), - next_c_block: Box::new(CompressedBlockState::new()), + prev_c_block: None, + next_c_block: None, match_state: MatchState::new(), } } @@ -919,7 +931,9 @@ impl FrameCompressor { self.dict_id = 0; self.dict_content_size = 0; - self.prev_c_block.reset(); + if let Some(prev_c_block) = &mut self.prev_c_block { + prev_c_block.reset(); + } reset_match_state( &mut self.match_state, @@ -957,7 +971,7 @@ impl FrameCompressor { /// `ZSTD_invalidateRepCodes` pub(crate) fn invalidate_rep_codes(&mut self) { - self.prev_c_block.rep = [0; ZSTD_REP_NUM]; + block_state_mut(&mut self.prev_c_block).rep = [0; ZSTD_REP_NUM]; } /// `ZSTD_resetCCtx_byAttachingCDict` @@ -1003,7 +1017,7 @@ impl FrameCompressor { self.dict_id = cdict.dict_id; self.dict_content_size = cdict.dict_content.len(); - self.prev_c_block.clone_from(&cdict.c_block_state); + block_state_mut(&mut self.prev_c_block).clone_from(&cdict.c_block_state); Ok(sizes) } @@ -1057,7 +1071,7 @@ impl FrameCompressor { self.dict_id = cdict.dict_id; self.dict_content_size = cdict.dict_content.len(); - self.prev_c_block.clone_from(&cdict.c_block_state); + block_state_mut(&mut self.prev_c_block).clone_from(&cdict.c_block_state); Ok(sizes) } @@ -1091,16 +1105,19 @@ impl FrameCompressor { CompResetPolicy::MakeClean, buffered, )?; - self.dict_id = compress_insert_dictionary( - &mut self.prev_c_block, - &mut self.match_state, - &self.applied_params, - &input.bufs, - dict, - dict_content_type, - dtlm, - TableFillPurpose::ForCCtx, - )?; + self.dict_id = match dict { + Some(_) => compress_insert_dictionary( + block_state_mut(&mut self.prev_c_block), + &mut self.match_state, + &self.applied_params, + &input.bufs, + dict, + dict_content_type, + dtlm, + TableFillPurpose::ForCCtx, + )?, + None => 0, + }; self.dict_content_size = dict_content_size; Ok(sizes) } @@ -1128,9 +1145,11 @@ impl FrameCompressor { /// A table for offsets that comes from a dictionary is only known to be valid for the first block. fn downgrade_offcode_repeat_mode(&mut self) { - let mode = &mut self.prev_c_block.entropy.fse.offcode_repeat_mode; - if *mode == FseRepeat::Valid { - *mode = FseRepeat::Check; + if let Some(prev_c_block) = &mut self.prev_c_block { + let mode = &mut prev_c_block.entropy.fse.offcode_repeat_mode; + if *mode == FseRepeat::Valid { + *mode = FseRepeat::Check; + } } } @@ -1175,13 +1194,13 @@ impl FrameCompressor { Some(cdict) if ms.has_dict_match_state => Some(cdict.dict_match_state()), _ => None, }, - symbol_costs: &self.prev_c_block.entropy, + symbol_costs: &block_state(&self.prev_c_block).entropy, ldm_seq_store: None, }; let dict_mode = ms.dict_mode(block.dms.as_ref()); let use_row_match_finder = params.use_row_match_finder == ParamSwitch::Enable; - self.next_c_block.rep = self.prev_c_block.rep; - let rep = &mut self.next_c_block.rep; + let rep = &mut block_state_mut(&mut self.next_c_block).rep; + *rep = block_state(&self.prev_c_block).rep; let last_ll_size = if self.extern_seq_store.pos < self.extern_seq_store.size { ldm::block_compress( @@ -1252,8 +1271,8 @@ impl FrameCompressor { let mut c_seqs_size = entropy_compress_seq_store( &mut self.seq_store, chunk, - &self.prev_c_block.entropy, - &mut self.next_c_block.entropy, + &block_state(&self.prev_c_block).entropy, + &mut block_state_mut(&mut self.next_c_block).entropy, &self.applied_params, dst.sub_mut(ZSTD_BLOCKHEADERSIZE..)?, src.len(), @@ -1306,8 +1325,8 @@ impl FrameCompressor { Ok(build_entropy_statistics_and_estimate_sub_block_size( &mut self.seq_store, &chunk, - &self.prev_c_block.entropy, - &mut self.next_c_block.entropy, + &block_state(&self.prev_c_block).entropy, + &mut block_state_mut(&mut self.next_c_block).entropy, &self.applied_params, entropy_metadata, )) @@ -1357,8 +1376,8 @@ impl FrameCompressor { let num_splits = splits.idx; let partitions = &splits.split_locations; - let mut d_rep = self.prev_c_block.rep; - let mut c_rep = self.prev_c_block.rep; + let mut d_rep = block_state(&self.prev_c_block).rep; + let mut c_rep = d_rep; if num_splits == 0 { let whole = self.seq_store.whole(); @@ -1404,7 +1423,7 @@ impl FrameCompressor { curr_seq_store = next_seq_store; } // What the decoder ends up with, which is not what the sequences were found with if a part is uncompressed. - self.prev_c_block.rep = d_rep; + block_state_mut(&mut self.prev_c_block).rep = d_rep; Ok(c_size) } @@ -1442,8 +1461,8 @@ impl FrameCompressor { let mut c_size = entropy_compress_seq_store( &mut self.seq_store, &whole, - &self.prev_c_block.entropy, - &mut self.next_c_block.entropy, + &block_state(&self.prev_c_block).entropy, + &mut block_state_mut(&mut self.next_c_block).entropy, &self.applied_params, dst, bytes.len(), @@ -1495,8 +1514,8 @@ impl FrameCompressor { } match compress_super_block( &mut self.seq_store, - &self.prev_c_block, - &mut self.next_c_block, + block_state(&self.prev_c_block), + block_state_mut(&mut self.next_c_block), &self.applied_params, dst, src, diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index ebf0ab1c2072..a52b68f17af7 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -113,6 +113,16 @@ pub(crate) struct HufCTables { pub(crate) repeat_mode: HufRepeat, } +impl HufCTables { + /// `*self = from.clone()`, but for a table that cannot be used again, which nothing reads. + pub(crate) fn copy_from(&mut self, from: &Self) { + self.repeat_mode = from.repeat_mode; + if from.repeat_mode != HufRepeat::None { + self.c_table.clone_from(&from.c_table); + } + } +} + /// `ZSTD_fseCTables_t` #[derive(Clone)] pub(crate) struct FseCTables { @@ -124,6 +134,24 @@ pub(crate) struct FseCTables { pub(crate) litlength_repeat_mode: FseRepeat, } +impl FseCTables { + /// `*self = from.clone()`, but for the tables that cannot be used again, which nothing reads. + pub(crate) fn copy_from(&mut self, from: &Self) { + self.offcode_repeat_mode = from.offcode_repeat_mode; + if from.offcode_repeat_mode != FseRepeat::None { + self.offcode_c_table.clone_from(&from.offcode_c_table); + } + self.matchlength_repeat_mode = from.matchlength_repeat_mode; + if from.matchlength_repeat_mode != FseRepeat::None { + (self.matchlength_c_table).clone_from(&from.matchlength_c_table); + } + self.litlength_repeat_mode = from.litlength_repeat_mode; + if from.litlength_repeat_mode != FseRepeat::None { + self.litlength_c_table.clone_from(&from.litlength_c_table); + } + } +} + /// `ZSTD_entropyCTables_t` #[derive(Clone)] pub(crate) struct EntropyCTables { @@ -140,7 +168,7 @@ pub(crate) struct CompressedBlockState { impl CompressedBlockState { /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. - pub(crate) fn new() -> Self { + pub(crate) const fn new() -> Self { Self { entropy: EntropyCTables { huf: HufCTables { @@ -242,7 +270,7 @@ pub(crate) struct SeqStore { pub(crate) lit_buffer: Vec, /// How much of `lit_buffer` is literals. pub(crate) lit_len: usize, - /// Room for the codes of `max_nb_seq` sequences. + /// As many as the most sequences that a block had so far, with room for `max_nb_seq`. pub(crate) codes: Vec, pub(crate) max_nb_seq: usize, pub(crate) max_nb_lit: usize, @@ -329,16 +357,15 @@ impl SeqStore { /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { - fn grow(vec: &mut Vec, len: usize) -> Result<()> { + fn room(vec: &mut Vec, len: usize) -> Result<()> { vec.clear(); vec.try_reserve_exact(len) - .map_err(|_| ErrorCode::MemoryAllocation)?; - vec.resize(len, T::default()); - Ok(()) + .map_err(|_| ErrorCode::MemoryAllocation) } - grow(&mut self.sequences, max_nb_seq)?; - grow(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; - grow(&mut self.codes, max_nb_seq)?; + room(&mut self.sequences, max_nb_seq)?; + room(&mut self.codes, max_nb_seq)?; + room(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; + self.lit_buffer.resize(max_nb_lit + WILDCOPY_OVERLENGTH, 0); self.max_nb_seq = max_nb_seq; self.max_nb_lit = max_nb_lit; self.reset(); diff --git a/src/zstd/compress/literals.rs b/src/zstd/compress/literals.rs index a942f06582f2..f143f67df024 100644 --- a/src/zstd/compress/literals.rs +++ b/src/zstd/compress/literals.rs @@ -62,7 +62,7 @@ pub(crate) fn compress_literals( let mut single_stream = src_size < 256; let mut h_type = SymbolEncodingType::Compressed; - next_huf.clone_from(prev_huf); + next_huf.copy_from(prev_huf); if disable_literal_compression { return no_compress_literals(dst, src); @@ -108,7 +108,7 @@ pub(crate) fn compress_literals( let c_lit_size = match c_lit_size { Ok(size) if size != 0 && size < src_size.wrapping_sub(min_gain) => size, _ => { - next_huf.clone_from(prev_huf); + next_huf.copy_from(prev_huf); return no_compress_literals(dst, src); } }; @@ -116,7 +116,7 @@ pub(crate) fn compress_literals( // With few literals, a size of 1 can also be that of a Huffman stream. let first = src.at(0)?; if src_size >= 8 || src.iter().all(|&b| b == first) { - next_huf.clone_from(prev_huf); + next_huf.copy_from(prev_huf); return compress_rle_literals_block(dst, src); } } diff --git a/src/zstd/compress/superblock.rs b/src/zstd/compress/superblock.rs index 91a2b24966d3..d95d8d0f9b18 100644 --- a/src/zstd/compress/superblock.rs +++ b/src/zstd/compress/superblock.rs @@ -522,7 +522,7 @@ fn compress_sub_block_multi( next_c_block .entropy .huf - .clone_from(&prev_c_block.entropy.huf); + .copy_from(&prev_c_block.entropy.huf); } if write_seq_entropy && need_sequence_entropy_tables(&entropy_metadata.fse_metadata) { // Neither were these, and there is no going back to the previous ones. From 6c8cf31a4579fd07dd9f3cbfcbcfef203add8c76 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:59:57 +0000 Subject: [PATCH 027/199] zstd: the buffers of a compression stream are not cleared --- src/zstd/compress/stream.rs | 55 +++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs index 5cfc01d8240d..bda467ea41c2 100644 --- a/src/zstd/compress/stream.rs +++ b/src/zstd/compress/stream.rs @@ -11,7 +11,7 @@ use super::params::{ resolve_enable_ldm, resolve_row_match_finder_mode, }; use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, try_copy, try_zeroed}; +use crate::util::{SliceExt, try_copy}; use crate::{InBuffer, OutBuffer}; /// `ZSTD_EndDirective` @@ -37,11 +37,15 @@ enum StreamStage { pub struct CCtx { frame: FrameCompressor, requested_params: CCtxParams, + /// As long as it was ever filled, with room for `in_buff_size`. in_buff: Vec, + in_buff_size: usize, in_to_compress: usize, in_buff_pos: usize, in_buff_target: usize, + /// Empty until it is needed, with room for `out_buff_size`. out_buff: Vec, + out_buff_size: usize, out_buff_content_size: usize, out_buff_flushed_size: usize, stream_stage: StreamStage, @@ -68,14 +72,29 @@ fn limit_copy(dst: &mut [u8], src: &[u8]) -> usize { length } -fn resize_buffer(buffer: &mut Vec, len: usize) -> Result<()> { - if buffer.len() != len { +/// Gives `buffer` room for `new_size` bytes instead of `size`, none of which is touched. +fn resize_buffer(buffer: &mut Vec, size: &mut usize, new_size: usize) -> Result<()> { + if *size != new_size { + // First, so that both are not allocated at once. *buffer = Vec::new(); - *buffer = try_zeroed(len)?; + *size = 0; + buffer + .try_reserve_exact(new_size) + .map_err(|_| ErrorCode::MemoryAllocation)?; + *size = new_size; } Ok(()) } +/// Writes `src` at `pos`, which is at most the length of `buffer`, over what is there and beyond. +fn write_or_append(buffer: &mut Vec, pos: usize, src: &[u8]) -> Result<()> { + let over = buffer.sub_mut(pos..)?; + let (first, rest) = src.split_at_checked(over.len()).unwrap_or((src, &[])); + over.write_at(0, first)?; + buffer.extend_from_slice(rest); + Ok(()) +} + impl CCtx { /// How much memory a new one takes. pub const SIZE: usize = size_of::() + 2 * size_of::(); @@ -86,10 +105,12 @@ impl CCtx { frame: FrameCompressor::new(), requested_params: CCtxParams::new(ZSTD_CLEVEL_DEFAULT), in_buff: Vec::new(), + in_buff_size: 0, in_to_compress: 0, in_buff_pos: 0, in_buff_target: 0, out_buff: Vec::new(), + out_buff_size: 0, out_buff_content_size: 0, out_buff_flushed_size: 0, stream_stage: StreamStage::Init, @@ -217,8 +238,16 @@ impl CCtx { pledged_src_size, true, )?; - resize_buffer(&mut self.in_buff, sizes.buff_in_size)?; - resize_buffer(&mut self.out_buff, sizes.buff_out_size)?; + resize_buffer( + &mut self.in_buff, + &mut self.in_buff_size, + sizes.buff_in_size, + )?; + resize_buffer( + &mut self.out_buff, + &mut self.out_buff_size, + sizes.buff_out_size, + )?; self.in_to_compress = 0; self.in_buff_pos = 0; // One more byte, to tell an input of exactly one block from a longer one. @@ -287,11 +316,12 @@ impl CCtx { break; } { - let loaded = limit_copy( - self.in_buff - .sub_mut(self.in_buff_pos..self.in_buff_target)?, - input.src.sub(ip..)?, - ); + let loaded = (self.in_buff_target - self.in_buff_pos).min(iend - ip); + write_or_append( + &mut self.in_buff, + self.in_buff_pos, + input.src.span(ip, loaded)?, + )?; self.in_buff_pos += loaded; ip += loaded; if flush_mode == EndDirective::Continue @@ -312,6 +342,7 @@ impl CCtx { let c_dst = if direct { output.dst.sub_mut(op..)? } else { + self.out_buff.resize(self.out_buff_size, 0); self.out_buff.as_mut_slice() }; let last_block = flush_mode == EndDirective::End && ip == iend; @@ -336,7 +367,7 @@ impl CCtx { self.frame_ended = last_block; // The next block goes after this one, or at the start if there is no room left. self.in_buff_target = self.in_buff_pos + self.frame.block_size_max; - if self.in_buff_target > self.in_buff.len() { + if self.in_buff_target > self.in_buff_size { self.in_buff_pos = 0; self.in_buff_target = self.frame.block_size_max; } From 84d8c68313f0b6ba8466ed07fca743f850e11aba Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:01:13 +0000 Subject: [PATCH 028/199] zstd: one way to reset a table, whatever it is a table of --- src/zstd/compress/cctx.rs | 26 +++++--------------------- src/zstd/compress/internal.rs | 6 ++++++ 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index b21cbfb295b7..4a854de0483b 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -155,23 +155,7 @@ pub(crate) fn reset_table( Ok(()) } -/// The same for a table of structures. -pub(crate) fn reset_struct_table( - table: &mut Vec, - len: usize, - clean: bool, -) -> Result<()> { - if table.len() != len { - *table = Vec::new(); - table - .try_reserve_exact(len) - .map_err(|_| ErrorCode::MemoryAllocation)?; - table.resize(len, T::default()); - } else if clean { - table.fill(T::default()); - } - Ok(()) -} +pub(crate) use reset_table as reset_struct_table; /// `ZSTD_reset_matchState` fn reset_match_state( @@ -217,8 +201,8 @@ fn reset_match_state( reset_table(&mut ms.opt.lit_length_freq, MAX_LL as usize + 1, false)?; reset_table(&mut ms.opt.match_length_freq, MAX_ML as usize + 1, false)?; reset_table(&mut ms.opt.off_code_freq, MAX_OFF as usize + 1, false)?; - reset_struct_table(&mut ms.opt.match_table, ZSTD_OPT_SIZE, false)?; - reset_struct_table(&mut ms.opt.price_table, ZSTD_OPT_SIZE, false)?; + reset_table(&mut ms.opt.match_table, ZSTD_OPT_SIZE, false)?; + reset_table(&mut ms.opt.price_table, ZSTD_OPT_SIZE, false)?; } else { ms.opt.match_table = Vec::new(); ms.opt.price_table = Vec::new(); @@ -946,8 +930,8 @@ impl FrameCompressor { if params.ldm_params.enable_ldm == ParamSwitch::Enable { let ldm_h_size = 1usize << params.ldm_params.hash_log; - reset_struct_table(&mut self.ldm_state.hash_table, ldm_h_size, true)?; - reset_struct_table(&mut self.ldm_sequences.seq, max_nb_ldm_seq, false)?; + reset_table(&mut self.ldm_state.hash_table, ldm_h_size, true)?; + reset_table(&mut self.ldm_sequences.seq, max_nb_ldm_seq, false)?; self.ldm_sequences.capacity = max_nb_ldm_seq; self.ldm_state.window = Window::new(); self.ldm_state.loaded_dict_end = 0; diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index a52b68f17af7..081599a8fb94 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1158,6 +1158,7 @@ pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) /// `ZSTD_match_t` #[derive(Clone, Copy, Default)] +#[repr(C)] pub(crate) struct Match { /// As [`SeqDef::off_base`]. pub(crate) off: u32, @@ -1166,6 +1167,7 @@ pub(crate) struct Match { /// `rawSeq`: a sequence found by long distance matching. #[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +#[repr(C)] pub(crate) struct RawSeq { pub(crate) offset: u32, pub(crate) lit_length: u32, @@ -1187,6 +1189,7 @@ pub(crate) struct RawSeqStore { /// `ZSTD_optimal_t` #[derive(Clone, Copy, Default)] +#[repr(C)] pub(crate) struct Optimal { /// The cost of getting to this position. pub(crate) price: i32, @@ -1462,6 +1465,7 @@ impl DictMatchState<'_> { /// `ldmEntry_t` #[derive(Clone, Copy, Default)] +#[repr(C)] pub(crate) struct LdmEntry { pub(crate) offset: u32, pub(crate) checksum: u32, @@ -1497,3 +1501,5 @@ impl LdmState { } } } + +bun_pod::impl_zeroable!(Match, RawSeq, Optimal, LdmEntry); From 9204768429ec188e12563be9638a9258123c180c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:04:22 +0000 Subject: [PATCH 029/199] zstd: say that it is a port, where it was said to be a C library LICENSE.md lists it as not linked, like libarchive. The comments that counted it among the decoders with C state, among the users of bun_core's state tag, or that said there is one xxhash in the workspace are brought up to date, and so is what try_zeroed_vec says about memory that the allocator recycles. --- LICENSE.md | 2 +- src/bun_core/lib.rs | 2 +- src/hash/xxhash.rs | 5 +++-- src/http/Decompressor.rs | 4 ++-- src/pod/lib.rs | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index aef802e5cf21..f650660dbd8c 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -29,7 +29,7 @@ Bun statically links these libraries: | [`lsquic`](https://github.com/litespeedtech/lsquic) | MIT (portions derived from [Chromium proto-quic](https://github.com/litespeedtech/lsquic/blob/master/LICENSE.chrome), BSD 3-Clause) | | [`mimalloc`](https://github.com/microsoft/mimalloc) | MIT | | [`picohttp`](https://github.com/h2o/picohttpparser) | dual-licensed under the Perl License or the MIT License | -| [`zstd`](https://github.com/facebook/zstd) | dual-licensed under the BSD License or GPLv2 license | +| [`zstd`](https://github.com/facebook/zstd) (not linked: `src/zstd` is a port of it) | dual-licensed under the BSD License or GPLv2 license | | [`simdutf`](https://github.com/simdutf/simdutf) | Apache 2.0 | | [`tinycc`](https://github.com/tinycc/tinycc) | LGPL v2.1 | | [`uSockets`](https://github.com/uNetworking/uSockets) | Apache 2.0 | diff --git a/src/bun_core/lib.rs b/src/bun_core/lib.rs index 4a8ef1ccfc7b..516fde342dad 100644 --- a/src/bun_core/lib.rs +++ b/src/bun_core/lib.rs @@ -27,7 +27,7 @@ pub mod util; pub use atomic_cell::{Atom, AtomicCell, ThreadCell}; /// Shared state-machine tag for the streaming (de)compressors in -/// `bun_brotli` / `bun_zlib` / `bun_zstd`. +/// `bun_brotli` / `bun_zlib`. pub mod compress { #[derive(Clone, Copy, PartialEq, Eq)] pub enum State { diff --git a/src/hash/xxhash.rs b/src/hash/xxhash.rs index 9d6e00a8a76d..bcdcd775fbea 100644 --- a/src/hash/xxhash.rs +++ b/src/hash/xxhash.rs @@ -32,8 +32,9 @@ impl XxHash64 { /// Streaming XxHash64 — used by the bundler's `ContentHasher` /// (length-prefixed chunk hashing across many `update()` calls before a single /// `digest()`), plus the dev-server source-map hash and the resolver stat hash. -/// Wraps `bun_highway::XxHash64State` so the workspace has exactly one xxhash -/// implementation; output is bit-identical to the xxHash reference. +/// Wraps `bun_highway::XxHash64State`; output is bit-identical to the xxHash +/// reference. The one other XXH64 in the workspace is `bun_zstd`'s, for frame +/// checksums: that crate is safe Rust with no foreign code under it. pub struct XxHash64Streaming(bun_highway::XxHash64State); impl XxHash64Streaming { diff --git a/src/http/Decompressor.rs b/src/http/Decompressor.rs index 6c32378ad9d8..89bd10ae4833 100644 --- a/src/http/Decompressor.rs +++ b/src/http/Decompressor.rs @@ -1,7 +1,7 @@ use bun_core::MutableString; use bun_http_types::Encoding::Encoding; -// The streaming decoders below own only their C-side state and take +// The streaming decoders below own only their codec's state and take // `(input, output)` per call to [`Decompressor::decompress_chunk`], so no // borrow of the request's `compressed_body` / `body_out_str` escapes the // call. @@ -22,7 +22,7 @@ pub(crate) fn has_zlib_header(buffer: &[u8]) -> bool { } impl Decompressor { - // Note: each variant's `Drop` releases the underlying C state, so an + // Note: each variant's `Drop` releases the codec's state, so an // explicit `Drop` is unnecessary. Callers that want a mid-lifecycle reset // assign `*self = Decompressor::None`. diff --git a/src/pod/lib.rs b/src/pod/lib.rs index 2957bea55592..5e9afd2b32ec 100644 --- a/src/pod/lib.rs +++ b/src/pod/lib.rs @@ -327,8 +327,8 @@ pub fn vec_from_bytes(bytes: &[u8]) -> Option(len: usize) -> Option> { let layout = core::alloc::Layout::array::(len).ok()?; From 7259097dfa5db27f37629cd8ac53619fb823fd8f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:04:23 +0000 Subject: [PATCH 030/199] zstd: workers touch no more memory than they use The round buffer for long distance matching was cleared when it was allocated: it is now as long as it has been written to. The buffer for raw sequences was meant to be kept from one job to the next but was replaced every time, since its length never was the one asked for. A count that nothing read is gone. --- src/zstd/compress/mt.rs | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 07f6a09a8619..4c73cc40c97b 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -29,7 +29,7 @@ use super::stream::EndDirective; use crate::common::ZSTD_BLOCKSIZE_MAX; use crate::common::xxhash::Xxh64; use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, try_copy}; +use crate::util::{BUG, SliceExt, try_copy}; use crate::{InBuffer, OutBuffer}; /// `SerialState`: what has to see the input in order. @@ -39,7 +39,7 @@ struct SerialState { ldm_state: LdmState, xxh_state: Xxh64, /// The input, where the C code has it in its round buffer: long distance matches reach further back than the - /// prefix of a job. + /// prefix of a job. It is as long as it has been written to. round_buff: Vec, } @@ -74,7 +74,9 @@ impl SerialState { )?; reset_table(&mut self.ldm_state.bucket_offsets, 1 << bucket_log, true)?; self.ldm_state.loaded_dict_end = 0; - reset_table(&mut self.round_buff, round_buff_capacity, false)?; + self.round_buff.clear(); + (self.round_buff.try_reserve_exact(round_buff_capacity)) + .map_err(|_| ErrorCode::MemoryAllocation)?; } else { self.ldm_state = LdmState::new(); self.round_buff = Vec::new(); @@ -92,7 +94,11 @@ impl SerialState { round_pos: usize, ) -> Result<()> { if self.ldm_params.enable_ldm == ParamSwitch::Enable { - self.round_buff.write_at(round_pos, src)?; + let written = self.round_buff.get_mut(round_pos..).ok_or(BUG)?; + let (over, beyond) = src.split_at(written.len().min(src.len())); + written.write_at(0, over)?; + (self.round_buff.try_reserve(beyond.len())).map_err(|_| ErrorCode::MemoryAllocation)?; + self.round_buff.extend_from_slice(beyond); (self.ldm_state.window).update( BufId::InBuff.start() + round_pos, BufId::InBuff, @@ -158,6 +164,15 @@ struct WorkerState { const CHUNK_SIZE: usize = 4 * ZSTD_BLOCKSIZE_MAX; +/// Makes `seq` `len` long, in the allocation that it has if that is large enough. +fn resize_in_place(seq: &mut Vec, len: usize) -> Result<()> { + seq.clear(); + seq.try_reserve(len) + .map_err(|_| ErrorCode::MemoryAllocation)?; + seq.resize(len, RawSeq::default()); + Ok(()) +} + /// `ZSTDMT_compressionJob` fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) -> Result<()> { let mut job_params = job.params; @@ -176,7 +191,7 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - let generated = if cancelled.load(Ordering::SeqCst) { Err(ErrorCode::Generic) } else { - reset_struct_table(&mut raw_seq_store.seq, job.nb_seq_bound, false) + resize_in_place(&mut raw_seq_store.seq, job.nb_seq_bound) .and_then(|()| serial.gen_sequences(&mut raw_seq_store, src, job.round_pos)) }; // The receiver is gone if the frame is given up. @@ -312,8 +327,6 @@ struct PendingJob { flushed: usize, /// How many bytes there are in `output` beyond `flushed`. to_flush: usize, - /// How many bytes were received in all. - c_size: usize, consumed: usize, src_size: usize, frame_checksum_needed: bool, @@ -324,7 +337,6 @@ impl PendingJob { fn take(&mut self, progress: Progress) { match progress { Progress::Output { data, consumed } => { - self.c_size += data.len(); self.to_flush += data.len(); self.consumed = consumed; if !data.is_empty() { @@ -604,7 +616,6 @@ impl MtCtx { output: VecDeque::new(), flushed: 0, to_flush: 0, - c_size: 0, consumed: 0, src_size, frame_checksum_needed, From 23e4312df2f8d7986d02d1c04e7e647560cf63c8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:04:23 +0000 Subject: [PATCH 031/199] node:zlib: take the ZSTD_* constants from the crate With zstd.h gone, ProcessBindingConstants.cpp had the numbers, a second copy of what the crate's enums and parameter ids say. It now asks for them. --- src/jsc/bindings/ProcessBindingConstants.cpp | 67 ++----------- src/runtime/node/zlib/NativeZstd.rs | 99 +++++++++++++++++++ src/runtime/webcore/CompressionStreamCoder.rs | 2 +- src/zstd/compress/internal.rs | 2 +- src/zstd/compress/params.rs | 38 +++---- src/zstd/decompress/mod.rs | 2 +- src/zstd/lib.rs | 8 +- 7 files changed, 132 insertions(+), 86 deletions(-) diff --git a/src/jsc/bindings/ProcessBindingConstants.cpp b/src/jsc/bindings/ProcessBindingConstants.cpp index c86f0b7b1d25..bda977d4eaec 100644 --- a/src/jsc/bindings/ProcessBindingConstants.cpp +++ b/src/jsc/bindings/ProcessBindingConstants.cpp @@ -54,6 +54,9 @@ struct NumericConstant { double value; }; +// src/runtime/node/zlib/NativeZstd.rs +extern "C" const char* Bun__zstdConstant(size_t index, double* value); + // One loop instead of a `putDirect` call sequence per constant; these objects // are each built once, lazily. static void putNumericConstants(VM& vm, JSObject* object, const NumericConstant* constants) @@ -1153,70 +1156,12 @@ static JSValue processBindingConstantsGetZlib(VM& vm, JSObject* bindingObject) { "BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2", static_cast(BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2) }, { "BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES", static_cast(BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES) }, { "BROTLI_DECODER_ERROR_UNREACHABLE", static_cast(BROTLI_DECODER_ERROR_UNREACHABLE) }, - { "ZSTD_e_continue", static_cast(0) }, - { "ZSTD_e_flush", static_cast(1) }, - { "ZSTD_e_end", static_cast(2) }, - { "ZSTD_fast", static_cast(1) }, - { "ZSTD_dfast", static_cast(2) }, - { "ZSTD_greedy", static_cast(3) }, - { "ZSTD_lazy", static_cast(4) }, - { "ZSTD_lazy2", static_cast(5) }, - { "ZSTD_btlazy2", static_cast(6) }, - { "ZSTD_btopt", static_cast(7) }, - { "ZSTD_btultra", static_cast(8) }, - { "ZSTD_btultra2", static_cast(9) }, - { "ZSTD_c_compressionLevel", static_cast(100) }, - { "ZSTD_c_windowLog", static_cast(101) }, - { "ZSTD_c_hashLog", static_cast(102) }, - { "ZSTD_c_chainLog", static_cast(103) }, - { "ZSTD_c_searchLog", static_cast(104) }, - { "ZSTD_c_minMatch", static_cast(105) }, - { "ZSTD_c_targetLength", static_cast(106) }, - { "ZSTD_c_strategy", static_cast(107) }, - { "ZSTD_c_enableLongDistanceMatching", static_cast(160) }, - { "ZSTD_c_ldmHashLog", static_cast(161) }, - { "ZSTD_c_ldmMinMatch", static_cast(162) }, - { "ZSTD_c_ldmBucketSizeLog", static_cast(163) }, - { "ZSTD_c_ldmHashRateLog", static_cast(164) }, - { "ZSTD_c_contentSizeFlag", static_cast(200) }, - { "ZSTD_c_checksumFlag", static_cast(201) }, - { "ZSTD_c_dictIDFlag", static_cast(202) }, - { "ZSTD_c_nbWorkers", static_cast(400) }, - { "ZSTD_c_jobSize", static_cast(401) }, - { "ZSTD_c_overlapLog", static_cast(402) }, - { "ZSTD_d_windowLogMax", static_cast(100) }, - { "ZSTD_CLEVEL_DEFAULT", static_cast(3) }, - { "ZSTD_error_no_error", static_cast(0) }, - { "ZSTD_error_GENERIC", static_cast(1) }, - { "ZSTD_error_prefix_unknown", static_cast(10) }, - { "ZSTD_error_version_unsupported", static_cast(12) }, - { "ZSTD_error_frameParameter_unsupported", static_cast(14) }, - { "ZSTD_error_frameParameter_windowTooLarge", static_cast(16) }, - { "ZSTD_error_corruption_detected", static_cast(20) }, - { "ZSTD_error_checksum_wrong", static_cast(22) }, - { "ZSTD_error_literals_headerWrong", static_cast(24) }, - { "ZSTD_error_dictionary_corrupted", static_cast(30) }, - { "ZSTD_error_dictionary_wrong", static_cast(32) }, - { "ZSTD_error_dictionaryCreation_failed", static_cast(34) }, - { "ZSTD_error_parameter_unsupported", static_cast(40) }, - { "ZSTD_error_parameter_combination_unsupported", static_cast(41) }, - { "ZSTD_error_parameter_outOfBound", static_cast(42) }, - { "ZSTD_error_tableLog_tooLarge", static_cast(44) }, - { "ZSTD_error_maxSymbolValue_tooLarge", static_cast(46) }, - { "ZSTD_error_maxSymbolValue_tooSmall", static_cast(48) }, - { "ZSTD_error_stabilityCondition_notRespected", static_cast(50) }, - { "ZSTD_error_stage_wrong", static_cast(60) }, - { "ZSTD_error_init_missing", static_cast(62) }, - { "ZSTD_error_memory_allocation", static_cast(64) }, - { "ZSTD_error_workSpace_tooSmall", static_cast(66) }, - { "ZSTD_error_dstSize_tooSmall", static_cast(70) }, - { "ZSTD_error_srcSize_wrong", static_cast(72) }, - { "ZSTD_error_dstBuffer_null", static_cast(74) }, - { "ZSTD_error_noForwardProgress_destFull", static_cast(80) }, - { "ZSTD_error_noForwardProgress_inputEmpty", static_cast(82) }, { nullptr, 0 }, }; putNumericConstants(vm, object, kConstants12); + double value; + for (size_t i = 0; const char* name = Bun__zstdConstant(i, &value); ++i) + object->putDirect(vm, Identifier::fromString(vm, ASCIILiteral::fromLiteralUnsafe(name)), jsNumber(value)); return object; } diff --git a/src/runtime/node/zlib/NativeZstd.rs b/src/runtime/node/zlib/NativeZstd.rs index 2c69e3630e33..5047cec4902f 100644 --- a/src/runtime/node/zlib/NativeZstd.rs +++ b/src/runtime/node/zlib/NativeZstd.rs @@ -1,5 +1,104 @@ pub(crate) use _impl::NativeZstd; +/// The `ZSTD_*` members of `zlib.constants`, in Node's order, for `ProcessBindingConstants.cpp`. +mod constants { + use core::ffi::{CStr, c_char}; + + use bun_zstd::{ + EndDirective, ErrorCode, Strategy, ZSTD_CLEVEL_DEFAULT, ZSTD_D_WINDOW_LOG_MAX, c_param, + }; + + struct Constant { + name: &'static CStr, + value: i32, + } + + const fn named(name: &'static CStr, value: i32) -> Constant { + Constant { name, value } + } + + const fn error(code: ErrorCode) -> Constant { + named(code.name(), code as i32) + } + + static CONSTANTS: [Constant; 61] = [ + named(c"ZSTD_e_continue", EndDirective::Continue as i32), + named(c"ZSTD_e_flush", EndDirective::Flush as i32), + named(c"ZSTD_e_end", EndDirective::End as i32), + named(c"ZSTD_fast", Strategy::Fast as i32), + named(c"ZSTD_dfast", Strategy::DFast as i32), + named(c"ZSTD_greedy", Strategy::Greedy as i32), + named(c"ZSTD_lazy", Strategy::Lazy as i32), + named(c"ZSTD_lazy2", Strategy::Lazy2 as i32), + named(c"ZSTD_btlazy2", Strategy::BtLazy2 as i32), + named(c"ZSTD_btopt", Strategy::BtOpt as i32), + named(c"ZSTD_btultra", Strategy::BtUltra as i32), + named(c"ZSTD_btultra2", Strategy::BtUltra2 as i32), + named(c"ZSTD_c_compressionLevel", c_param::COMPRESSION_LEVEL), + named(c"ZSTD_c_windowLog", c_param::WINDOW_LOG), + named(c"ZSTD_c_hashLog", c_param::HASH_LOG), + named(c"ZSTD_c_chainLog", c_param::CHAIN_LOG), + named(c"ZSTD_c_searchLog", c_param::SEARCH_LOG), + named(c"ZSTD_c_minMatch", c_param::MIN_MATCH), + named(c"ZSTD_c_targetLength", c_param::TARGET_LENGTH), + named(c"ZSTD_c_strategy", c_param::STRATEGY), + named( + c"ZSTD_c_enableLongDistanceMatching", + c_param::ENABLE_LONG_DISTANCE_MATCHING, + ), + named(c"ZSTD_c_ldmHashLog", c_param::LDM_HASH_LOG), + named(c"ZSTD_c_ldmMinMatch", c_param::LDM_MIN_MATCH), + named(c"ZSTD_c_ldmBucketSizeLog", c_param::LDM_BUCKET_SIZE_LOG), + named(c"ZSTD_c_ldmHashRateLog", c_param::LDM_HASH_RATE_LOG), + named(c"ZSTD_c_contentSizeFlag", c_param::CONTENT_SIZE_FLAG), + named(c"ZSTD_c_checksumFlag", c_param::CHECKSUM_FLAG), + named(c"ZSTD_c_dictIDFlag", c_param::DICT_ID_FLAG), + named(c"ZSTD_c_nbWorkers", c_param::NB_WORKERS), + named(c"ZSTD_c_jobSize", c_param::JOB_SIZE), + named(c"ZSTD_c_overlapLog", c_param::OVERLAP_LOG), + named(c"ZSTD_d_windowLogMax", ZSTD_D_WINDOW_LOG_MAX), + named(c"ZSTD_CLEVEL_DEFAULT", ZSTD_CLEVEL_DEFAULT), + named(c"ZSTD_error_no_error", 0), + error(ErrorCode::Generic), + error(ErrorCode::PrefixUnknown), + error(ErrorCode::VersionUnsupported), + error(ErrorCode::FrameParameterUnsupported), + error(ErrorCode::FrameParameterWindowTooLarge), + error(ErrorCode::CorruptionDetected), + error(ErrorCode::ChecksumWrong), + error(ErrorCode::LiteralsHeaderWrong), + error(ErrorCode::DictionaryCorrupted), + error(ErrorCode::DictionaryWrong), + error(ErrorCode::DictionaryCreationFailed), + error(ErrorCode::ParameterUnsupported), + error(ErrorCode::ParameterCombinationUnsupported), + error(ErrorCode::ParameterOutOfBound), + error(ErrorCode::TableLogTooLarge), + error(ErrorCode::MaxSymbolValueTooLarge), + error(ErrorCode::MaxSymbolValueTooSmall), + error(ErrorCode::StabilityConditionNotRespected), + error(ErrorCode::StageWrong), + error(ErrorCode::InitMissing), + error(ErrorCode::MemoryAllocation), + error(ErrorCode::WorkSpaceTooSmall), + error(ErrorCode::DstSizeTooSmall), + error(ErrorCode::SrcSizeWrong), + error(ErrorCode::DstBufferNull), + error(ErrorCode::NoForwardProgressDestFull), + error(ErrorCode::NoForwardProgressInputEmpty), + ]; + + /// The name of the constant at `index`, whose value goes to `value`, or null past the last one. + #[unsafe(no_mangle)] + extern "C" fn Bun__zstdConstant(index: usize, value: &mut f64) -> *const c_char { + let Some(constant) = CONSTANTS.get(index) else { + return core::ptr::null(); + }; + *value = f64::from(constant.value); + constant.name.as_ptr() + } +} + mod _impl { use core::cell::Cell; use core::ffi::{c_int, c_uint}; diff --git a/src/runtime/webcore/CompressionStreamCoder.rs b/src/runtime/webcore/CompressionStreamCoder.rs index f64093107c0c..65080f81894f 100644 --- a/src/runtime/webcore/CompressionStreamCoder.rs +++ b/src/runtime/webcore/CompressionStreamCoder.rs @@ -248,7 +248,7 @@ impl CompressionStreamCoder { (Format::Zstd, false) => { let mut cctx = Box::new(zstd::CCtx::new()); if let Some(lvl) = level - && (cctx.set_parameter(zstd::ZSTD_C_COMPRESSION_LEVEL, lvl)).is_err() + && (cctx.set_parameter(zstd::c_param::COMPRESSION_LEVEL, lvl)).is_err() { return Err(CodecError::Message("failed to set zstd level")); } diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 0ae2205bd39e..ffd4868daab7 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -23,7 +23,7 @@ pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53usize * 9 + 36 * 9 + 32 * /// `ZSTD_strategy` #[repr(u8)] #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] -pub(crate) enum Strategy { +pub enum Strategy { Fast = 1, DFast = 2, Greedy = 3, diff --git a/src/zstd/compress/params.rs b/src/zstd/compress/params.rs index d062b759acb6..e003c587677b 100644 --- a/src/zstd/compress/params.rs +++ b/src/zstd/compress/params.rs @@ -52,28 +52,28 @@ pub(crate) const ZSTDMT_JOBLOG_MAX: u32 = 30; pub(crate) const ZSTDMT_JOBSIZE_MAX: i32 = 1024 << 20; /// `ZSTD_cParameter`, as far as `node:zlib` lets a parameter through. -pub(crate) mod c_param { +pub mod c_param { pub(crate) const FORMAT: i32 = 10; pub const COMPRESSION_LEVEL: i32 = 100; - pub(crate) const WINDOW_LOG: i32 = 101; - pub(crate) const HASH_LOG: i32 = 102; - pub(crate) const CHAIN_LOG: i32 = 103; - pub(crate) const SEARCH_LOG: i32 = 104; - pub(crate) const MIN_MATCH: i32 = 105; - pub(crate) const TARGET_LENGTH: i32 = 106; - pub(crate) const STRATEGY: i32 = 107; + pub const WINDOW_LOG: i32 = 101; + pub const HASH_LOG: i32 = 102; + pub const CHAIN_LOG: i32 = 103; + pub const SEARCH_LOG: i32 = 104; + pub const MIN_MATCH: i32 = 105; + pub const TARGET_LENGTH: i32 = 106; + pub const STRATEGY: i32 = 107; pub(crate) const TARGET_C_BLOCK_SIZE: i32 = 130; - pub(crate) const ENABLE_LONG_DISTANCE_MATCHING: i32 = 160; - pub(crate) const LDM_HASH_LOG: i32 = 161; - pub(crate) const LDM_MIN_MATCH: i32 = 162; - pub(crate) const LDM_BUCKET_SIZE_LOG: i32 = 163; - pub(crate) const LDM_HASH_RATE_LOG: i32 = 164; - pub(crate) const CONTENT_SIZE_FLAG: i32 = 200; - pub(crate) const CHECKSUM_FLAG: i32 = 201; - pub(crate) const DICT_ID_FLAG: i32 = 202; - pub(crate) const NB_WORKERS: i32 = 400; - pub(crate) const JOB_SIZE: i32 = 401; - pub(crate) const OVERLAP_LOG: i32 = 402; + pub const ENABLE_LONG_DISTANCE_MATCHING: i32 = 160; + pub const LDM_HASH_LOG: i32 = 161; + pub const LDM_MIN_MATCH: i32 = 162; + pub const LDM_BUCKET_SIZE_LOG: i32 = 163; + pub const LDM_HASH_RATE_LOG: i32 = 164; + pub const CONTENT_SIZE_FLAG: i32 = 200; + pub const CHECKSUM_FLAG: i32 = 201; + pub const DICT_ID_FLAG: i32 = 202; + pub const NB_WORKERS: i32 = 400; + pub const JOB_SIZE: i32 = 401; + pub const OVERLAP_LOG: i32 = 402; } /// `ZSTD_format_e` diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index bc932de17ddf..88906b1c9d4c 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -34,7 +34,7 @@ const ZSTD_MAXWINDOWSIZE_DEFAULT: usize = (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + const ZSTD_NO_FORWARD_PROGRESS_MAX: u32 = 16; /// `ZSTD_d_windowLogMax`, the only parameter of a [`DCtx`] that `node:zlib` can name. -const ZSTD_D_WINDOW_LOG_MAX: i32 = 100; +pub const ZSTD_D_WINDOW_LOG_MAX: i32 = 100; // ── frame headers ───────────────────────────────────────────────────────── diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 0c3995e8ac8e..530eeab2aea0 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -20,10 +20,12 @@ mod util; pub use common::ZSTD_CONTENTSIZE_ERROR; use common::ZSTD_CONTENTSIZE_UNKNOWN; -pub use compress::params::c_param::COMPRESSION_LEVEL as ZSTD_C_COMPRESSION_LEVEL; -pub use compress::params::{ZSTD_CLEVEL_DEFAULT, compress_bound}; +pub use compress::internal::Strategy; +pub use compress::params::{ZSTD_CLEVEL_DEFAULT, c_param, compress_bound}; pub use compress::stream::{CCtx, EndDirective, compress}; -pub use decompress::{DCtx, DDict, FrameExtent, frame_extent, get_frame_content_size}; +pub use decompress::{ + DCtx, DDict, FrameExtent, ZSTD_D_WINDOW_LOG_MAX, frame_extent, get_frame_content_size, +}; use decompress::{decompress, find_decompressed_size}; pub use error::ErrorCode; From 553042bd4c4704ff538ddd1420f05457ebe97d86 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:04:23 +0000 Subject: [PATCH 032/199] zstd: test frames that no encoder makes, damaged frames, and abandoned streams 116 frames from zstd's decodecorpus, each whole and damaged in nine ways, through node:zlib and through Bun, against what zstd 1.5.7 makes of them. --- test/js/bun/util/zstd-frames-fixture.json | 466 ++++++++++++++++++++++ test/js/bun/util/zstd.test.ts | 45 +++ 2 files changed, 511 insertions(+) create mode 100644 test/js/bun/util/zstd-frames-fixture.json diff --git a/test/js/bun/util/zstd-frames-fixture.json b/test/js/bun/util/zstd-frames-fixture.json new file mode 100644 index 000000000000..e5b07fbe3c93 --- /dev/null +++ b/test/js/bun/util/zstd-frames-fixture.json @@ -0,0 +1,466 @@ +[ + { + "frame": "KLUv/QQmfAQAihAYARGwJekMZykICx3exGGUMUlyGgsACwANAHu32//Zdz27iv//9d/d+f/v4///ZgPrLJ/f713etje3e+IB//v/8ipeL/nf/j8aICCEEKRcH7YGTCGMw0GrcAdaEiTQ/TOYGxS1XSkQMCQgoxpAg/uIp8nleTaAZvbZQUHjJiXuNSIggPDACga/VVhmbgg8AgCbBFQABAAEAAQA8bd/Bf3a+wG7/24B3S7fC6iQKKTgjn0HECDBQ+wBEEgUQxRQvAz77zSBNlDa6dnlDd0QIOWvVwsFDPfZAgwCAA8JQAgABgAIAAcAb1P71/8/sWFu9jevUw9/X3fcaX4W+Td/z3ffBagQJsWe8NJlDxCIsR9wXrhwF9FZlv0M0ggYAAAAaAAAwEFeQd26rAikQUFeQRwAACGNACAAAAAnXwsAAAAJAADTJM6/Ig==", + "results": "803 1eb85d07,803 1eb85d07,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,612 808c5c76,ERR_ZSTD,803 1eb85d07,ERR_ZSTD,803 1eb85d07,ERR_ZSTD" + }, + { + "frame": "KLUv/QRIFAQAMskRHJAVFCTpn79ATN+tjW10TvMF24iNbnRO+/X4lgh5M5G3dVIgtY6gwB3d5EStexsL5BlAiBsyXBcOnGlOB2MSMyx6JyNHEggPqJBqZiZNdqnLARAcgYYPMMoEKZiZyq7WPwOpFSlX+fdJ0SO8eIBg91Bib8C+gcz7sfc1RLQuCwUBAIODBFuCyjKeTZYA1glO0EGTVCUsAgMAQAMSuBbkXp5EloX6Ww==", + "results": "430 b9de9245,430 b9de9245,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,430 b9de9245,ERR_ZSTD,430 b9de9245,ERR_ZSTD" + }, + { + "frame": "KLUv/QRTBRcANv+CjhEREREREREQfwB/AH8AkEAMir8VENRVlONP/bC9/OXrU9FrnA8VYM9zz4/NwjWeZxRLKoMtU/mVbG+V4Yu/JXvui7gMXaPVkGk5YkdwB1YQHTnup8Ag9Zh8amot540JRii1h+2GW3fXqAzIBUyEI+LPfKgO5uHWcF8ysBe3i4rRuqREbF1gdlsGOL9LGkb77tLHVOCWruEj0VfNBzxPllK9Ag9E0wzLnVwhow+D0wT4VUdifMCRNUKWdZNMB7S8aNMmNM99ABnRgnEEPLS0SPFAd9YOolxQa3jMuQnZZhxqK0hgQYp9VirQh7RQdLvrNFblslWncAhS6uXK7jszTcDzld4IkqCvOiXBlhCXF/fyfRmfZoYO/5pCrJBwSAdFbwQ7sRP83a09NPTi9yD73S7CXxIUAJvsLYybV4hqGq7WJuHBfp//vETum7/VIe/jMrGSOjOnaYgz3IKNmTUXzgJZjYfPr/MPdKgmrZyhoZAdW9weJRW9Cs06E/RXrQjgco+csP2R2OyMO0UfQHtPuu8NS713krOBPIec5zmqqc5rqEHkyrelTz8SKiS6/45MDLC8h2yEPZUqCyGW6Tk3t7G5zuk6g7S9dBIRTU0iXukXKQnQAcqinoQwx0/7kIelEtd5vtwuKQAKANk8yfn0yraP+4oMwvYpm8sFdh9aK/yOEex7LC57m55fAUsoMIiIDNXdAYKAGI2UJI0ZY4TQkM2ECcssbTGNCCdb+fgeOzZgM19HAXKgjkW2+GHx0Zm9jGAgShlHVNg0E4g3hvqH44XrUi483sFEliOVbR3Sd+6SzplnEhKPPNBaJmpLXG0DxMViz3lL7DoAsH5d0Xanerl6Z6CRadMgyDJK9ke8pfjmbmG7yqrOGK4RQddUiy6gIpVBcIiRFje8bPcSJDzH4HJbi+bDRRjvQw2iCg8RxR/HN7EJK5IRT3Li2YMFZBOsFVuePxuQkpJ7OoDNKyEn/KM=", + "results": "1833 456abcfa,1833 456abcfa,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,1833 456abcfa,ERR_ZSTD,1833 456abcfa,ERR_ZSTD" + }, + { + "frame": "KLUv/cQ2NgYAAAAAAADEBAAyzRyOERERERERERAp0lvBh+qowsqs4pI2OSjia7lDt4ChbxBpTwtjI7K/iuSnc5ZLRPjgAHLhO1PZEYyVu21nd0GN8UBKxIoPMJxQfNpwuf8esBxBADKD//7Vf4QNP5mTzTxLgKcyCzpRLDf8JaJkguwRzDgbA6gUgFYB76rAf/8T0FUAq6A/FMBWAbsq8M8/Ryg1Z8hLj70FAG4awCEAGoAVJEk6MxXTAX0DM0ZmGdR1zsKe4hJ6JAS+GQAbABgAfwvhgBD0fI/VGMlLasfuxsM6vH7WXMrbuH1QVFzM5lGixdYeJteDVhWa6hWiTLrctfCuC/9fXLcwa5KQPxH9ZtVqk/NulttbqiHOBtfrlGy8bknAiSwIyrfQi3ydELWuD/X2XskBCQhQAlLiH4r/q6gCpcYBA3jzhj4RC/0ILxAg9ErmZOQXnJsm6xnhpNUMEm+GOAw=", + "results": "1590 cf337341,1590 cf337341,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,257 7a917cae,ERR_ZSTD,1590 cf337341,ERR_ZSTD,1590 cf337341,ERR_ZSTD" + }, + { + "frame": "KLUv/QRHHAMABkoJBuAPxw4gyAYABgAGAP//////Af//3/e/D/2++///H////9/ODxOoYMpQzUSFxa2WAxAoY1TtAXCCgmbCLgcgQKTwRO04jcpK7Lh4bokE/naFVtgbEbzLMj1bbaAn8HwBwgUAwDAAAPjFwJAKHwgAAE8AAAAJAADv2oEcaw==", + "results": "482 4f2d578c,482 4f2d578c,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,482 4f2d578c,ERR_ZSTD,482 4f2d578c,ERR_ZSTD" + }, + { + "frame": "KLUv/URbNw8UEADSAwqOERERERERERBOqpJoHuEQhc7pIYoWGlDF8NyLAiQMukBUNLZwEhsQgPaokM8BIIgQmTG0A4EsyCRDNCMjSVJpDgtsQ6NoNPkEMiEXa0UVcEBq3JB3E8fYd8xPSye5q8R4vEgyiu6q0oGyr0mCneZqTofvCtKpd5bRbIMUN+oOuCXg5atl1JYY71YMOImTu2KZAj1OFzXtCJbUmsMQ5F6m00OF3wIeoKWh4+Ky5FJPA/2ikCjeUNwsaIJC1gydtWd8ESd32y+kYb1d+4wfLTJHJDQuBuZYQHan7CQ9aAQTJOQ27jXD9C+jKcpm/1av71nd/T3EILX2wjquqClP05vxC/a/4REKVjcP22g0YUSgD6I2RNz2AS/ccHLdQarZsDVDaRQsAqZWMKF2RG7gmTifDIXQ1RZr/YE/m5gWs6OCZxYQeJCPDnQuhzs5ZQy2FwdHFuj2l1ctYyEIE5yJ7RZ4DrJPhQhzA/Ant6fSeeG0YB/EgDXrv7bBm77x1lc/tHnvGAAUwkBq+10w1rjNO8WMpp9ry5Xdmlwguni91yheDHU+q3w0FGZApSOom+At7Dqv2itUSRcwdeUUIrdzEot4su4JrkR0oB60ZM9FYnYPyUKN7twjapAR31R5wVbhMHwMEQ0DWdOzJYMTqo+rdVBRNyVCcAk8RrfwJuQzGUcBJAcA/w9AIgAhACEAIQA2Yk67JwAf9geZeyIj/tU7HqTUMW4ionoPZyEXNKr9AwFwbmI73HzRbU/oGdlxOn18ejtBw/RKDsZJKmROijyapAGe7KFa3Z6ptuCIz+YA42HcFuL4jeNhIrbEfAJdkZq8NgFLS8vhcy7p4a32/L84mFnz3CwfeOVl3y3sbpnB8hbGEh+oUIIyJBKhJGmWAzBK7KQPQF7NNo0BfBgGXHOg5VWlAG/WECYrCWXU7LEpPnsfttxg5Xl5EyJWi9muRHjim4vOvMPLcJ8RrstGV5OtzOaRFXxICAJaoAIASOkvJlo7CjTq3QsLL2hd5LGzCYW+pzHAac8+GIGqSjNYvSr0bjVI3O/igcPo7/paCRLy3vBCxzyYThyHNYdIaTwUDc7JQNxBGh8XgZLHxBy4yvWRAAAAFQcAzhGAJQAigCVJ0gEgywzseK5lMiHAeYs8M7A7GzfNDOxwnmUyIeA0Mx0AGwAaAMsN5sonrb7+BSND17YEicfc1H+w4Zi8HzDAO0AGvzD9Frx/unVJDFx09iDQKD/COb6vmoJCvM0OAoHZWKUJ878r6FAdnM5p2707cu2/tnC/cAgrppY++BsSliy/wwvG9ixJnu8m/OYpD/zPsB4SKGDiU+8wwpT4n4lnoploghnbAdKDscRgvRKmlAu2wBeVS1zNg46YZ3R6RcmTBbRdr1dLMCuXYlS9vRBaU55q6+Rbymxy2mL16UE=", + "results": "4151 5343f820,4151 5343f820,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,2447 7791b47b,ERR_ZSTD,4151 5343f820,ERR_ZSTD,4151 5343f820,ERR_ZSTD" + }, + { + "frame": "KLUv/QRQJAQA4lwRBuAPWHFRRLf//7n/+//////+7/////f/v/+/9///////////z//fv///vv79///9/vX3///+1///9/////////////8DD6iw4iaYeCaaubQDEBghNGgPUIJR0oSJS9v6Z7j8/QA61PIVRkvkgwOlVOB1JShO/y+wyhLe9LL1W1QG2gMAVoUBABoGmAAO0GU4ACyFGIAiwMEW5wEFAAQABAD73P6+Af//zxf7499//X/PBwFUAAUTNHyF4Es=", + "results": "867 e531346a,867 e531346a,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,748 223bca28,ERR_ZSTD,867 e531346a,ERR_ZSTD,867 e531346a,ERR_ZSTD" + }, + { + "frame": "KLUv/UQKsggAAABcFwCOEsApAI4RERERERERECYAJgAmACZdtqp0ktFsBGe+HbTIXNN4OXSkUomDH8HkhI5omD4jYSizM5oB3rPO7K+09WVBeJvyIyn4QsZAu6mG5qGpRGTEhWJL931tOQLAMwHR8MhqZEMjEfc7dosMZPtbG5zwoNO3N7c7eeS3hdZ7vFyjjbypAWtfIjZXrpJRat1XYMItRT1pstruuF2KOU7Xi8Ik9cuiQLKajBkBgVGo8dXkBkEYYhCDEtwD8eGdMWIESLnZ0WtQLALlHskME8eMsokEh6IWr4RaE6OCCkP9dRgIWhoZLLRipBj4FgXVwlJbt7AoYhRAcA1VaEv5k7pJvBR1qJFcAaiCYYiNRxMRL0LwdfgWWMSctgjTokbg5+NVDGNlnkjDeQZoJoTwFP+QN4u/6VjqLKb+zDKE+s6RIdgbr53a+t2AqB3CQqOtQnKxhHKi7aBNCraH2fk8a6HeVQ2tBSINZg2BiovKV32VLNFCGeEMFgTjN4yzLGDKXlgffgTjaiR8Az7CXqqHQjYtQuIhlpCipgdbMdOmLD7mOgO1wkO3vErAgv8dteFwxs77nOM79lXmwiPaiGS2S+cUL01eVJE4iv7FckUu58j9XfnfjImklTwiDvDs4A2lg3jXpMhvmJVtuqTt5jygBcihEGgxwHjQY2DDjZJZLyLHb0WHndekey7i6dJH8tCSqH8s+IH4HREyhW6B6mwLdQLyBe00nEmA/6c3+15rcuRJd70yCrEU/wB7bYB58ThoWzD/UjDyC7j2bXMAyrZAhZC3xqKXefNYHNVohKUv5nQYMBXxcxQGInZO8WXBLJhLsSIj3nt0/mYEJeTJjh+kZqyotjMLNs0brp1M3B6KYApJQc9F8BW8fX7XsRu5vErq/4IXGphOTEFEJpHIo8oth1ZCpNCBkWcdvN7iG4FEmIEAlP2gNLSuVGwG4gl+mmNMlHMFsH4piAdtL+c0qxQOJ1+P7xBkaAwBeyxkrA4UBQBrCVQBFAAUABQAD6Mkpe4JCKY8+Am/TzRhy8GORAGYTnj5i/WIo20wdoi0WQhJTGYqAZN6X1SJIVBH21ubHKttlnMDz7EBGyP1qv7/7Bii4nD26To89DH6AROogMpYOBUWV6sBgCaYcNAeMMQIRjAyE8pMPJSWHyYPZHKQpwDreH1M+qagDWhDY9mj69NvdcEJ6lCqWErO0IgIB1jkV/owAumFAQCuCMAIAAjwOS8F+z8RAQUABQAFAP////8P/5///3D/////D//8//8JAVQTCiqJygFH/ROH", + "results": "2482 de10fc0a,2482 de10fc0a,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,2482 de10fc0a,ERR_ZSTD,2482 de10fc0a,ERR_ZSTD" + }, + { + "frame": "KLUv/QRReAYArsDi//amUBIzmYBRfcFzF215S1YTAbfl1zM+1O1R+/wbhsKYefpH2fpCdZF0KXQBoPdwe/vWn+Y1M+ulCOvg73yEpF1SkryUCinvz3YA6jkkDhHzF7VpJdIbLqwlPIwoB4x9w2rvLeNqE2Mxad1tqcUcme/rJLpUyMG6iT04SItb2MRxSoWvJ6Z79u0DRAwTizngCMeIxxLNudDuVyQL4Gaz/kQxv4iPNVgbxjqvZ6BBi29T8YGpFg750fIN9SJGcJ/32SkiLustpH50h+jpUQAAtiX8CGzoXoa/MFuMI7I=", + "results": "217 bc83152c,217 bc83152c,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,165 9420d09b,ERR_ZSTD,217 bc83152c,ERR_ZSTD,217 bc83152c,ERR_ZSTD" + }, + { + "frame": "KLUv/QQBUAAA0am8vb29vb29vXEAADDYqLIGvb29vYcGvb0rbFU6Zw==", + "results": "24 f4e73620,24 f4e73620,24 f4e73620,24 f4e73620,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,18 e5061c8,ERR_ZSTD,24 f4e73620,ERR_ZSTD,24 f4e73620,ERR_ZSTD" + }, + { + "frame": "KLUv/cRbBwAAAAAAAAAwAABo7WjtaL8AAAAJAADzJDjxww==", + "results": "7 6d8b4702,7 6d8b4702,frameParameter_unsupported,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,checksum_wrong,ERR_ZSTD,6 952b8efb,ERR_ZSTD,7 6d8b4702,ERR_ZSTD,7 6d8b4702,ERR_ZSTD" + }, + { + "frame": "KLUv/QQffQYAShgIAh1wpSBJ0gH86/+8CgA9oao6/b+uwv6v0+u/XD2FwRgAGQAWAJXuEHSX8ayCj23TahaacwTcXdwW9q3KBnH8yndEoR16IHdcfcKD6Rhv8faZMgiEHirjHn2ttN///ZcNatFfxE6lCqKPCwI7ixJzn5o8aEVwCGN7z6f/5Gct88xTegMWoHBGAhoJJpwYGdIUGxAggWFo8AEnlPTQ9k8SWh1LgTDDy1ZPxgsrbT1y8GmgSxCACgXBbHa8OuIllB8cyV9fwYpAUcYfGAAGp1F/bQ==", + "results": "650 802d9d4f,650 802d9d4f,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,650 802d9d4f,ERR_ZSTD,650 802d9d4f,ERR_ZSTD" + }, + { + "frame": "KLUv/QRSEhoAACwKAG4owCYAE6AlhXT/P+s/+vd///0/tP0/UjkgACQAHgAV9S6onk+36jbzYfrK/9g8vudpVb8/75gLe+28/386T/1oufsRvOs1RINU6JbxfFB8HDK39+knbVf3tz2fIX1blfxSMq1fdu9/Na3b4Ow2anvPvr/+hftZX1W+8ewn/c7/X3uKT/uZIPv19mZ4/lfVV58Z92/q/pdctv1bePYv/xYviGEQZRFEiBKGCFKIqCgdURAGJAwpJVFJRIGSooKSSoY5SAVkHBOADh1yjZhkM0w+/iyBwro0gGU7V5IspJQkZR7NnyzhbMbhmiOGIufESq7+5ocblkJGDEF82ZbeVMjxSW5Pi6tLQa0zs4uU9xCykvYVRYNa3100Di3pG1S4yFcHmQSOLgURpyEy1BVAbN1ZeC5mK3FAiorkblAUs7YJVeiJiiikAgBCQgQL4BkOMxYAAAAIkZL/v//9fxkoIEocMdUHMNUDywigETvoKSgD6BpdNwt1XDVKoYAkfmlKdcQenVFqHzwmajkWIIgNQBERY5QOlvXrbFYwkg1VBQCvDcAJAAkACAAIAL/7/7r/f5/9Af9///93//cP////z/75+z/3//+//b/9BysAcgjSAeCJYeUBBuQECFxSIER6YACoz4DdgOOUAbwoAwAYENA1AIDGKQDCyYDdIIOAwfGQ4Cqmo79EgPcqSAFglg+4KWYgEIydAQhyAXcDtwMCzH3AKDcBtCdgE44HARjAqmRARZEB5ykHCwDDFxn4amdg4kjgQA9gUAE5Dmf0uHY=", + "results": "2631 ff0a9765,2631 ff0a9765,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,2266 a3492beb,ERR_ZSTD,2631 ff0a9765,ERR_ZSTD,2631 ff0a9765,ERR_ZSTD" + }, + { + "frame": "KLUv/QQCOgUA2bgFAAfZO5jyem3VUbGQGmiqd64U/AJ4O1amY54juJWoqSDc8SJFfO72Hy67rDxIzhAbeaiWeWs6dD2PQ3ii8+KVN6Yci7TPjED4Y1dDGN9fAmU7RS50Qbif7lpyEsyaDdos3hHCn4LEpz0qHmx7FZ6ZlZiDWlqmjqs+8W71cgNpN6zB1S9GXS+y7QUbV/oYgurMcRtlLmflI9Ta8MJqp/JLSsjBADeToQknCihIbltQEnqnikt2EAZzIvADAPD/Gi6j7Mhtp9Tu8WEV3ArnSGa3WI0Wg5OHoC1xTB/o06T3vKu+1Z1Y+rmeHpqtP4MOtqceq75fd9K8Pewuh4jyuIS64p7jb6+0gLKejb9E4b+f+t4G+SMi+2CfZdpi4LxztLQ6LV8sUSwOl4oJ+Y8yeCyVjynrac1UVGdeMAAAAFEAAFcZXjBXwtnZ2dn7Apho", + "results": "486 52e1be8a,486 52e1be8a,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,496 740d3515,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,409 e7c87075,ERR_ZSTD,486 52e1be8a,ERR_ZSTD,486 52e1be8a,ERR_ZSTD" + }, + { + "frame": "KLUv/QRKAgAAR+QGAHJYM44REREREREREN8bnIvc4C4jFAMDiKdfvl5oH7KeIsOjfWMF0bX+yDWkqiMUMunSX4AFQs4xaZedz6WKpRunoaEDuCBv3iWLqCNVkBxunM0BJ/HqqNCLrAQ5T++KQekrzYmUkNM3aAbsMPRJOphq4eZAkTwban8SvwUUwbmrHiLpD/J/mP/HkPj8SYrIzHw/bahbvHYNWfaYtJ3PVxoCJgLgikZaSbSyrPo1QO7ZBmkqhKyqMxvS08YS7cA3VIwg3eHN9dOqKAIGaDqMpB8DAHz5bqVTZ4I6gSsZBQCT6S/9Mka7Igt7A69+XHkhlfvUGaYqGF5j0l1MaBBAZx3DexUjeT8OeYjKTGbQpvDI9Z11WCVbQwr/0hS0Gbvce6LOoGWrjTBQPV9gMcMIDJ9IACe0G0+VFGnHmR4aLc24gER+gtbSqiM8AnH8hViY+Rq6oR9rMaxv3LMvKcwinY+6/bpnPjV6ObZyJjRbfsdPj5GzKY3T3Q9C60T73wMAlIDudLeZKA==", + "results": "790 c7c4b7be,790 c7c4b7be,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,693 2857d7d8,ERR_ZSTD,790 c7c4b7be,ERR_ZSTD,790 c7c4b7be,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYfAcAAlM5N3AnSdIBcQXN24hxBZ3HiPEr6jxGjF8jymPE5VlEeYy4PIsojxE9YxFFAaJnLCJSQHvGYqqj3U0HyXiYHEy8HSaW9cA6BEz7LdJXi6Cizj35mPvPEC2l+KydEWsGSKSUjDtWlf80fC7ojHOM0jyE1DhnUQMf0LO8Ax1PexOrJA2GPEXo216sIHYMAIwN+s8K6uTLHUVgxp+It2eZfk4zvvWnOQZLxtLRs7LZXlpfXBZQoqaBZzeCgSlof8OsZ77zxZBB8WYgS7AsFxnjVYofXF/hWXoRpvCTbZeJypDF2oHciSsVIgFUGQAoNwdsAACRoQIkExjYfwVX2EAL7QEA6wKAAAcABgAHAKa2sidJ8HS3S8ZLueiYP5xnUpEDqxJlabWoA6AUoFYBP6sC+w8TgFXg/gEASkZj8EqEBEcMtA0=", + "results": "533 e253b797,533 e253b797,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,383 eb8559b4,ERR_ZSTD,533 e253b797,ERR_ZSTD,533 e253b797,ERR_ZSTD" + }, + { + "frame": "KLUv/QQKbAEAWgOMABDAJY0BD/zf9O7/4FUdejAcAwADAAMA6ek91yJ/72hPVb4FAVQBAgwEjQQAskUKFaClIOk/6tFqiKqgADCojr+uEKgaH29qd4buq97U5EY6v4zfbfKf9wEtoHAYnWoNECCBUUx7x93uCBDhFCRSMFzU8WSzaMgCgsB5GkgNthJQFSiegFk5gzaDtIFtAIMFCA3sBBQ5AIb/okMP3y2T61I3jesAECNYCt4F4A7g4gAXIISMw9Fgd0BowAQEH7VIK+Y=", + "results": "339 81975be8,339 81975be8,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,68 f6257a6c,ERR_ZSTD,339 81975be8,ERR_ZSTD,339 81975be8,ERR_ZSTD" + }, + { + "frame": "KLUv/QRWAAIAQD06Oj06Okw6Ojo6QDo9PT1DOkA6OkY9Ojo6RjpGOjo9PUZAPTpAPTo6OlI9OklDPT1AOjo6Ojo9QzpAPTo6OlwAACAT0y5tAVQABQYgGAIAQD06PVs9LltMLltMLltMPVtbxRBMW1tMPT1bWx9bTFtbW0xMEFsuLj1MTExMW1tMTC5MPVtbTFtbLltbEExbPT09Q/sBAHUiS2q5", + "results": "207 b8c220e0,207 b8c220e0,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,110 12f15dd1,ERR_ZSTD,207 b8c220e0,ERR_ZSTD,207 b8c220e0,ERR_ZSTD" + }, + { + "frame": "KLUv/QQztAIAhkQNGpClIEkH/3/p5/345U02GzcuxkSujZsEMT0BBAAFAAUA3Y5tLnnNOqMBG5eS9QjdY1EQcQgCqBSggAH/P/8BE1BgoH8U4P8JGPD//QdGbAfAAECsAACxxAKgFDBg/gMTcIEB/gEh4F8GAQLtAQAHwgMDAAIAAgBTYBzZHdU6/wERAAgTQVhxTK+QN1CUCYIDHOGFJQKhfjgjIhQtyGsuOAMnDlD/ulOAEYAisowx9Q==", + "results": "301 78a96b35,301 78a96b35,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,301 78a96b35,301 78a96b35,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,192 95dd61ec,ERR_ZSTD,301 78a96b35,ERR_ZSTD,301 78a96b35,ERR_ZSTD" + }, + { + "frame": "KLUv/YQWkAAAAOQAAEAh59YjHFuLNgMoExDrHxSAVqsCzz8vctHj5QrsAADpVgKoFOANGPD9BxNgYIB/FICAAf+//wFBDMAgQEQAAIHvAVQABAAadQAAkRsCIBNAYMB/RikHgyR6wqCV", + "results": "144 b2b4cdb7,144 b2b4cdb7,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,92 f95a4bbd,ERR_ZSTD,144 b2b4cdb7,ERR_ZSTD,144 b2b4cdb7,ERR_ZSTD" + }, + { + "frame": "KLUv/QQF/AkA7i2AHAAI8DmvA3L6qwcZABkAGQD7///f////f///f53v/+f73v/////7+///f/f//3v///v//7//9//////3//////+rD//v///3///////f/////+/v/+37/////wH/v////+/v///77//3///////9/////t9DqJEsZZghSJGZCUuGtOkgiohMnXmCOChK4RQFoiAlcMYJDDQTU8FM2EIH+pnU8OVXB7qzxxEVENwZvL2YSNz68TijhJYgFKVro7TkGQCIeP1QICHuYpdBiy76B7+L0fJTH/eAwCw8XnKYovxlYnBCriLZUf7iAclc/Gv7Za4q/ofTXiM4ufD5OMPLOF7EiyARHcREnRhgY55ji6Ljqwogr1kz2gIMG+2qHRFUtRiayfIlo4BgHCwgzqkgbXr1dGfwT7HAbfDPIqgAAMXFxUjFxcXFxcXFxejGTU6wSvMnRQAAAKEDAFOa1hBnOYo4Uy4F/LhG04toOIa2WK08dYYlrUL/v4xQ70FVPyguxiEMO+ajV8s1sb6NBIeIRzE93E6/p/zLjzEaHUx9yhYLGVvKrFlHcoKLlhgiTeNwyFsJcgMZp6pedUwFAbJPLfHieLNtcvslMY7DmJAXVZRW4A==", + "results": "1801 7eeff673,1801 7eeff673,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,1685 147f5c25,ERR_ZSTD,1801 7eeff673,ERR_ZSTD,1801 7eeff673,ERR_ZSTD" + }, + { + "frame": "KLUv/QQB/AEAJsQIDsAlHZtfqqqLx///dmQSAwAEAAQAM/9f37LlA2tqrqw9vw8HqCA3/z8HEDKm2QEgzQ4SXCDoYu6ZC6v4AAAAAAAAbAIAwkUMF6AlhXTzqY+qqi7W47v8/6+Gej1VVQMF2bVWjwgnLvOUdIZFhXBmuUdLGFQY8s7YPQOgFOBaBfyuCtw/E0BgYP4BY4k2VYEHIQEcAABhngBEAACJ9AFUAAYAfSgAAOzsDQ0NQAAAOzQ0hzTY6mchAAA0NIc0aJz17Q==", + "results": "249 cd614a9d,249 cd614a9d,corruption_detected,ERR_ZSTD,249 cd614a9d,249 cd614a9d,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,93 ad20e926,ERR_ZSTD,249 cd614a9d,ERR_ZSTD,249 cd614a9d,ERR_ZSTD" + }, + { + "frame": "KLUv/QRaLAMAtsUOHIClIEmGA9NCyAzMPUY6JsZ1cDGGJsZlPWiGXhcFAAUACABrZU3tPfvwy9iLZZKA3YOSwQGXbWiN1gILoEACJEgBRUSBGcYBEDoBuwdugHu2x2ncbuzyKlwL3RbnkVkAuBIAAAA4AABrKChfa2treAAAHTlra2tru+FJc1uDNlcVCAAACQAAAAAAAAAAAB0AAClWAFkISb0=", + "results": "182 22aa3495,182 22aa3495,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,161 843ded7d,ERR_ZSTD,182 22aa3495,ERR_ZSTD,182 22aa3495,ERR_ZSTD" + }, + { + "frame": "KLUv/YQn9QUAADwJAOYKExuAJUnSGNNBZwaqDRXLkAldM8KETIU7EWQGAAgKAAsACwD3K99F7G1W7mgI4f9OEw5by36M0AIV2DnvW2mJLQN/BdF7Q+6bl/xZFQJXqKCSjZLWMIaQqbnzchge03Egx0GQESjkBEYhNBVRngpLjQELUJnIL5zr3PNvXYSevLA2GwI5JYYvDpXi8XjwanyIijPiCkXoWiqoiWVyrPTtVwaHoCoybXK2ZUALxyubD7VJW2n61ygsaV2MYalCpR4N+oS1faqa4PCENP2EfdRvjXBGRwd5HIElDep6dJ4PAJqDSLJk8+g4oUxKeDTQ0Yh9y4QO2szk2wEngeXxWxQI3WOZq1WbcaHCCqVtxo124SCgT9wsd8+LDNmtsBhIlEReYj5n/gK6AgD+aAAAALd5+oP0vr6yvvu++wAAAAAAAAAAAAEAAAG6YPg=", + "results": "1525 4d7f4a3d,1525 4d7f4a3d,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,1525 4d7f4a3d,ERR_ZSTD,1525 4d7f4a3d,ERR_ZSTD" + }, + { + "frame": "KLUv/YQ/IwAAALwAAKnqAqgUYAMGfP8BE1DgPxTwP6kAGAIEHAAAIXcAGQAA5HSghFDoaA==", + "results": "35 bcb26766,35 bcb26766,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,28 b904d2e2,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,28 b904d2e2,ERR_ZSTD,35 bcb26766,ERR_ZSTD,35 bcb26766,ERR_ZSTD" + }, + { + "frame": "KLUv/cRSBwIAAAAAAAC1BwCSBgoSsCWFDgxzSJksyI9hpcMoi6g8e+r24vdjf/fp91Xdxp2vAcvX25EBgIKIAJczBrA9gGPh5t2tpHQKUYuW7/Zo8cGqrbvLPFteC2Cc44fYdWBMu8Kib6Z9GGm77VKTuKzaaFtrJG4/rXgLFMsYUhMwD2nDGj/flgZN3AoaYnFe+hqzy+XGAwhQOuY6SAQ1iwJYmSZKWE/zqXYIp8FrfYacIrYXG2LiIp+3hkE9PN9npgpxtqfYC06uFtSBh9kRoNCwiQGuLbVWhhVE3epVKMHOAQa1LJDUJPClvgK7QtNVYAJApVMZlY5L3dY5zCgDdnNXdy0C+5cb", + "results": "519 1ebba85c,519 1ebba85c,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,519 1ebba85c,ERR_ZSTD,519 1ebba85c,ERR_ZSTD" + }, + { + "frame": "KLUv/YQL5AEAAJQDABICBxSwFaTQAQwLGrJrQxYGQpzkloqy65f8e3zImy4yqBDZAyCEITo+0JhJB/vYvbunKBm99dB6+/inZjV7YyY+cvSYGvvdMj801adQLva97Rb2UXxE/iJBg5FKpivIVOdA9DqgCfqaglbTMwZ2Sjr/AZsHAOldOqtp", + "results": "484 d8645593,484 d8645593,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,484 d8645593,484 d8645593,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,484 d8645593,ERR_ZSTD,484 d8645593,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYKAAAVVVVVVUYAAChKTQcAAAp7wAAAAACAACBAAAAAAAAAAAAAQAA9DAtkw==", + "results": "13 b80be561,13 b80be561,40 9534eab5,ERR_ZSTD,34 7add1e50,ERR_ZSTD,8 bcd434b7,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,13 b80be561,13 b80be561,13 b80be561,ERR_ZSTD,13 b80be561,ERR_ZSTD,13 b80be561,ERR_ZSTD" + }, + { + "frame": "KLUv/YRIZgEAAN0CAIJKDhaQJUk6x7p7NTMmd2e6OyN3Yv9fnxUjmQffnhtZVW5ZX2sP73TOOtxg5//vxzbr69Fg//n7px2bOgQoExQgKGDwDxSgQIEC/t/AwP/9AxOQd7peGgoEFK4de/RA", + "results": "358 64a81c6f,358 64a81c6f,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,358 64a81c6f,ERR_ZSTD,358 64a81c6f,ERR_ZSTD" + }, + { + "frame": "KLUv/QRGEAAAtLTkAQD6A8gAjhEREREREREQCQAJAAkAQ1x5UELS6Y4BIpRMnpQfmwMBw9xV20MN6c0Ba20+IpXjdxMBVAYDAgohCwAEig0crwpQHCBqdzXunahoujTqUo6hXiLo+9ZD6f8EQuYfu8lVgC0vqYwjHqyqcnaJpQ/exUI3jOrYxYMcx95V+eFWEw75glhb0hpIhhCPjfj4MxNe2SzmspMZNLLdpsBHT3XsrGR24OBT95k8mo7Fh48if2K2Vs0AACUkUTuxaS6UeAavNwQytX2E33LChNTdsYCYdDqs96S105M0aaW1Jy1eGttqFkc5XyV0j0Vf7cFkb0o50F879m4rjbL4oRq5ytl0EUXPpzVrhJqXOmevsNzytB+t9ej9ZMFfC7IC+1pGGs1xAHR8ofbPQnUfYvuphiPF6y+uvs++SC77zK6Z2fpjdfZFk/RI9a9AWf9wm8K6gx1BynuMzFjdbIn0GZdECdCMwrhfDGpoSDEh3Q1CnR8y1wIROBAJGUrRt5EtgJ9nNqsY3EOSSenrImBbQKNsS/gy6x3a/jxG1pvrGkKnn6s7l1xm9FA=", + "results": "426 7eb103b5,426 7eb103b5,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,320 684422fb,ERR_ZSTD,426 7eb103b5,ERR_ZSTD,426 7eb103b5,ERR_ZSTD" + }, + { + "frame": "KLUv/QRXdQMAygosARmQpSBJB9s+MzMzHyrbdnd397MS2e7u7u4gDAALAAoAQyoMEFHY1uS9h83471qlVhspfNxpAyDfa4rfzKZJQf0Gb6a84ynKMVn9MQSIFCjgDRTwFxjoHxSggIGB///vHxwFqXbYmTGAWSDsaCSE", + "results": "231 e71c148b,231 e71c148b,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,231 e71c148b,ERR_ZSTD,231 e71c148b,ERR_ZSTD" + }, + { + "frame": "KLUv/YQTIwMAAMoQADEkAgACRw4bgCVJMhwrBvMSUwbYHoAZYGJglgUyA+DUxFQFV9ZwgDcxpU8O8OXWkI6PXW+VDJ76qPFlhM0FgQsBVBkHKjDoBEwBAJBYDZN/R9YmhlVnKLq+/pEbZRgFqHBO+A4QiLEfkFKGB9zhI31yo5ATIAAAz6bPMRAAAGYpAAAAKQAA4uIxMTFVtilZ", + "results": "803 f2f9258,803 f2f9258,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,748 447f9bc6,ERR_ZSTD,803 f2f9258,ERR_ZSTD,803 f2f9258,ERR_ZSTD" + }, + { + "frame": "KLUv/QRF1A0ANjsnCuDpAGgUDDoAgA4jACQAIgD7v/23+9bf9///t/N7//l99/3/f/97ff/939//9/f43e/2f/tz/Xvv++///3897Z533vn////P771/+/7/9+/9/vOd7+//A39+/////+/3/P9//////f19/N/z+/+7/7/2///313X8v32///97/ff/v//v//+/59+f/v/r7///Xc////H5/3//f/8Ga6jBmELSEESGKhgZGWnTASCMiBCR9bK4MIpyHEkqRhQjZVOSgkqxZuK3+T3QF23bHzqAS5lUAfEyLTu8SXdE6XRUmSgjQNy7WEwRxKpyJAVmlDo9oVv82RRDRdJXbvPrZK6iCuRNGaIHy4nWpVSkVgGMzN00krlsYzJGsbJqJxpUB8oL7UZnbJ3fVhds5eokcmSRlZW1b0TrQbPh0cGiKOBgc1r8OiZ85tMgc6g86k/i4VRGjbPTefylAHu93nz/pBxfqTLcTsITFQxBlFNWK2JVIS+xoKES6xq2zJMnUi8PhDUgcweUb7I+7kcxYk+RYEdY0vEnAVN1vjwY5ohu2kgUUwG2roUcq88Hu7T+JVsiCn8lJmglgF2hDGwCAFZEDY4REREREREREAoACgAKAJoyPc++q+tnDAG4HVDHlxsw3AMB2+sBZkphISVaAVyDb9pkSk0RAugTQGDg+wcUgIABP/8BBEgMgIEPcgAAAKQAADlpBaQQM/MQeLF3BwAHspAiXxYRAQAAtDSvFg==", + "results": "2172 a71f8964,2172 a71f8964,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,2172 a71f8964,2172 a71f8964,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,2172 a71f8964,ERR_ZSTD,2172 a71f8964,ERR_ZSTD" + }, + { + "frame": "KLUv/YQi5QAAAMwAAIEWB6DQVrA8BxBa9gE2sAFmcC6TugEvYAOkAgB6A5gAE7AlhQ4MCx1cMJwl7ujkYEw0FQ8DAAMAAwCLvzP7vQN/tQ3fZwMPoJCq6tJyMTBMMLN6RCB9EA5MSIhLfUAICADwBXAF2ATQgcxBcmAFqDKdAQCaA6AAFKAlSQf///////////8ooA4YH/xvAwAEAAMA5cd3o9qPBp83H33ZBQFUGAcKiQxyRki4", + "results": "229 375600f5,229 375600f5,229 375600f5,229 375600f5,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,159 aa630456,ERR_ZSTD,229 375600f5,ERR_ZSTD,229 375600f5,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYaQAAtUj3gtAnqp3oJ6qd6CxvASw=", + "results": "13 ffe63e6d,13 ffe63e6d,prefix_unknown,ERR_ZSTD,17 9152ae84,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,10 14d028cb,ERR_ZSTD,13 ffe63e6d,ERR_ZSTD,13 ffe63e6d,ERR_ZSTD" + }, + { + "frame": "KLUv/URNJAAsAgC+BEAGAAbgDxACRgIDAAMAAwD//w/7/w///w///wcMgJACooAuMA+bAyAD0RsAsMtelpfR7mMCMC6FYZEkXGauv4xHya14AAAF0tLStdLS0sfS0tLS0oMAAAAAAAAAAAAAAAABAABFL0U3", + "results": "292 7c123ae8,292 7c123ae8,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,292 7c123ae8,292 7c123ae8,checksum_wrong,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,281 c9a4702f,ERR_ZSTD,292 7c123ae8,ERR_ZSTD,292 7c123ae8,ERR_ZSTD" + }, + { + "frame": "KLUv/cQZWQEAAAAAAAAAAAC4BwCBeaSVQQm/cZpFUHjaFhupTDydM+wdNoxwuGQC4Xd4+lgG+QoGKJ/CQ893D6qITmLdA3DgDHMzIa+7mrBnjZ//98btkdbJURweSpYoyazCbj2U9yvojJAriey3Y/3j9U5ZUO/l5fPq8I7V1wc7N3CE1ee+JSnbmWhnlAN6A7YyFVk5IONk8wo0nh0pSuFwGcSkL8Ur/2y3z52gFlM9Ab1eXrIKJPv70IEZ59b+9AawltNprxaWOXqXJKkvSQowzh0tylTQEhHqzzIAmwAmt1wEADeznkGasgpvSEKv3dw9qvWc/x0S/x0S/x0S/x0S/x0SZPMKnh0phQIAlAKCYzLljRP2pLfre8bo70MuI+wXsvQNJiqnbGGAnGc0ZQdDWpHYyTPHrhOEsJJZbQCAZ8BBKK60ojfs+IFa87zhOYpibVh82DQfXW2XjQm0I420", + "results": "345 ee15bcf0,345 ee15bcf0,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,245 dec9c463,ERR_ZSTD,345 ee15bcf0,ERR_ZSTD,345 ee15bcf0,ERR_ZSTD" + }, + { + "frame": "KLUv/QQwJBUAbhzAIwAUcCVJkg6OB3vmL7xdQweY0P//ewEcABwAIAD7eY0m24A6g3k2eBgpf5ZUqC4+8L4dO7q2wF8B3aa2aKOipvtjLwP2kUVmDNl4GHjLH6jkbi8/EfFEmF26kBC5dxtDPNdjENgVcrpyKbtAVrT6P4A4RCIB2y6SnA89/P4OSvLHG1agJuTPNJTYz4iPIg9lBYDniHDQ0SQFSWuBMOqgKhpJkqQwzAHu1EdOEBRawRnBkFcXMkzoDhH9jQnI+JJCvNsFf2QfR5yLhAJJlsvMr8MCdKUQEY2WaYp68Ysga/mPlG1R2Y4ukTHodiAIHcnfSrQ2dbKUvLv9P2hvUpjw96WXJBD6DySCc8ng0qyglynZsbc89mZhTRbsa27Ka7wIBvhPOWkViW5pQCAGEshcr+i4l8NfuQSV3SDzjdpd5lEe5SDxBCrnWzgzSQ0hBpAvMhPHNA0qzVOW0MsXHcyBTEhYGhfg5gdMOnyTqImxOZFtYm0ACwglX6yXhMwWlNMwyROPENCIWeSJIbqXrRMNkIDywtMowhdiD8uusrwCn6j/ggT1U4VKAGkFC4jfBMyXYEDYbd5vD9WaLAHyuSQEC1VZj5dMveEq8wQQXQHhAQrhqPxpm7VvwltIu0ccaFWfD9MszFvfIbHZthrgifU0246NIQADUgMGvh356NsanJkSREi5DiAy25bda9btDnDFH8L6v7oDnQd36jxF/hFQ6AXmVQhReHOLJbw6a+KFT7+JQnlOudC61PLYnA6b3a1IW5c+aKK5FwujwfecfGU8nPUim0XARyrBTXtwM6JtDcAuHOx7uecHMGHx2xfEDExgj6tAB08WeczAJwGvIm27sFGySrcXsqNq8KSpBE8DqeE+RZy5QH8qKKJznIlZMNpuBOwGAI4RACsAI4ApSQcrm3QugEPbmLFuwMbGDnqW8ehZ50rnSm8c2oNHTxQBJAAfAB8AG4dJBw3hEsy4mEVxitBhLbyakZEVnKdIS1LW/X6TUGEwICSgdCDTraclhSvdNqQQbr7aCO11ms6pC8grcSubeDbr6jVqcrGvBXJoKrfqfR/kvMuk4CZKyD+I41TfQXAn9Myb0vbiWFv8XtOxJ031BV9AxecgfO2QSkz52uhJ04UaAg4oEJ6h8AGg9AELj7dzX9SzG2KFakExOe0HIoHsFF81xj0zhMIDf5cTMrEC1AIAxsUPjhEREREREREQDAAMAAwABdv0f0OSkEYtUCQUuc8Qlz8CIWzjmTARk0lXFocUfvWKorMTaBQ0q74m26o50Q8ZBSgQnpSCD/DucgfSsJcc+LYXuwGlRKMMjAAAuQICkBSAgAH9BwrNCbHsAATVAABwB6ns0fhRwVNKYRoTxqMDAH5moFTLwJ3cIgF65Zk=", + "results": "2582 9bb83f27,2582 9bb83f27,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,2048 f011a695,ERR_ZSTD,2582 9bb83f27,ERR_ZSTD,2582 9bb83f27,ERR_ZSTD" + }, + { + "frame": "KLUv/QQQogcAYCgAAJVepoXBAAAAWAAAiJpgkafpWXY8LsMJAADWy7/UDQ==", + "results": "261 575a038c,261 575a038c,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,265 9dbf5947,ERR_ZSTD,257 235170b8,ERR_ZSTD,261 575a038c,ERR_ZSTD,261 575a038c,ERR_ZSTD" + }, + { + "frame": "KLUv/UQERQlUFQCeQUCGACtgR5IkHYRFMwRBQBJWKGgQnLv5QaOjCNzNDxodxZLuBscMQbAkYqEZASBOfAB3AHwAK4NZCamVJkAejWHWNDIDZZ2/rQjGkOv1bX6VoaIlpQA4Q4u0sFd0IAwIwsjMICxaZyOi89/9F3x6Tb9NjU3LowjDCgyib2ogagmKq21sQFSk2SVIZGwnYQKszrDJM/4B/mxyxYaiuJBNZ918XqPyVIngnFCOhrT9Hvy0BWQQsb3LbPyNz6hqW+S9CzKocqDYwwoWcCy1Y1fv3Q8QG7WGjxotZZyC9WM/qsHm4IiDcSWLG7Fdc+U6Qkhl9I4a11sk2ob3EWkbdNOVYvOw9bHGyKDFwt7KP1uRyklRUqijuyid1NGO6Oy2VjdwKRCvfS4AMqVbq8u6WapLrBZj4FJDRAeGQWz2XC1H9sHaP3UNe/P3bJMnitH8BLQoAH/YEMu9ovrgZw5Azww6Gk82o3tDgKMRW2+iuszZzZQwswOiomIdpRMsdRkEQIbdFW3F49vpOuK6hEADNjFjGbuA5lTl2YqkEXCbJoU9YUzBRrJyGV09dovzTDFYvxhVOoT0lazzY1z3tMYVssz2ONcqSNlopRpTQYhAd1UBAsLqRK0Fq3U8+PQGJqH6KEzbdixvi0uabB1J6mobV5OCCyhGyxbZQRb6BkLgXgX/HTyPGryYgGiaNuPRpx3as27p8DY+/jBCgMvhS6eDAi+AURAFCYggCRFDCCEiVUge/omA0SMAfwEgo4Dq2LDLWDGwRQCrqwDoBQgr8GkiGuCCYvHAdEiiZSZIdwLiEDLRxg6wAYNuANQHYMERALgUqF0gLwWivQVyuuA5Puh3C3LSAJsZkM0NoI0GZRMCPpjARx8sqoKJRYmtnufAFqgWbwLHArcSsLsGwF0BjQoArkOALAAL4BkOADMQCANReQEpACcAKAD/7P9f/3//f9/f+3//e/v2/sv//pb//nd38V9///P9//+9+/+Pq4Lvb+////6/+/fXt3/3///++/1//195/v7nv///8O+//8//v/v////fDv3//vb+/+fu+02c/e///+/31v/X+/3/5+/f+f///z3f9/v///7v7qe/f/9rv2/+cvftv7/f//6+/373/7/+9d/f333/v31929yt////9/8/M6gBiRIUQpAAEQERECYQY6c7ECBhxkh8Ea1kkElVpjGFALrGuSdfdw0m9H8pQFBkSdoqWBhIfLYcUHudbh86KZHaAz7HrVP5LS1h9z5Fijio6tijGTxuPOZn2F7AKrX1Mb8hxyKKkXiVI9KJ+gnfwApxGMZ+NIS/kQYyhRBp0vpf2DRUKZHM8QUZNYuVoOSUmZWidTzYpmEfAh10zo0=", + "results": "2629 cd0b69ef,2629 cd0b69ef,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,1216 8d231c93,ERR_ZSTD,2629 cd0b69ef,ERR_ZSTD,2629 cd0b69ef,ERR_ZSTD" + }, + { + "frame": "KLUv/QROAAAACAAAzwgAADEIAABSAAAAAgAAdgAAAAIAAJkAAAAAAAACAAAZAQAAcDMKdQ==", + "results": "3 903d964a,3 903d964a,43 fa10c648,ERR_ZSTD,37 bbe9d677,ERR_ZSTD,31 5918ea3c,ERR_ZSTD,24 c752e337,ERR_ZSTD,corruption_detected,ERR_ZSTD,3 903d964a,3 903d964a,3 903d964a,ERR_ZSTD,3 903d964a,ERR_ZSTD,3 903d964a,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYEBIAnyL1eOu2yY70mE7Mm0BDOWSu4y2HbA+cwnrd5M4TXq8VvLO60v6ppuWLck/mUNx9apaMgIvHjqaeq4/2nfXUFkR5j1vWkoVNjXCQVN8BW4WAZ6tZUBL1D80oblLjAePZqYK+eMT25lLB4bv+MAZ17baEefPL9QeFMufwvw+eK3dvBV7AKO9KprBLoWmSdI4tGyyJtK4NdVhr0m5EmV93yezg5UP2f30pAKwNWeoShUs88Mr0KLW8egHWAFGSgYFEPD+WekBWe07vsFk7OS6mGeCqBE+5sK2f77rvvWs+gW6p1Ff0YlVICRFonK6fHF7YzgK+AwQ3mJXpaREwAL0HRA3/N31tdosbGioDv7KeeHQBZdSvWhycQs0/HyzWjGLBUM/x792EXdkTwtnPxSDxOyruxGgGtJnNzoBxIJL7PncnZbAyYH2DBB93tm3yVGadANp8rQz8BRCdWcd4CHY9kSXpyd7twpK7bf4UoUjrF09csKq1oy2zHY/qTcUJ767Xk5HtPm6uUhSqmb8tsSw8FJr5qYyaeI5GCj8nrQM8wwJl1+zIQWDnq+4chGEVg1BV+AFcaKl0hy/ADGZS7mGjyAWAHp25OuGFkKfCpFlLnl64MjYTQdnkmyC6sCfAO7UDf3fCbRVUstEpERcw171vAKtX36cVq1NPnVDiBa8X/MncoIw5Cs6hVsGhaBxIqiX0MDWGRB//XXb0BeB1caWoAkiXTmUaw5d5QhbLkYkpDYWOp3yy+20lLzZM3DZ5XY8/4x8CCgCLFAsAOiBQAQjwOQ9U3P8kBhEAEgARAP////8/5///z//////P//8//M///8n/+f////////z/+f8B///8//////////n/z////x////8///////+f//+///n/H2agsTymGEJFQpJoWAMgSCikHB+XQAYNLgC2+gF9n+Y16zRiKBYGA/aJBAigQwQ4BgyXAbAtIPrDBioAmksQDeCvABTJAMAM+gtWNQxH4hCBqieQvwIclUC4AcLEgqqIoPkFzq/QBw/PErjHQa4KDAB0yQVtQIEnHgABFrZAtBSAWMBihXD3oNAGvAhgqwC8A6LSdXEAKCCgfAfqiAKiRGBvCsxIQPKI2BdkFsDcGPANA141HE6iPREsegAEMB+BLUaAKgF3fkA9AbrhgHsBbxIAgY8NqIDUMEDXga98YLcB3AOAvQOQLagQCE0piAF9AAy6MFMYKDqBPQCaFkAQA7tAgQWkohDgBDgYEAEA4xkPUuLBqVyElZWVlVLtaf30IAviyEEifnt5H/+VgW6VYAAAAFgAAHxhh2k49E/9hvStAAAA0gAAmRkAAPn5+aZzh8g=", + "results": "1822 237e37d1,1822 237e37d1,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,898 f33c6fdf,ERR_ZSTD,1822 237e37d1,ERR_ZSTD,1822 237e37d1,ERR_ZSTD" + }, + { + "frame": "KLUv/UQ7aQJUBAA+G4AZABHAJY0Bv2vQBaj6+nevWqjODBUAEgAUAD3P+0Xy9sf09/bO1f9bWq5a9PvuCO//+3//lTTr818ff7r8////AXdq73dfT2m+e7kvdv0nWkf//98Vt/fs/96f2Vjd98++tN+H995vLgSo1GpV4P+ffxMUIChQMOAPFIBWq/4BcBNlPRFWAWziBQBcVAMADgmAFwAekCdJB3nXZyIizpwgxDcxa3CCiIgzJwjxbXbiBH0FDQAPAA4AbvRTvChVic43yBxfKiKkQCnUGeJZR7eI9twNlI7rMGfx3tcOOWsVZg8VInmShQbY9UhlzBjjSP4KAVQWBShVEDgAAMbGxsbGxsYAAAAQAACzswAAAAAAAAAAAAAAAAAAAAEAAAju6x0=", + "results": "873 db4c47cf,873 db4c47cf,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,643 3ecaf6f0,ERR_ZSTD,873 db4c47cf,ERR_ZSTD,873 db4c47cf,ERR_ZSTD" + }, + { + "frame": "KLUv/QQhbAcAnhDADAAH8Dn6r8nKAwoACQAJAP2v/7/v//3/7wf39////////37/+/+///f//3/////3+////wdBiEGxdEZkRMSkhQOSeDzFMRRNcpSySglmoglqJtVqNUbFKVEiTWR9GrKMuGqMAFdQuPdcRzVFsuqidvV2SAvqiWVP8OyiPWPe62vELWfISOxggXgkQHKHJcNzaGhhS0yD98CgcQ/7u6lPNafhnUwC2Gh5S3lka4Sl7hRyuMjP0Cz7HorJjjIH5bppCuQx+ddPAEasQ5ppMWsJmZydQbflqrAy0Bc2uYdXC2biKDiWKplVTO4O7BIAYhovE8DWaNoGKdLa8GOMaRwaB0EQoAIb+7NC+d6SbSiBHAcbHoThTHP7zaruSYU5estj+JuV34jNEcZULC+eIJjNKbn44vXE1q6Pn6FNBR9nSGNtZidoSaHnwLm3EytW6yo9MUuxdVbozh06o42I19uewte1GSYENbegjbCazSO2CGNu1TGCsP7JeqjYWPQLJ2FXk5DaxH5ui6v5KNIVMrwtq/xm5Cvh7kcYTrj6mh496WdfQdQbwkYKvYERcSGAnYAhPauQJkmGzUFYoCE+lhZxIPINDJ8T1HJrkoKuN3lGDCiEJBcIm6kSFPjOVeDZzgYHYECLPHQEmLMC3ENeZHc3uyhAHA0M7JiJ1Vs8PDS3N4S9l/CdmbikDJgy02ATKNcrHEGFFyyN5PptAgleYm87OwnAVPnePttWlAvIlE/MgObY8qBPARKcg9gXhzPC8tYvSPy0LiBM++oJ4qD0HunnUIc50y1AiTIzqNw+AgjOqGy5CVZSzdoQiyEecQVZECsk1grRib2Lhplzf51AxYqTGS9JHg6TDfa83CosuaXE0vnFZcZu7pm3CjAi04gbU7UiA+fp0VgZqDI5TxTZ6u5pWHl/LAwZ5AYR4l3Ky/OtSffFA0IzYbgZAbIY+ZUAzEsGqSizksY4oiDig1BhBAFcjxfoHWgmJbVPeHez6R45QTQqKPeE1iALMH5BlSeiUmNgKhGYJ0PfCnmQy60lj+bEDV6rBuSGLJp6BSEHy9IAd0EA2Rey+Swxhi4B+Ur4KViw8TGD7nZ8gTRyBlSRByED9TJPICdSxIYyW0aMBAD6CCABHXAlSQrp/L9e/f98vfqywv/DcdXP9f/P/z/cvwkBCgAHAAoAC6EQy/TombUeDf9Vk16qu285CLozReshvZoFzVz7GUekpD4iE4hgTkAzQWnL5wEwzIwFU4HCgnN5gFiSgFQPwHzQATcQ0zq/o+UhFzuZwGSfMmddhIgCr0n7LMymQLBZRxggs0Gp2mUTYAAAnSmdnZ2dnZ2dKScNmAAA852dnaQqKCW2tp0qKykrnZ0kbUQAADFAAXQBAR7nAAAACQAAVF43o3k=", + "results": "3184 8c27883c,3184 8c27883c,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,1097 d2b56ac0,ERR_ZSTD,3184 8c27883c,ERR_ZSTD,3184 8c27883c,ERR_ZSTD" + }, + { + "frame": "KLUv/QQmfAIAdkIHDtClA2DEcSKqVgAAgNYCAgACAAIAv/D7R/8H7wcTACCApgCkCgAPYAQGrleAAgUwpoDXCRhQELlwLgNTgcczfBdYFwC7AIMDboAbAioFAJ7NAADjAgJz/s/9zd+fFwIIFMCAAfcfbwQwAIYI6Wi56g==", + "results": "333 d4e83dcf,333 d4e83dcf,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,269 fc78f411,ERR_ZSTD,333 d4e83dcf,ERR_ZSTD,333 d4e83dcf,ERR_ZSTD" + }, + { + "frame": "KLUv/QQsxAUAVs4VHHAlSZIO/B/2/7+reroQqNP/h/pVdVFV4fpfOkMOAAwADgDZUt+i08iR/uXLkhp0GdXF7nenfnemjtaiN0HiLjnpCH7s7PvLN8kDLUeE/F9Rc/9X/XEDLagxoUNKUG4CSjMuBzCKCCrt4FbtLmEFTMzSdM14vmbINz1WoCscCzKTZrb7UmR0CriuAEz2Ym1bgXDv6KmJjPUAi+LIRw6rZlxIJe7QygsQkAVQF8n5aEU4GcuoPAIAAMAEBwC+EcAnAI4RERERERERECQAJAAkAB+jRbFVV3G7VSyPTCSrjOrzv9L3/ApRJNVr7tDzBqMBEgBYHYXvIKop4GsNOi+/6bJusfI9+J6iHF9iMIjM25sU2UXvGz4xE9D9XeOXZRniWOI2jBfScyvxzOUwOPB7IO6Ak/492Vzmay4IEuPhU8cE2fxlkvxEdDy2gBLAnyFudI7mO8QVdNJtgzLoln4mARKoUMIxIdXEzITaFgMQHGOg6QOAUGs3BBJoT9kls+lC10sFz/fxWeX4f1zEsrjkk/1PrEIELjnRkhQ+B+wCAHaKDQ/AJR1v+qLnTcR98f//dyYJAAgABwDhu6vO34J+uwbJ797bn93zBj2/1/vO/H/bHppf+79/BA2EYFZoYTKLC8sNAHXkinV85UIzshjfbF23IiosVpVIIQgDLm0HAD4MAA0ADsAlHf/fgq66VHVedaIvCAAHAAgAf1//dXr3j/3v/9f///9z5a9v//f//ge//79/v927AVEoMEgkMnN80JhpMVsG8EPBUHQ4YA0c5y6RMbBqMY6OIh0bP8BAxYCSSHqBkyjy7rAQlr4xx9kb8h+3DuxxwJwoCsUBDIlGLHzQYcYAfANAkDAii2bZrbs2FsyibNbzGPtA7TG3h490jA59G1FM4p3+CZNQjCNpbVcSE5HOCFuzqcJI3+KsAsPnLGIJSEw2cgYcgAzbuZguMCRMLoNPu3oOZwEH0XzDadBgyA6Sz1S6zpVnBV5P+ao=", + "results": "1523 b75c9135,1523 b75c9135,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,993 8f26b8a,ERR_ZSTD,1523 b75c9135,ERR_ZSTD,1523 b75c9135,ERR_ZSTD" + }, + { + "frame": "KLUv/YQarAAAAFIDANIcAAC5YACEAAABNANgAHMV0CrgDwNgu+AlHAAAKb8AAAAAhQAAGRkFhBDzAQB3UASa6h6eCA+9/3s=", + "results": "172 9680b0ad,172 9680b0ad,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,149 684a5809,ERR_ZSTD,corruption_detected,ERR_ZSTD,172 9680b0ad,172 9680b0ad,154 1e27346a,ERR_ZSTD,172 9680b0ad,ERR_ZSTD,172 9680b0ad,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ0xAgAUuI/H5AnSQceK3aOnWPnKFf77erZd9yRF+aFOY6shXhMhCKzbjQNi42lXDzFyPJHRvoXaghGxvkTXtE8e5rbqz2bOf3fGW51VjjOlmhonZRjl8nGzL5sVRjnKiQ+FY2ZR9Z3nhPc/6Hqpal8LOlFkWikq2Dkx9hkCpleNPrRehDnm265Sj0lIzpEsyJBdhD/vEcey7b/bgzD0JDo5EO9TOKJTEMkoWxPhERS1Z4ZZYxEcVsZrZVkC/G94uR5INZDl5KYtulxxEIHO3S5EKcM5y3p9WcLEhvpATxEIrtzTMpmj6VW+IxI6+eOjqe/uPecQD5HstHmiJgQIA9vk363lWgdBwByIGLyP7MnC4KNztrPET0J+Io5BkwDADoHwAATsBUkHadJkkRlHSZJ4sB+YxiyCwUABgAGAP9/93sD/6Gvj9oH3xP5T1vYe/7TuzcPqDDSBBfMZVaLATBMzEx7YAYas3XrnwHniISI4sURXNd8IwYJFrmwTROSYwslwUhF8N8DYLQIAP4EgA4AjhEREREREREQCwALAAsAD3wuRVlUr99tugGSjxtzqYW2vZ88AbyRL2pP87mvgrwBndaZsWsy+lPdFWcoIIiIRAr20JotB2QElsZOC5n5oWgox0qDOTJJpSMqGSuDwJrKAm9etxhbUv0qd8z3Bjb0DI4xj4UQg8EweCyrsCcyUBC8K5NDulWMoWqZtm1OmHolhsVPOa6B3AMBBf9e8CG8qQnJQMx1sTkzUe2Ayh2SybNbWmGZL9O/eM40EwSAmZ5axHZDNwANObc0M8clw16HhaBu4E6BPQBDocbxO00zGgTeQTArGDIk7BD2FSADIP312wMg8zgYpiSgxIIcgj+/WGEydB9Odu2A8BQbiwZMUhwhQgQAZa0EAMsN2AEcABwAHACjf8OnxwZcIybcvCWuDQ+0Axpq+5XUs2SfIPYWNyEfZhlGoC8H8cBMPERvXkiKeILTVG0EvMe1GqcqccFI2mVHEpg/8OFFHqYOCd5e940OlO7ysxzFyB6YJqIJeF1YjHs2+1N6qaAF15T3i87hE+ccBSgQOHEnuAeQYlJMpgeIL4mNL2Myhtqo2UFtiYmh2Q==", + "results": "1713 3a4d9898,1713 3a4d9898,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,830 8b8126a7,ERR_ZSTD,1713 3a4d9898,ERR_ZSTD,1713 3a4d9898,ERR_ZSTD" + }, + { + "frame": "KLUv/URRUgsCQACZpAYAzgsADQAOwCUdb7oKUBW/6kDV6bsIAAgACAD3+vf1/v2vA7d7z/9799sWay/b/23/2zv/t///5/81M4gRnbROXRBDoTEhGE06JyBEIhRIDEqZDlo7UzMDjmTR2tOvJGgoS7nametsAcF7xbkPyOatThfhyQeMyhQqOo/muXfyzQcgWAdZL5zIvWi+jTEYaZGcfxQZ4o6xynzWcbgw8JH9b8jPbtKrOUKhrNiprQTA4Ph71xCJcEKOjbTG5XFFIwTQoJPNR4ARKlLZocqpePgSySRH5SfFBQBuCUAQABmAJUnSATMQMjB3SIqzjKzVDKwQZA5Ahk7tCQAHAAkAq30uW6rjoN8B66+cvL/x3o8ZrfzVFbwyD//bvpc2dBIGLKghpXTOUbhu3TBK3NTtAYCMWanWzZPQGoJb2ea5DWgdLnuriwB9oFStEKOsKcGTRpJ1/XVqv+3OmRMce3L2lMscsH5I+pLaGdC/6AMC4LNHnuQHvx8tRj1XIl+Qzw2Q8QY7XjDwSepjsCFDgDhBaAgBov73tg==", + "results": "3154 e6208dce,3154 e6208dce,corruption_detected,ERR_ZSTD,3154 e6208dce,3154 e6208dce,3154 e6208dce,3154 e6208dce,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,2763 4a6a2f7d,ERR_ZSTD,3154 e6208dce,ERR_ZSTD,3154 e6208dce,ERR_ZSTD" + }, + { + "frame": "KLUv/QQK9BEAvmXAegAYoCWFdD+vCbLdtNkpuKrgBS94wQte8BhXeQBvAHMA/Xeol4c8Cnu9xfEAzJE40ITAQsBnwiphh+lROxnyug8Jo2/l4QxZzDu0rJpunmLEAVncdDieUDu0VDJUiKsdMgS0REwxAh0qDES+UKfcr8jQiocdWoW+pDbwYigVCrthGo/ZVGEMJ7OqXEwvqbBwPW1nCqlRbCj8A7+F8BR6c21YEdKWVxZCsKpkhzb8WP5JL83R7V/vOtY0751O/Gr0XNVzkogjHq0wIPdFQriNdoLhfTXSEDmkww6zOCyNJnRfW73OGJakdUCWmBKpMzaOLQoxBlSzMkw8ZzgWFuYU1ToykjBucgeABKlJKszFOO2G1oQn9N1VZWiIkZ+gTtWGXwOGnf+6lqFWpICE5gFmhYSHXRwIZ658DdkElD0xIeu0Y/EMx1iMp+ExwyyqZ8MNSQ/rRZgNV5nCyOiToR/HsBIGulDvG7ChqkN8q0Ov46NOJNXkyi0XQvoPhwO5MWM04jf0aka6yxUmr/MBb41agYVDKtRIYdFwbH4P0zf7OSsGpKpv+VBHtUEELqzehA5l7C6yXwyJzOJYYUQGomsKYw5tBcTjYWYJLerkeOfxFk/FEGBN+MrWfFi2aVsGbhULR+5wytC77CEwJfaKAReoQEIiEDESWySSQCwPEDCBiFCZDwCzmgOlF4IB4JOOGFAq/kXZl1SwsR35LHbEV3fDxjNgpIgIRJK2in+kt8nOUcN01Phgg4jZpzKCFhoxAJEAAAB9DQAaLygGjhEREREREREQXwBfAF8An7eRMDuUfcFKRxrpTZzEs7xMz6em4wQdbNy1KKgeD5JK5n5GThZ8Zbcg5SYtLtA9QLlCni7zcEvQuOno19B0JdZ92keq7kscUC0ha+rogcmS8RtcJ+gzn6NCNfk16hhZQBFmBn72IjckFHHu/tvV3htsoU2oAW8rpzyKoswuTK2Q4A/ZI6+3yB5yiy1hKGgkIVHkg2BiFTgDx0ayAkNtH7VbtGpquwCH0bSlnDuzAbJbkxO9G2X5YAGM1cEsEQAyOH3J23GaMIoMc++EbRU15/LAGCqru/wJ6xMfgyC36oXvmT4NmFJpz+tLYEbgOYz9mUOXAtlFZSo39NzdVpFCc1Y0yMwGd6klXQ/JxvFC5JIFnsseKg88W+1Kf7YY3nUqQIal/cEBd3lullQhDbF0arir7zfrAtbcfm9iq1eQhMfqz8p6RNSLqaB14jxNvLB7lY/tXSnRg7hG/Gc/nJzJUdbBTz52l4naICQT7LGHCS/v/L8DoMkX+r2HAQKoFOD/n4AB/QcT8BP4DxTAgAH//3//AX3+uGmcAGAAEPoBTbQ=", + "results": "4117 fd2dd5df,4117 fd2dd5df,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,3292 b626a551,ERR_ZSTD,4117 fd2dd5df,ERR_ZSTD,4117 fd2dd5df,ERR_ZSTD" + }, + { + "frame": "KLUv/QQrVQcAPgrAGACOERERERERERAVABUAFQA7qwdUoWR8KjNxyo2l1uuZv8SqBBqYS4WAmP75urQojFMZjMsqYZSxmh9NO8BlVfYzG1VyGjnKS6ABAb7SNB1Qx1spVVd0vrjrtqQke0r9m/oVYwE0oHDKEYkLaQwQIIIIlfZOeB8gu5G6GuCh5epfLNh7Sru42BAXpeJcwA4IHs7obmm+4oeVKfL8PtYg62c3ZChrTyGsWBvR7sgFiOmQ0ad6q8sVX6fczGa4n9gqCEzZWMKTQBr96bQcWMW/wBEhg0MnknJZCuKOtQZ+xTWgCFVvRTEBoPmlDw==", + "results": "858 59d36777,858 59d36777,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,858 59d36777,ERR_ZSTD,858 59d36777,ERR_ZSTD" + }, + { + "frame": "KLUv/QQThQIAoscRjhEREREREREQ8BE+5in2YPyQK68lx5gqvatHOj+8zW1k9by/MBxxeaJFmSPC5IzNYJ+A88nKpKuPVyRdqD/eAuSXyOQTrwEBVAYCCgREU5c2", + "results": "135 c0d6b541,135 c0d6b541,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,135 c0d6b541,ERR_ZSTD,135 c0d6b541,ERR_ZSTD" + }, + { + "frame": "KLUv/cRSrwIAAAAAAABtCABKCXgBH5AVJEkHl2u9zIVrYMyXuXBnTvu4KjIz+7iWOjCb64wPAA4ADQD+KDXfGNNwCk5BzYTzrAKu87i7Eo3H6B1SwYBUYutPO1e0k2U6h+9CgDILJRURrcwvvfESeFVkFVKgkFBukg4gypiauwMDpE0YWxI7na4WMfxeKiNEKoyhPQGL+Zy2Vc5DSMETmlGpXXCtkQzgHFxy+oOqtj54EHCuEVNKzERW8EjjSkFb/i6tFwHFNJkdQ5zRoWtJTRgW5gZhKpeYnRTy93zL34Daah+s19NgszH7hBZWDPsmAtES99TfDGCSK6snqDVgekAJoBVQ8alKwPCyf4gPoaCQQEUdxK/ypoaVNANAvYxCmyQ=", + "results": "687 3115c274,687 3115c274,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,687 3115c274,ERR_ZSTD,687 3115c274,ERR_ZSTD" + }, + { + "frame": "KLUv/cQA/gEAAAAAAAACBwDoHQUAkooXjhEREREREREQGLJY/gM8xohiyB3RKUprtreDFMZpg7SnbnQ9OwQ5zhysJP1HYvBp2M/a1ML5arooz0mNP6xFwe/mpt2q/AUX64gV25EyGoLSMYssRXShK20HFF6uHhmokEwRBUlqjwMgSECkKh/wzqoDlBBIcKzy2/v6oD2AWeaRUtdLSWM3eQMdVMDhsM/VDCS0CDBTjjzndoyKtaVUBunhyCc=", + "results": "510 802003ab,510 802003ab,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,224 b39b54b2,ERR_ZSTD,510 802003ab,ERR_ZSTD,510 802003ab,ERR_ZSTD" + }, + { + "frame": "KLUv/QQd/QMAxkYWJHAlSZIOHPzv+uEejvAI3dSABISuyH/+dw0NwiNcsxAJCF0RuA4ACgAMAKLhGDK1sirQCCEOAQMCUilBUVNdXgH+GhIMWoTHhGjVgZMcfLIUVV3TXVKJDQEKoEASRRRQoMQMmwMQOuzuZqGlAjEAJN7SBmEKDWmgdgD3AuCpcZE=", + "results": "153 2173cba4,153 2173cba4,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,153 2173cba4,ERR_ZSTD,153 2173cba4,ERR_ZSTD" + }, + { + "frame": "KLUv/YQ8BwAAADkAAF9fX19D+OdoDIQo", + "results": "7 5e21970f,7 5e21970f,prefix_unknown,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,5 514e8c54,ERR_ZSTD,7 5e21970f,ERR_ZSTD,7 5e21970f,ERR_ZSTD" + }, + { + "frame": "KLUv/QQHAAAADAMARoMIEMAljQGu///MnVNVdWF/BwEDAAIAAwDz7wF//p+/Ae+1AySosJTZYjAMiZ0PsB0DiDAccBkRhNaDj9QfhzhlvObcBFwapvQiXCLcIA+9UJVipRZSfJoHpT3QRWHCCqQFAHIMIiSQKelPF6+D5WgBWoAWoAVFmUdHnpI8JXlK8pSy6TKvZV4r0YFuYnNJgERDQw0jZCxaepdRZRPZyvVEbRSaPWfTyNlqWCdHOZrKn6VTtagVVr1lfQhvb2hH20EkmwyylASIELa8I7xu2Jp1Umla+FIUncCHCG/gQvgyfdG04dnjMooRYsYvhgEHqGCCFP9YnIUHEDSR0QcQ1oQlsvixoILlATrCzKIAudT+Dw+g0h1ABKAAAJdhlm8gVAUYyop724NilRMCObAbWgAAYgAAACkAANRmjyOorPqKdw==", + "results": "564 70b3584e,564 70b3584e,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,169 55ab6f5e,ERR_ZSTD,564 70b3584e,ERR_ZSTD,564 70b3584e,ERR_ZSTD" + }, + { + "frame": "KLUv/QQnEAAAiGQAAAAIAAAkAAAAAAAAAAAAAwAArp/3uNg=", + "results": "3 60ad3d79,3 60ad3d79,3 60ad3d79,3 60ad3d79,checksum_wrong,ERR_ZSTD,20 b3962dbe,ERR_ZSTD,corruption_detected,ERR_ZSTD,11 6607ce1d,ERR_ZSTD,3 60ad3d79,3 60ad3d79,3 60ad3d79,ERR_ZSTD,3 60ad3d79,ERR_ZSTD,3 60ad3d79,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ+fAIAdkMGCuDpYAoAGBgEUQICAAIAAgD9//9//3//PxWowM6IuX4GEBiGlB5AEtmQHQ9grfvG1HkqiWkV2rk0wtz3nGv982U8u6oO2c6Wc4YbARgAABk1qXQEAD4GwBMAH4AnSTrokwmaU4hQITIyjAjZBzPAjaJ9GZHrGD+ZEFgKAAkACwD+ensmUNDV0oYSqj2Ld/ns0rwCImJfM9KHB8drxRBLBLBpSI3bhAm7BBKocErNxExtBxAowQgGtQMQWFNB1XJr1zeumXGBsxr1CwKZmkeufggcS4jOF59M4lpc4MVSAe5tOV3sAQBOA4AGAAvQpQNSAQAgpq2gNAIAAgACAN/v/z+9//8/BqjQLMai5ecQ7rzw8OIu/1z+Dhw64tmnVEF1zFcHHQQA9sYYKXAnSdIBhF+hRwTcVYCuSKxHUHXFFR02qA8ChV4QgK5IrBdwD2Phu0G9DgANAAsAB1cVCndF0V9tCJJ7FowTo15Q+DhCLIlpA3m1nWp7plfvt6moBz8zLQfFwYbjXuWfNTb4BogQMPGZoGUse9AsMMsdOkJAa2dtOYmErJASvlQ02JJ0", + "results": "721 24ac2395,721 24ac2395,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,574 df390fb8,ERR_ZSTD,721 24ac2395,ERR_ZSTD,721 24ac2395,ERR_ZSTD" + }, + { + "frame": "KLUv/cQ/yQAAAAAAAAD9AgDmgwyOERERERERERAJAAkACQDiUBNiuX6EzAFgpddWIJB0hAH+7JSsIc38UwHiRWc35qXTAQyoQAxFzvo7ECAnoLwDEMYcokvMMQ/BinVTMCNIe/4wrNjdd64VZN+KBGdT8xw=", + "results": "201 34b975c2,201 34b975c2,0 2cc5d05,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,201 34b975c2,ERR_ZSTD,201 34b975c2,ERR_ZSTD" + }, + { + "frame": "KLUv/QRU1AAAEXsIqIAfEEjK7BAopbJaBxIaJ8xczbcI6j/cAQDmAgcM0KUDSe4HAAAAAOC0AwACAAIA344D+z99f38dBqiATGSB5bF4EOa8B2ACYy0eb/RQZTishHOQWxIDAJQAAAAhAQBlXxyPFNgZkW/u7u7u7nvue3t7e5SUlJSUlGmTTSmQqSoCF4QF8EOp", + "results": "279 1fc320be,279 1fc320be,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,283 6544166b,ERR_ZSTD,checksum_wrong,ERR_ZSTD,246 1d1842ca,ERR_ZSTD,279 1fc320be,ERR_ZSTD,279 1fc320be,ERR_ZSTD" + }, + { + "frame": "KLUv/aQ1BAAAqSEA8mHhDidojuYWs1gr86wxaHjU2EpZsPDSM5H37Wrke2TdA2gmRdvK7QkkijsIboW13q2wC+Xo0yc42xsyWJ3bOV1OWTxR7of0cVXtTQ7QlQ6h6DnSypEwLLksAGqulJXodETdtSctRhWKuZwPGmiRs4ntvhh3qs5xLQXb12kRSglkSJcYSgpcHC3KHFX4AU9YrRKUIIrbhL9lnkaobZzgAFoXVsEXTxrVZ1Hj/0fGbWJso0FajJI6z1RAhkagf1ptN3SsWNHyJbk5fin9AQZUquUcvCz6jBVoFrRTkZypf8/XQMi2CI20HieKiFu08oPspQTi8M3Vb7IsTTcnpLueDnmoUKvNHFsOmMgb/QTmqVT14SOULYlFsxFN41M7gcOggKvlCM8otIVTZynFZG+flDFpXZt+uk6m1wswEWK5kpQJKTHCRS86U+ABAsDHD6/k+7MM/AdG+IVRZ6C6AX65XrMy+wBp1+IgRVvEm7s6yUB5X2HHiS+oZXbjVXmzlj4NIojc3Lli0Muo4Ant1YQYrPCcTmlxdDa7u34dxSpvfB/TyvjIICCJNZedaAPuV1c2nV3W4Rpz6xr5yhaOrVjEuzf+TKrn3CYR/7kUeAy29Q80qI104KVwsAutO+LBMw04O25MMO/93Rucal8DQfcF+k0GJTk97iVIs1C30CLR/pru2iK/cx5Lo4OrXG5X1ArNaqCRtCftqLBnjR49nC6FPu3O2Y5R2Lq9d37ZgsBnTlliEn094I4WcbUljgmDyCjXm9HYPyKuA3aRZWLj+vNhFToZ+XXgZdWPYTjb5FXzm9daEX+7yZ7gePz2KmUjQ4ZBul9eRmY/y0picILgHxPSfam22WcW4/kKPuhK0DMxQsqYIJWaCZr1dX5FmYxE+CIOqKikXAcSik5FkMx3W8UIYG0jXDSWPRD5wKbdsmitl9EdccMv14xsXszy+U0CMHPIsQAZu/F+JWzllKwyFz2P/onGA+fQ89A9Jk54LOas8wBZN+uTQgkK/R4mm3aE5YllFkMHaL00/1WzAQGQ5ElRa3L0TFj5Up/qbqR4FpqZf9AUsdWtGHwgNSOdN91dgqaLPtjIvake1YHup+BFcr/JWiFS/1WCgcUidXtNKd6MR/Yo9XxWr2GgwzDvUifVO7drRhRhHrhKhFVeYbIqLqwqmbxaDKP8y0t0C6UZueC+rDFUncAzLR0H6P4kt0Jtv8t7Q3bXxwNvHY3WmGGAP0eknd16BGuDFPKJzpgRfj+WeeylZKXpHiT2JrX7iC5PjM8zlH3Yg0QHUBwsCniXbmnmDNzopd+3oiMFIVe7i+FYuTULvTYyDf2XNUo9BLyWnv5yVOXoRWZ/+o+y+E8EyLdsjgc4CK613EDaN8JkavdU2aXRKwXx1lWdbjk010zt0J4AdRfmGqoq1ILVx5aeMjq75OoT7FjPdIbEEA==", + "results": "1077 46b59332,1077 46b59332,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,807 d1fb6ffa,ERR_ZSTD,1077 46b59332,ERR_ZSTD,1077 46b59332,ERR_ZSTD" + }, + { + "frame": "KLUv/QQudAUAqgPkACCAFRQkSQdzBvdEZwXosxJVJAzVrNTKpJhwAFSVCHbRAQQABgAEABPrLwM/C0AKBAtVm6GqPw8WAUAI4A8OJF9lhnnaY4CXmoHDwBYItCjjuhHDTL37MHi8y4utBzLMNeNtC2YYc+MqwBS9wC+jg/WwKaIJ4N2DmUwHaAIGgm8my6ohbUUQrXcImFW4NylqClgA5kEVRoGUyxOB4p2BoEOW0AEK5EAHOHcFrAEABgQLFaAVJOk/r0lVjRWh/v5fKxQ86P03KgMABQAEAPO+ehk7p2QBk32jFpsSTZ8BVAAFADawAQCSd9Ibzcr1Z6tjFmm51H80cL1ByXvA+hP7Ob8h0JviQLj/8QhbKAscS+p0b+TUE8ujQGGVPwikAAAQbkcCmBTwPwYU4A0Y4H/fHgAAAVEAAO70k8x/LuWb4kC2AenD", + "results": "412 bb073008,412 bb073008,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,412 302a58c0,ERR_ZSTD,checksum_wrong,ERR_ZSTD,328 fe21f2b1,ERR_ZSTD,412 bb073008,ERR_ZSTD,412 bb073008,ERR_ZSTD" + }, + { + "frame": "KLUv/YRDxgAAAIUDADoDrACOERERERERERAHAAcABwAEyIw9U5Uc3DXQU92FF6eeubzUSx/YX6DGXkgBJKjwtGY4MMYwlx6wnG2xBHWdhJOByh6SRYjWwPYaRnSyLbm8P4NYqCMykLtw7mEL23jMPTHSQVwmIoqSMM/w/AQ2MiPB", + "results": "198 2ad56024,198 2ad56024,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,198 2ad56024,ERR_ZSTD,198 2ad56024,ERR_ZSTD" + }, + { + "frame": "KLUv/QRaJAIAzgRABwAJ4OngkFJm8IFGAwAEAAMAf+9v9/1iB/fvX//vPgyokIJziRjZ3wEQOunuUOnqQgAqoFep3Qp30u22wSBTmThEAABpWgFUBAALAZwBAKhOPiXfdOLdY1xDRVKj4V7Y/oI2ZxYDqBRgWgW0KsA/E7CB/9QqYFYF/p9/A3YuF2IgJHwBAKOHBO/bz/bv///+//X++/////+/AQWokGJP/HN3EJ78kFKGB3HgA3nARDVuD0MBAAAA1AAAkXQHoNCW5eeQTIzZA9MHjFxG67jmFjABhAYiAQD3BQEAUdsEqNRqVWD/ARNAUKCgAME/1CpgVv0DgauhuV4dzgb/6HeS", + "results": "479 a75bb445,479 a75bb445,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,358 1a225456,ERR_ZSTD,479 a75bb445,ERR_ZSTD,479 a75bb445,ERR_ZSTD" + }, + { + "frame": "KLUv/YQU8AEAAJUGACLPII4RERERERERENLl1E16q7li5SQNG88rF9tVl/TT+DYRKspe0KmCNsvG5+e2eJ6WN9p2+PjLPno2LCsmDEVOmg4RA84/SJZ3kULYu9nAO1XNJOOpTVEnHqfMs9MQoRDeIvBLXF8cwCdbYDOJFzZlMKIV3fxZADcoM8g5ylnnFkG4kTIBFYhgxAQTTCQWJiksTYsBcGYkGAskKGW2UrfeATsLoLLHybh/bam3ZXmuE4yWGYiPmAle3yucUwtdN5AVVu8v+OJAYtiDJiqwMKJOBW8gEBo=", + "results": "496 205f0138,496 205f0138,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,496 205f0138,ERR_ZSTD,496 205f0138,ERR_ZSTD" + }, + { + "frame": "KLUv/QRBhQ4A9hQnIIAlSdIBc4IQBCEgRzmDSxzTWacjHVm24jTX7GiGqgReHQAeAB4AYlWm+aPVNKhfPxbvGfkr/l571ySu5ed7/e0iMgHqDvREUG+zf9/8XI+77/Z7BtgWOstHTQXNi/HXrQKbF8y22FuG9Yjt8u9tR+ha2V3vT/5cI3c4SAf8Xh17XF3fm/Um9EvCr+5I2PO9csHsL363j8MdnUsHgIKocNDNbJLWIIYYMlXoA5KAWJInScopJ0BVCjWVad+0g3G5Gv52yJTqRYHR/eAlmgCUSDprrF1PW5L3b2rgintAQKZ6ifxwAQhh1QvC012uC5eYE7e+WpKc0AKjYO7QiOHoFIGy3vQO6CG3TIOO//Omhn6/ijATwo90byRkRSuYSBXxGqhJDq7xezPsllzrCEQi4tE5+sOQ2wI0cr9oMi0ruTG4KMsdErA8rQtzwf9RT544KgTDeIkyhiPI/Mxmc4syiBE0q8RO9f/QJBLlWxLQDQBJtFmSZv1luz9bAyD3mqoi6IBR8ZJ1Scw0ylcHIhiSAZsVcs7Lqyv7JiikF8MPSGLVNepMcg0E0yA9npn6Kk7bZC9iWr0/11FsPn8q85uKmDGXvoS7wgN0n7ELif+LLCKE", + "results": "1549 daa98a6d,1549 daa98a6d,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,1549 daa98a6d,ERR_ZSTD,1549 daa98a6d,ERR_ZSTD" + }, + { + "frame": "KLUv/QRcnAEAxQo0EyAwSDBM9jBVqHDpw0PBhp5rdUBYRcRVXQCRAQYAqAEDEXBswVVgpbEcibpwAjATnQUAqhX8AAjwOR9zl7sDHAwADAAMAP//3/9////+fv/xf7/7////v/////3/Hv/3//7/v//+////D//93/3///f////vDiio0RBCzDCEmAmJiGGrATBEsFDiAzDQ0Ny2mDU1IEvK9L27nHnYPRQEvReJ4uUh1ypmtDLPVki6C5A+DWNsgm4HJ+STIq5Qe+mvtkKpA1SHpLQqnYZztxKOsFAoB6MAkasLWPEXRWpvZ+0c3Q9IhwMuAi21", + "results": "893 a57088dd,893 a57088dd,corruption_detected,ERR_ZSTD,893 a57088dd,893 a57088dd,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,893 a57088dd,893 a57088dd,250 32755193,ERR_ZSTD,893 a57088dd,ERR_ZSTD,893 a57088dd,ERR_ZSTD" + }, + { + "frame": "KLUv/QQJ/QUA/iFAJQAWkCVJhgP9/48R6u7WDLu75by728szNB4AHwAdAPXKy84/dn1sb+ptBCC6cLm5UZfAfTrh4y+n61+/AVtWH0pWudfr/JI6CUA555EQ2E7bec3v2WVpt0Z75wuL8/Xx2u8Bg3pXW7/Bogeyu/0mL6Lue1268n8LqvmKZR4Gv+o/83uyIz905zyrD4L3LKzjtSdla618BgOo1Crg/3dVgH8TwFUAq4A/FOBpFfCuCvz/8w8og1Z6RSAqQALZb+M/", + "results": "707 7144d18a,707 7144d18a,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,707 7144d18a,ERR_ZSTD,707 7144d18a,ERR_ZSTD" + }, + { + "frame": "KLUv/QRGvAIACgbEABSgpSDpP68peMGjvr/XBdlu+t6vKAYABQAFAE2zyzelMOvvs3ZCb2nz61jllL+LTwEJqBAsAUFBRZUotQ4QKAfVHhDzAVnCebIRkQxBszQkDAYBAAAABQEAl8MFBAAFAAQAneXqBENWeDkC/zUoDQV3hgkBVBYBIChBITIs", + "results": "222 1e1e8c73,222 1e1e8c73,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,129 d772798f,ERR_ZSTD,222 1e1e8c73,ERR_ZSTD,222 1e1e8c73,ERR_ZSTD" + }, + { + "frame": "KLUv/UQlhQDEBgD+BQAQABiwJYUOjcPAZjlMN8thlTYOq4RwmEqN2QUJAAgACACOS6LXrXUDDzxxYZ3+1ztzV1r1f8x9nU0D0/9KE9Jus1RFoODSJGkMIAgpZfY5ArAt4HcAKsD4ANQMOAngeEBcAKYAkzZgAWkKCIEdDoAIYIDZAKoamDHAA3xwyGNAP4A10AM9AJMBlAAxB1IIRA0kBPDAPcCcAXlHAIAVAIpAMqAClAHJgGsAM/BEAAW+DMAFFgCpAEqAC3QCogAagFagKuAUmAUEwAfgAyAEIIAYaAEAcBFEAAAICQFUAQYOXRMCAPelE2mR", + "results": "389 fddeb050,389 fddeb050,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,389 fddeb050,ERR_ZSTD,389 fddeb050,ERR_ZSTD" + }, + { + "frame": "KLUv/cQgFQEAAAAAAAAFBwC0BmhrixRy2RqI4ngZQsE/YFRYgTnAVA/uvzneW1HNgVipE5w9TSQQljvdzro1aD0eiDA4fG4mdB7udUQc/a+Gw9flUebIPmA3Mv5dwFP0Id8XOll0NH3dB7xc/sZOiDBFSnWw0rmj/nWx/cb1MiAgiJGj9BMBACRNNOAw4LMuLQCDA8QGREug5IbGg5gBEACWwcUUcKdgZQT4NYAKAkABwBjI/5VssGSdISDiAaAAwJChmgdNAFgWgQAAGeAHTAEWCTwfsEvACK4zQBJgPqALYAQCocO1gMsAK2ABWIBZBCIvM1Y=", + "results": "277 3e0d973,277 3e0d973,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,277 3e0d973,ERR_ZSTD,277 3e0d973,ERR_ZSTD" + }, + { + "frame": "KLUv/cQ33AMAAAAAAAClDABKIWwEjhEREREREREQQwBDAEMA6Y0wnB8aDdLdvOKANehAPat56nLPhlCRyMqKDf8vhZ6yp+OGxLT1gxGJ54/389m7LH+KCiP0K4jxGFbIJfcqEO3GGBYMB+AKwMM1zqzC72dKyH1/b+gUtF5rlJu2y7RvoUxKp83UrzQ1I+EMfdKZMxrHu/Pcz8wrkbNNhiwizexWqijJExMchFSqIVznNEdlzd+a0MTTjfzHZrAsuoNUG3tz8rMQoMxBMwOYSkCA6iE05jgpTO3jYAcnku34X0IRi8tIBGwt7lURsLc6ZmxyLSOrSnCPpne2WIRlecog5h4DOujJgjiK3g5uG7lmohC9FL68dUU2ZF5gfvRTEOfzBXSzPg88G7Rq7P+tFiSooRg0igkNE5Rgookmwu0wiIwQzh5RDIVBo5hSZiyczGxpPRuqu0uugJxZchO7bc53JFLt29zbflRW7SJR03LtqE7AMnuSKG8Md0ZjTCY4UAbG0g5azVzM+qFe6WSTeJ0UYfOZRBlxTGFj7tc8X6nKk1dgAipgFb8=", + "results": "988 304a3bc4,988 304a3bc4,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,988 304a3bc4,ERR_ZSTD,988 304a3bc4,ERR_ZSTD" + }, + { + "frame": "KLUv/URHwgHlCwA+CIAPABagpSDp/7P/WXeNblZVAnhiV01U/S0YCgAHAAgAJ5XawhzyRlArCH+c041FN7e77gtWB7dvAnkQ+e2WLbUqgLQoMIjI0J0PwD1H8N8A2ouQ51qJfQbulgmjUOvVon1JBnQN20TgA0RpVgQsNd5AkwyaBYKU4NBQPiQCggf0EQokQgImsCiwzvTMHAHrwu5QIKrgF+IBjx4o1kZFrohjFgnIcAdUCVwHU0B6EdtBm1eIrcc6AWOVuZakCVSJWzBBmWnbIYAciCAuInE3MU4V6jyycGvAEVAQAlvg5mcg1uFo+OBG5gQWDn5hbJCYoPkS0jhuIdnQEWAHgHDANL6zQDtOYvCLD7oud+xBC/9g4uCTTQh23JEzmAYAMDinicAV8mEk+LAQQSm8ADUhwAXJHPIcLgXnSwzfIaaMk6gLHqACQoM03EkHgyXxYepLFpNaItvgUTmY7x2AHNZRPEoDISwLEgi/0GZFK4ZpCATguHEFTdAcZgDMKwB17k8=", + "results": "706 1a6e44b7,706 1a6e44b7,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,706 1a6e44b7,ERR_ZSTD,706 1a6e44b7,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYDAYAkgsNE7AVJB0Mc5Ak12FqSWEHk5ObBgF9Wt97LK1/69j/f72/75fa/02z5///3sv3X+O/um7rA0mogA4ZaUEyjAEwxpBR5wPQ1E0He43Huhc7EvYMjHa239qfqoVRZch8FECWcb0qvZI3IdbpVqSPAIpZ5k2ST/FCvLmFqBqsE0FSz121vBEYOy8irRJ4qjbe2WJ/LZ60zP0T5o5YfLQun/oHzys7Ee3MzgLCuEeOBEd/lg6g0R4E9YxBBYBmyBBvATgBAPbLQUWvCzgLCzg4Cwu/+pALODhlOAv3yTj6h8bf72Rk3hYLCzg4OFwBAKPEA91Zzf3b5/Oz656D/HSzAQO4FKBWAf+3KtA/1CqAVf9dyqbMhYQkIAEoAADno2fNMQIAAH8AAAAAAAAAAAABAADY/JEy", + "results": "617 99704fbe,617 99704fbe,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,518 ed670b63,ERR_ZSTD,617 99704fbe,ERR_ZSTD,617 99704fbe,ERR_ZSTD" + }, + { + "frame": "KLUv/QRWlAoAIssYjhEREREREREQjKN3aWZM2gUfeGOcWHTh2SE03RS1feMTuBPjmFDir2n2sGbz/NkrALDzHh6pZCklkjgfgm8Yhhok2bEotI8XuhROVW6jwF2RkL60mjEDJ/+0deOBjm4ZhQYBgICoQWHMGvYOMIYQUmc3kFQRpwM/tFUyLZkXJVAlktH4paP3oKec6OKEwlPi05Ud2SCeGMfQwvLZFbnhaMlHa0dG76PvxkhZjeXYzCwuvtCIgErQ7aINqdFKAQUlN70Vm5TNt1wBGfXVOzguubvs8cqae9aJh+7/IDjsPPHyt2n429tmvFq7cdjMQFXC8qMFKw6xNkMTsQjK5rLL6hJDeuKXgPm7DBHb1Rs/x2fCmizQEmrcKUkIIGSbdHG/jeXpndxrUY75ZcUOsUYEo4jGxCD5vBsEcERLVhFtmZ1RZMWEZ8bDhe0cZ89d53kAAADEAgB2CA0SsCUZDpAlcS6JwzlJ1SQ5IeJcBgAHAAcA9+tf+fM6n6+jV5P/A/1V8930Swt/up76OukoCbhQSpDCyn97UBym9nO4QyLm2z1M6WBs7HBZ+jvC5WMRHQQAZscYJoAlSdIBPQdnnLVKzvQ1YKxd1rOsF+rjrFU4464BY+0FMNYu67kgDQANAA4AhXexjZfyt51l6hpjZdQaUaP5TE9Nj4tOzBsd7nOR3zYemHJVe90WASzVSgUxuhPIwkO6Y4IFBoiAQhNaYPksd1DThQMJGDG+hDIRpuYrR6RcFiuZtK1b", + "results": "1113 e322eab4,1113 e322eab4,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,775 e5bdec53,ERR_ZSTD,1113 e322eab4,ERR_ZSTD,1113 e322eab4,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ/dAMA5ARVy4sm39sJgn5n6zATXSgc/+rb/EvGsMM9b84Qb3mDkurJ8H8RFoVS14HjP1Qus/xJZgk7Km5NV0ihLDfNnf0Jw8g7g63nn0KfnwsGulkEKBNAUFDA4B8U4P+fQAE/fwP7D9QAsUsOzJwBAkncBQCShQ2OERERERERERCFhd22osVIy+LyvhGYgJWEuZqAjAh25thaUV8Km4H1d0Ffk3NtpQ/dKwa4cxE2qKCYrcYwRiRkSh+BPGqgIiMzYmFRjSybAfhpnu/xu3f6eLL50M1eoSKy1mbabfn4QpPd0es8mJLeMKOmiCFJvW1+WGqwxVneFhToSKFEjKsNjO7oQsW2v+FmKouprFpKZF/wQDUMMkDPNpDQwcN9ufwAbPmOqHmKlQ1L6+AsoKuujAMAShIoARDAJY0BD/p/1rdgquoi7n+aDAAPAAwAv33dP/2xn6X///5zrf3Mz59p9E32cxWfvXwF+//97Xt+596f+q03/b217pW//eu7//cfBaiQ4n9PdOkOECZFKcMDEP5LJ7jL/xwBBcMGmmiOmTuyycQCAADvIgEAVrgAADB+Z+swfmfrMH5n6zB+Z+swfgeFUoVSGAAAra2tAAAAAAAAAAAACAAAiQIAAAcCAABrAAAAAQAAogIg/A==", + "results": "1508 9558be51,1508 9558be51,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,1036 9e023764,ERR_ZSTD,1508 9558be51,ERR_ZSTD,1508 9558be51,ERR_ZSTD" + }, + { + "frame": "KLUv/cQapwEAAAAAAABcAgCKBoAACPA5D7S4H8jEBQAEAAQAx//YfQT///8Hf//3H//H/z4RqJDQBBa37DYQGCunDpCvBuicOLv2zlX2DjGaZGgcixDf6zU+Za4AAACUAABY1J8GcyKlJGskyMQB9BMPAQEbBwDKU1WROA==", + "results": "423 544c3a61,423 544c3a61,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,423 544c3a61,423 544c3a61,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,163 38d24635,ERR_ZSTD,423 544c3a61,ERR_ZSTD,423 544c3a61,ERR_ZSTD" + }, + { + "frame": "KLUv/QQpxBQAKgZoASdwJ0nSAbz7YBI31+8JCnRedyjSobhDkU6E+2ASPVftpQgUXnco1g4LAAwACwCE+chdrCiVYcdnW+UIAc1kWiRT6rqNNJeLJffydOobN1ADNSFursdDfyUhHoFAqPF2BzEUQhBjFtMH0UAuVU2SDqQwV0SEpQwrrraRrJeYcz72GjjFbnTjamhH+zHKZJk/ILz3h/WMLRbIJKR6CMQgRXAVLz34v5f3Rk8ZRMxLxbu4qagAQONx+hJFwvy1mxESLKOov5OEFCWUm4EdkEZ6J91dD9lYwHcs/bwUNoYaY8iLKmRYBE0hVKihsIMAbRwHIQtpIILfA0RrAIXTQyWN91DPECxu2tOSwSYOAXkag1/Ou5lto5MW9kL5q/TIg3gsKFGDeD+jlUDCUGrvd+EjfwiEPLznVIc0SbDxh6nN+Hg2/770wgxbRESNtKlvoo+II0/zVMeelWGY+D4DXy//nD+BTL4OtQ1dUHKEMITJ3dU1Ag82KOdy49KpE/vreuRLw1KJFw8MTTVhoxqqmX4+/77qihMe1iwC/CaEw+ymQM6PLRtaCJxqsNoeMN1p828C43sJ3bVDAGrXtUE0BJINhkWWz+YDkb9cjUjqwCIgQOpuIrVAeL72OHqUg0WeOAhslQHjhxiLjAVat+6jZkoiZtQp18Kt5s7b3OYPbiLgdTIcemzfHFs3gwHjkT1vKZEhKjgPOFhleMK1zJ2cCcYCyumV7xTC7Oy38+ghD41dBEiHRzLZSgPy3JsmAahg1FLBQI8cxihiUt54W1eVEk79md8UGLDiEFciG5syAzo74ClOcQH1vyTtBLsZCz7RQuBbYxJa+yXbXYdlk+HYKj1UUNIMNakFo9CJbbdwMuzSeu3RIocABYUFADYFDhuQpSRJBw/efRGxmY8j5vz2y1T57sjEB85OdAYFAAUABQArTbq3E0/XxSW036PPBKoPGMNRnEYkNaiwEtpmMEYgREQsH5ASlqQxXJw9jptsG4jwJQJV+metN6lujd+AsOcuxT+r9725rVx+SSvZyguFzZapmyXOfdkg6j4pl1kGZevMWjoJWfI+USNgy2rcRRPDPUBBxu+2XsjUIsysO7ucfFo/2P3jVKoALUhKvOJhww==", + "results": "2120 562a36b0,2120 562a36b0,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,2120 562a36b0,2120 562a36b0,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,2120 562a36b0,ERR_ZSTD,2120 562a36b0,ERR_ZSTD" + }, + { + "frame": "KLUv/URE9AGhFwAsEr0JPKjRrgx8dS5LbDQPNYifjYnYE8LrLyouxky6x2/Pj3SXBcpQbEEYfkdW4dQyUek0id1WMYG9O7YxzSWQGglVk5sp+KSTqzHC+2+nwq9vJAH4V6ppo3RHHjLqLUgb8OOwWw31CM2IIYvE2DYK86i0s10NoMU5r3zFAy8g4hcznLp/Pvatw9xfALbXhIVGR8Bo5DSlKENs36sBXHDJheBiuT7+18ozgIjZU4JofWE2uDa4veZGb/8eTHaQaUELv4p8fjYfFU+ZIWKFo1LQlxGli4MY7Ovx5EUwJb0YPyfukPjTxoVmyiPbK/csQIidGJi1T01xd0gFxi257AMXjflafs9229EjUapNiRCY8/xVBNLw8Q2A24dDOpeuS/43xjPmHbN6nps6VDuhriMJXUgOvv7+Ysesz6Zq/PQ1+dZmTlMtIYpeDbKolBiJWJrJmgVudZS3vUT+9WWMQ59jVEN4YVFDoS3TDKv7Y0ajtqwN7l4ykgKG3cc3Omnut+QB8wxM9tCnrhW8yDCBPZWXRAk/4qZTmk7ycIrI3ADXZfpDAvj7JsKoE1yXSx0QlvOj1S6nLqtiWbLSHrEj7SEqfkwx8Xr1MFp1UnAa2So0PDduP5Y+yblYbtJA3PLiIfXmdXd1BOWye+RWZJOzaU2NJZjXsFxNEkZ3f6smCxu6NfX9V3X9xd5AFp+1dtzl+MC7j1Yc/+qgSvmQO5+quneHWseWcMZlWeDik5Z73RYULuKMpKm9PA7AQK+oKYPsks7pIN/kcPNcWf4SE9aPMRoUqrkZlqcpaukRZAIsMAxDSMOwt6LQq4aNZzg53mACZ/tN5hKk8etTFcz5EZyOGhRuLTQ+bLkU/IcDpwL8/EYJAKn30VElgSZa5bL35hRzxHp7VoasRlyHVlsh2NIP0YOQS8mkZnYjepmVWiXzJdXNTdZQhBqVSnbzUtXNC5xoOmuZ9PO+wX6S1cyRtMksQO4dUAqF95E1N+qGpNvGyM8HYoPDpwsF80pY", + "results": "756 3f16db57,756 3f16db57,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,567 77ea7019,ERR_ZSTD,756 3f16db57,ERR_ZSTD,756 3f16db57,ERR_ZSTD" + }, + { + "frame": "KLUv/QQKAAAAXAQArggADwAVcCVJkg6OtxfmEL68S77q/////3+ECgAHAAgAWTNjUeBy1C15Ade9/XobtA9brfzuhwm3MMF79e8fbgofqFCWBRIqTJLmEBhinEEfQBCb6oIqhSACVzDrD4Cxn5E2AEGx+YRovkZPv6CzUKyau5npwQmJD9TmOpf9blDkcLQMJlgsFJBzOSABAGCEx/AUNdnoZARuOa3Ownncdj+hjMuPpljLRd6eIpuLufiGzgAAAGkCAEMofAAu1/R7Z5aFHDNqvie4m7macpoZ3rti+vqtRlBYFLJYwhSPrXEKpYyA9sciHm1KGsMTr37e44KERv9f1sCH7/w7vC6Jne4aUo97qPHerw==", + "results": "418 fb164dc6,418 fb164dc6,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,352 f82bc7ad,ERR_ZSTD,418 fb164dc6,ERR_ZSTD,418 fb164dc6,ERR_ZSTD" + }, + { + "frame": "KLUv/YQqHgAAAFgAAM3Nzc3Nzc3Nzc3NAAAAmQAAjCUhEBpZ9x63ytaO/BjQDz7t/zvD1No=", + "results": "30 44cda6b8,30 44cda6b8,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,20 446543dc,ERR_ZSTD,30 44cda6b8,ERR_ZSTD,30 44cda6b8,ERR_ZSTD" + }, + { + "frame": "KLUv/cQFVAEAAAAAAAD6BADLJAIAMggKEbAl6SzKMAwokGSYw5W6MVUCf91075Z/nd3a+8z3351+337/b9r/HwSg1CrgWxX49x8TsMEBfzBXK6gTWW3tgA0BAAB4PiX1", + "results": "340 18c76d0e,340 18c76d0e,340 18c76d0e,340 18c76d0e,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,159 b3d3440,ERR_ZSTD,340 18c76d0e,ERR_ZSTD,340 18c76d0e,ERR_ZSTD" + }, + { + "frame": "KLUv/cQV6QEAAAAAAACcBQDmkBobkKUgSQfgfTkTzoQzoZ0wn3Nj7vMdMJudLQUIEgASABEAidDmQ+ziky3Yfw1EKQd2Q6R1o4zRjbFcmSt2q+KGDMGSX04VD6WprgMHp/OgzCB3lt+9CHmBVxaPEeksep3VMXiBmgpoEh8OGghgRNWEUlgorQEz8cBDO2lABU2z70/gPMz/9sJQvR4MdvSMAT479AyxswE2piyAYQGrgOCtOSpQhWr39J9hLahcgKrDAggAABEIAAASCAAA3hAAAJ61AAAACQAAkiISTzA=", + "results": "489 a2fdb50f,489 a2fdb50f,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,489 a2fdb50f,ERR_ZSTD,489 a2fdb50f,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYRAAAgV8BVBAECCKIAwBCwFPiUhMOoXGLAqWga5B77tYfTS1wI2I2YwKgxgsX5XWTVjQxPcFg0uN6VEyhUY+HuBDJUqA3YysD1dh2ghc2A13X1yfRaOUKs3iea9pPOqeu/WIKo4378yYEEjE53vO3VYkCX1cbBNlXwHVRRTMeVHAAAOcK72TSo5Vc0lLLnr6njAAAeVIDiBQwgP909R8viQfkAggAAAAAAAAAAAABAADG5FcA", + "results": "179 19570d79,179 19570d79,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,143 431e5de1,ERR_ZSTD,179 19570d79,ERR_ZSTD,179 19570d79,ERR_ZSTD" + }, + { + "frame": "KLUv/eRuAAAAAAAAAEwAAJUDOgFUDgQCEYEBAFWK3nE9xapOQlCXxfJPJ5VkKWOHPAeq3zGiL5EpCfW7Ln6EvuEeMwcOlxzi+ZPvfi9o1jI=", + "results": "110 40d144d5,110 40d144d5,frameParameter_windowTooLarge,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,94 c9e03014,ERR_ZSTD,110 40d144d5,ERR_ZSTD,110 40d144d5,ERR_ZSTD" + }, + { + "frame": "KLUv/WRIAAAAAAAAANQDAD4GwBUAJIAlSdIBazNmyQW76AMKiIoMN/aeWxg4kBqoDggnzDLLLLPMnQ0ACgALAAORVRLoIN4FkwIzjwjb57FAqy3FxBonCkciMv0KphOR5QSS4RbIs1QNjWHcBagQJgUnuvxcOJCiGx4QftKJ98LlPQdseD6+soABAAAAegAAwEUCAGoF+ACOERERERERERAMAAwADAA6OREmWhVCaAYniAE+vYZS85i+XchXHwGxReYRs6DJT9arqQFFpuoVDIwVqe4HAQFUAwcLuZlX4x0=", + "results": "328 f709df2,328 f709df2,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,228 d09f618e,ERR_ZSTD,328 f709df2,ERR_ZSTD,328 f709df2,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ6NA0Aos4SF6AVJBkOP/IH/eoKBZyAE3CozpUA8bkJ3eS6GkDN8Pe/+Td+Wbh22zd73QHisvkk59P9K0A9nUfXWTvN69pO3/75+A3BK1JLSrIBgJmAkJZx0hxuJYDqmrqJkJcIMiNAacQgMwBAhlYwUhLAb9i0QRDmBiISkasbC61CwA4UoEQ7K0w4cX/zUaJZSASLBrxg+G4iEM27vhKLGxfaA+jsUTQ2IEL7FnCDmGELoxF7EdBaIBiSdFxWAWDSgPcXvG/Af13A5trsEFYOsYRboJ8LxXEdXVDQgOUWdFux6jplIfDtpmzHmxjlsIHRit2x4AawdDhT1CCkdRVhmBIMQYH6B/AA+sdGhC/ZS4gB9ixQ4c5OOyLWBMIOhUEAYWsgF+B8JhsYL2slGBC3m2MsryxQgYPFEbsbbTRDQBd0YAVgHO2GhAEm9n6QYwYAfkOEGmBMTBMH8hcTg2TsamNAqkPRAIoFABwAiwKNAAV1gyUxgJ1mwAGuESMRDAQwamDGABgVOmowFXSNo4hBCAIgECKqYk5dHwX2DQAKqhkkB0QUQDvgAiZEAQCA+Z3GgHmnqgCKlz+LD3jsWgKoFPA/E0Bg4PkHFICAAf/zH6wVAAOARAsAcmhTjhEREREREREQHilx9ew2azfRrmH8JEt2Z5+ydRGwYW1kAKpsZa244TesGjc12Xe8n6LGaYTEeWbcUXX0wufM786oNAON/zyc0YD1+3tQRpqqcmJHKl3t4tu0liPq/ZJ/8h/zLHE5r2tUk91rVTNSxE32rUzMKdjN6AAZqM9EyPjky3aF4L+U5ALYJzowdGdxeaMJXpppcPhbXBSVZ7m/wukGkvb4Kt0vxNzkCfdFJ5nHKPA6hLIn5VTFCoc72W5VWIfqd9RL4Q8or+jadAGImACUCooubphjsDpQAh2UGa0xDMKH3mPot82xDRFauDc3XDbfb//GV+oDZnhe1BBmIa6RZivYpze15Sq+dpx+MtLW860VZI8/SQGuqIC13gpg3q4ggco0LCG6XfPK0UKo017rwreXf1NnCngjCn5k0aGlLsnTSGM9mQ5PUiccApgU4P83YAD/AQkU4AIG8B9wwrKTAAAChQUAppchF7AVFNIBDGcpWApCVigZ9yrcU+A0jvtLGQAdABoAVRDVQfmjJTFj7pKOWijppsKCnF1Ojer+r0XoVpVIDmH0sNI+aNRCkoZvRCd9KmVk3DMEkCwFpWVlLkKZ7EJug0NZuOHWDIoa9Zcd3D56s8HpE4T57FM3rFrjF4xd6EvRsE8PZsfbqQEHqNBCiz/LWeYQHtP2YEqsWVQ56/8OWZqhJWwn5T3T+khAbsV6EQZb+oxz", + "results": "2080 8668c61e,2080 8668c61e,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,925 9345e1b8,ERR_ZSTD,2080 8668c61e,ERR_ZSTD,2080 8668c61e,ERR_ZSTD" + }, + { + "frame": "KLUv/QRQ/AQA3g1AIACOERERERERERAdAB0AHQDBjEg+jO1tEg6ZzojGeAV9UJSi5rx18P4LOySyAT4eXIBxCRhHWCIwyb6tW4zqdNvnnWsnXzFJhwcBReIDF0IWrDlUUP9Is55hLsL3HFnVdGONlHKyYwG77zKWtwr8coGk804KR1aSQEnbSaRDWexcyxwGiGBM/Fo8lh5wg6UDZiiXoAMON9WUM8Lh3AAAEdgIqIAfEFCRRR+Qqmq7AwZE7c878a2gt8L+RAQADgeAGQAkkCnpuaBVSZwPAWOokuoRF2NDlcT5EHjAtCqJP2DGUCXVA9YBDwAOAA8A7VFdA6GX4DHtORI/v2Zup7/Rrzdl9MaSqQldghN0Hx10EhrzcOar33TepA/3HxmBizUt0Ts5iI0LM7IHoJACFVp+LXdgYkygD1CGJL7c8hQ5zrw9WwwhB3gAAMK8zY6MeGNBcn8xOHdM82gAAA8FCQEGa2vx+DwGLslMAAAQY3YBVAEBBwIIAAC4AAAAAAAAAQAAFmdXvA==", + "results": "497 7879869f,497 7879869f,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,296 ed1a706,ERR_ZSTD,497 7879869f,ERR_ZSTD,497 7879869f,ERR_ZSTD" + }, + { + "frame": "KLUv/QQunAcApArkYoHVPfGacpXFdyijjEvC8m8paAC0AnaLJfJCeF+bnI6aEaHZC5KgqGuNmbyat0Smf5fCMs8r9i91f0jJkKFmeuP96NrEvtKQsb1yrSUEj8LYEpWrKV0E4dcRANyXPL9a3hznABPhhVyHxYeTB7mF7OdZWqGSysp34PblP7NCS6GQw9UckTDRHfExYAdjwLLpjRopDAwOyZhAtta9cUne2sK1ZGMD1t3QDiGoET3jzFwwQYVarRtARBE0fADZMdBsahY9wLm9AXDsHP+CsJ8EN6ivXshjyige++y8AmYz9qVDZQTi557bb6rE3xZQhsnhFAMA4kULGZAlSRoD//8XxXV3Vya4qirjh5Wpm/3jKQEb/2d1vFe+3XUnz0J3b9NxO+U0EaDQaiZUau8GMEKjVN4BphCxcNGutgtJzEXW4KGIPnAg1EgYSAHUQ/wFjJ9M9GWAUgJKCQBuAAAARAAA6WcBVBIFB15qAAAg7AIAogMLF6CnkA7bbwMAAHiflt4GAADwPklqs08MuovXXOeaobW8+a4fRORBVu1Nr3gQiHDWTJhQHdAqmwd4i0uHq2G5AqFKWi2IMRyoJ6XIYR4QeIJbEMI/sSFUpVAnCAAAih0AACFgAP9LBPQ=", + "results": "946 77613f00,946 77613f00,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,753 b00c93be,ERR_ZSTD,946 77613f00,ERR_ZSTD,946 77613f00,ERR_ZSTD" + }, + { + "frame": "KLUv/UQ+3gPMAwDCQgkagCVJ0gFzFjSaCDahK3dhpoAZqCIhPSs1A2Ft5r5Q0Zbxp3YPIyAwCCET9dg46QAEJN+DaEZEKJUjFnsawJ1gUQwGP43loL2CtuBcmaXbyMzwVp7awOdgI4MDQEi+ewTtEmEDtd0IHNYe3QSoTjLY7QoGkKkrAAAAAAAAHAMAOgKIABKwpZAODAMJOOa0gikstCShogsCAAIAAgC3VbsPrT4NlQYXqJDWbRswSEgOPiCUiB2VpNAaP8QG5lE4LxT8xf5MVVes8A+Rlp21gzbtvjQHIpQx2iDXQ4l1WeQrYZ8MgxYA2QCspMs=", + "results": "1246 9619c71,1246 9619c71,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,1246 9619c71,1246 9619c71,290 e53bead5,ERR_ZSTD,1246 9619c71,ERR_ZSTD,1246 9619c71,ERR_ZSTD" + }, + { + "frame": "KLUv/YQqCQAAAEkAAENDQ0NDQ0NDQ8UpJk4=", + "results": "9 48ef29bd,9 48ef29bd,prefix_unknown,ERR_ZSTD,corruption_detected,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,6 ec519e21,ERR_ZSTD,9 48ef29bd,ERR_ZSTD,9 48ef29bd,ERR_ZSTD" + }, + { + "frame": "KLUv/QQp3AQA5ogWjhEREREREREQEwATABMAoCJeD68+B5NdADL9vreceY/fAUB8WXlD+AjfLjQ4fyWiBwrsgQEHcq2joWpJLOwUTTaox47vmMEBc4cAIiCVqe6K4vUP/VfJcWcBGahwSkkFFUrSbkCKlKUOkJqJNrYDOM5BxzL/r7BNdU5UQA5Qw/U3IYOkXeq37c2HDOYVHEd7n9IuYbhQzQg5BQCaw6DmShMJHknKPNzgQk8xzJ1AXJWSxh4zKvhqHtQARqC/b4LwhxknzTtPtyFN95dWCfPPdoD7+S2meVYeAc9btdJ8bCcUfcRH7/FqQlTbvFGeLbb3zR9om1TRwZtveNcB0cvbk0L2ROFf7LeBzbGEIqdLZeMDj/k+RuVCub9zA3KXx4speNnt8aQ+A4eXPopkGEpt9vQcgZ5H6FfAIdO84l5xqhme+HPcOls=", + "results": "444 3f5168e1,444 3f5168e1,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,363 612befef,ERR_ZSTD,444 3f5168e1,ERR_ZSTD,444 3f5168e1,ERR_ZSTD" + }, + { + "frame": "KLUv/QRV5AIAceQ9iKBvkM8BNkD3WWR60E8AV8gheVyGEdlxhcfyMpLjFL6ekRcWFJeioZlybKtMDeMGdpNiqYcPJEAJ78RpVfA+VU1edLqqM3FjcxVgQ7Xzrp1dyFTpztsd537cAABIn+/AN5V9IlPaArgUMMD/FOA34H8SSwjABAhFAAAIswF0AQZOE1cEupE=", + "results": "266 8b2f35e0,266 8b2f35e0,corruption_detected,ERR_ZSTD,266 8b2f35e0,266 8b2f35e0,corruption_detected,ERR_ZSTD,266 8b2f35e0,266 8b2f35e0,srcSize_wrong,ERR_ZSTD,266 8b2f35e0,266 8b2f35e0,216 f2bf9995,ERR_ZSTD,266 8b2f35e0,ERR_ZSTD,266 8b2f35e0,ERR_ZSTD" + }, + { + "frame": "KLUv/QQA3AMAtggcJJAp6XmXKgZLMRbFGUTdNL0VrlQTKIRAIQSKM6TOIDzadWkCHxEAEgARAFeXi/o8putkL67TY+hq1/cPLLYMCm3jNWMF+7rsG3epncQWiO10m3Q0iaXPjfE7FLl2dxiutqw8QmJF353U7F5Lj3C7AQFUGgYsC5EoDAUARhcmHJAlBSl0x2+J2ExYN2PwoooPnHDmCO/Eji+L+IgdAB0AHgCzJNGl4KaUGVONJ8yuABOKShIFbRYFhaiw8oGqMj1DiD30GAGrTebICgR44kCgQ6GtW2MqoyIfZMAUDzn4MBpIkrBiEGTFU6B7QyA5IYJwssTXyn0XBRIFPSvwzUCoMByOAGnRdVA+Cky8ZSaKrKLQ6In3QAUBVBAGC9IcAQDzRAYbPBZPQzmTYlYRMvAMfdOQQuGBsQOBBcUkAVQRBxg2AfoJAMmMAgCGBAgN0BU6CwFA4wEA4AusBQMAAwADAPd9X6/u9///B1/93QmokFJQqfLbAxAocaU9ECgooPinosr/UOgB24Efa0am4+G7MFr/Sk8BpU/MYwJ8AQATCgb/d373//ff2/P+///u///vu+fvv9vf/wMCpBQw4L//E/AF/gMsgqyWSWAAEMAHAF3ZwxwkKgG6a96VjJ+YPiZxG08/d25inL56u72LHiWdwvr141wTXr+73oyjnauVbXkUiji9bjP7foY84KOTfY47mLWeKfpL7oSkfr8S3iGsaDxUWjYSgqskvHD8gmajBhG3BlbugQNq7TRqx2avztIuMZedWVqsQTcfqVjC8oVg8QPDlrUJrQbpFy+Uowl8R6JfLl50UiWEfwcsvgU+c47p2K3CFC92Iqe7yykqzNn283Vu7Fzkk8EZjyar9Rzv0KwRc0jxMI+PRFjT0obxhhPKmTJFmj9tATu3OcTBb4wRLvlrqsq8n3ktFiGMCNgibox0qSEJu8pBIAcAzqL6REe/ycnJZaWCKy+bx9queiy2PvYttRrEc69WsDANcOWU09+Wt2kDmnh6o5x6mnFWDKGWxXaP3B8VSgqRWokJV2+hskaM3y+q23w/UBLBWNGnsRQuwcyZ+cEG1Tz3bqysGNy+B8p3XZyY2meTnum70cAxrtTecd4SDJnNbxD0oexqhHXeaWKJWrQyeEOsEXPPf+ZrFlScw0zPWcnJyfBdtf54ePVTLgHVAVw3pL3s3gwA5Pdv4WZTLiLK+6FlHQOUd67F8y8orJSeHJrHRxN/w0AhX3yiw2smra2XLRYhycnJohAAFngAAN7oHxLcma24Pd5cc9g7gjkBAKmNI7P5zRYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFprK1eDxmSxY2Pm51d4=", + "results": "3570 9be53c42,3570 9be53c42,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,2796 1cee85e5,ERR_ZSTD,3570 9be53c42,ERR_ZSTD,3570 9be53c42,ERR_ZSTD" + }, + { + "frame": "KLUv/UQTegGUCAAuEIAkAI4RERERERERECEAIQAhAIIZ8ih+8XC70fyZ97+4MzLQQ9JpfJRDX09m8Coc0cpNFelkCalRxg2t081TntgrrpIYzcIvjGrsk1rbTCgMk5DGH9bzqICg/Hd31QIglJHrNz7jnvi/awpldCPWC570w8uWEUOdrs5ZT3mx/ECX4IWlEVTBdGSLmhcKUAdyD+p17oEXMqABKXWKBRRUkhTYHSDEGMTWDpiiSFWAiOujBBwDQY8RsECQJwXIwQZssJyABQGjEgAF4TLYZTCOQsYpGrY2hdqZs0oihQ8ZlEGYBuY55m9rQoYB2cGaBMxBiTBSPIDgPpgBgBIHfZAUKN1EY+lGaIUJwAHQQZlgAK4LgAAA/TUCsutNXY3imC4Z4cr/omAAAC10ZktjUBUyDAoLDAEFAMewWNopArPuxaR+UKwBspfAadYAFXeCR1ef6K+b3ApiUyPEHsXqIweJF7HZtUJ3xpevtxTQuYpTwWGlz1ngmtCfkfdTWUFJMbC9lUBX/22yTI2EhcEHF7dgze1idMNqMR8dixc1uevTtBF7kYlph/xMBp66Dt6K8I/1wrAxjRtcXDIBR8R/S1Gq4dt1c2WDwMtvI0/UEN1mG0NRoJirYcacO36h", + "results": "634 63c1e83f,634 63c1e83f,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,516 912cf686,ERR_ZSTD,634 63c1e83f,ERR_ZSTD,634 63c1e83f,ERR_ZSTD" + }, + { + "frame": "KLUv/WS6AgAAAIUHAEJkGAvgaQwAo0AJaC0A1P/3/viev//99/9v//7/9//7///9/3/5/99/v/7+Pff/r9/ffff/P//5s9fWT//3///9N/ed53/9/b/5+m+/f/9f/tf9//d+//7799Pf+7//3/a//3wvqAEVU0gRgZCACJFIIFKUYyCEEMQU2QPhKEOKSUhBUUGOAb3MlFUDQVtB0F1pF3WJspfJuPomLtY8orO1s6UdM6E8OcRs6gU3zFxBsLYUtABT0f1TrRAZW1E1g07me4yJVcRDSwSA7oNpizD2qI4Ocir34EOYDqn2tPIsVtTIcp8c/H0kqIvLi3IXBgw9ibY=", + "results": "954 2ef0a76a,954 2ef0a76a,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,954 2ef0a76a,ERR_ZSTD,954 2ef0a76a,ERR_ZSTD" + }, + { + "frame": "KLUv/QRKAkAAlngMAA53SiMX+4ZHFd/U9lTVXbE2LaLWmjExxwxP4oF2B8+NqPixOI5Oo/qgpkwx3504U+lBUahtHCkPjcjPCFzIEw0oqLfDfA+o5DlTMQeblzfbwoJFaGC0PCB6O30MAWhvPMZuVDzCDrs1Ot1o1cixCXrB1lSaOLpgIJw7yHDRpc0yar10Bsa3022nNLXRWVH1OH9uD1o48p4bzYpO0FYRvgkm6JLXpjEvXarSPaf9i30hCduONYK9666QPOJHF/+JJbwbblFo4ao3HJ3BOB3OqeuTJHX7qkYbPJKOz3bVZRBHn59ACg0IwlOsAUstsGyoTyqwxynoLAxWY2EYrcVc1HwrdhI0Nv17mcRa3Vz0krUa7goOEAKZT5z92l4hTdtuCf/I6yMYG5u5g14sar7UaoUxRxEAiZcG2AlCd3bARZkxYZ3cVyyBrWs99GXAHK+YVb/hVkBR3xlzJ0Vy6pRkcnzYVrbbRiILisX465OHif+1lI3qrDx8sNOr8VXWY9Ft6xfkoYYfZ8I1NX0SN+Ze0DouAAl1DQBuZ0BNABHAJY0B/++r2Addnf5n/wtwA0oARwBIAP043+9zPlr7p3b6j+R3vsL5//lfB3m2927Xu2397Wtzq67BfS96/3ef91vP399b1vXxuv7vr31q//Ha/Tev36fV8H/966ubr3sO16/3vvize3wa977V/6/l6/OP7/9Pz/svX7Om9/xP7/O9XtL4/OT35b6/f+X/9W1+p//8n//mmV3///ZVX/3bf/fbv7OnbwTv9c/+7o8/P9zde8/Xvuv373vZnen9LK+3+77bdk756e6zelb7vPt3hc3f7e8y637nf///tyw/5av73f4p2f46/H/fX1v+2k/9+z26+/9e3f3/ePdV/f/1S59fd/+/fv3U/7XOXz6XV3f+94/+l33v+d/ueH//fn/r/uv/X/1c9ZuV//wu3hvXv7a7/iQoME6ckFwPMTQLjYApNcEEtcYGu3dwnBIA8P00EOvYHIU4YhcskUKS/yq9zMDq5+cX2+9iZKjZ9hY5FwPmaEJNiU7r2XJsMyipx58+htIuT+blrGu78bH12Ia1o2DBifELeKbPbrNG4uh8Ww2iisS5YVXaAhCgNg==", + "results": "5900 1171bfdf,5900 1171bfdf,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,3926 d5e9c65e,ERR_ZSTD,5900 1171bfdf,ERR_ZSTD,5900 1171bfdf,ERR_ZSTD" + }, + { + "frame": "KLUv/QQWIAwATv9A1IMxhFoQcteST+n6v5Q+jTwazih3M2WxzQcgFXmztmYaW+rPDyaIwYRTR27GNG+fcOulUOPpM2l210UVlULJbe94xKf/Q+ZlFbg5qDa/nas0A32mQxJf2qL7i/poWNRp6mCErmHLw0rirzNlJ94bEbgTHgkuEEzvOfROAFXS4eGTBinfvpweHg59qP//lob01ZQxnYyEuEniBVMCXK+yc8a873KnR8vrtCqNolO4oQxlhsOF/uSmO7RiLTQ1Dd38zpxgDLZWLTtB8b9m5wXz7VDCAfrCJk9X3HPiiNvigjADJLfSlaVp1WYOY4GG0ke7hjvHqwB51yOAvafaSBhQ5whiKFdaf1K9NSB9vtpMIgTaIqLxeasWXFG4wXIGs3fR6O3kS8Bur84oK6xMinuYdqYyq00TDRmw2iLCfzRH3RTi/LO8jp24hmxTFCoYcepDm0W69nlZdsK4LX5KRFGf25dXCVmAPmfqtaRtO6Zmy5jr6ESEbyp0hWbWA+DqJ0QyziszAMOBM1Sj", + "results": "2025 575c0567,2025 575c0567,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,294 1b560bc8,ERR_ZSTD,2025 575c0567,ERR_ZSTD,2025 575c0567,ERR_ZSTD" + }, + { + "frame": "KLUv/QRZ1AcACg4MAo4REREREREREB0AHQAdAJGeNBZffkFItQtY4ubHZ/KdMMsr5DbvkFTL80YBvbq8kOgTZprkizirfH1H6K/st13TEhGzsCwFKAERRs9w3y2r3Nw5Ksmvt52Z0pS6HVyFuLiaIknFAQ17OaiFTjCK1kcQHB0/q4kPhGYz8rB+neJ//t8BLYCxLAwNzdSEiWOsBngI1Lz8MyP4Ualdzmy5zA4MMYAX0HgASxqAZMOGDVtxaCwLRzR9wAcLaSz4HEByg4Uy4WgeYwweNyzr0omJ97gAB8Bil3YAvxIZdfPaISV2BqCY8JY6fddOpsCiYs680qU904blwmQCAJLGCBHAJY0B/+2qHn/Q7/rda0HNED21nzv++6+/r8j//v/+/fY/CaiQKr7MVLsDUOJSsA8QUFFh5esBSDjoCA6M1JWwjQEh3e3oHgooAABqAwwNC1UFALQHYPrAq638UQgtyFcyuj0obk20628mFMAe8ErAThOwD8efbzf2uBWLVnJhEtZfsWDW2lFcvyie8DuwTlM4DE4UwyuPfFNVwYIAZxfjZjejJQGFV8tmQWN9kTgJdx0Wii6t5bys9I3V96Z067vq9gR6qDK8SG8utAo16Y9uDaBAQoOQES3L0BxAUkSBmR3AAXd1G++2N3TDUtKBARWgD7g3pkx2RQkIncETBZFKzA==", + "results": "804 528c00ee,804 528c00ee,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,617 98c27afb,ERR_ZSTD,804 528c00ee,ERR_ZSTD,804 528c00ee,ERR_ZSTD" + }, + { + "frame": "KLUv/YRDVAcAADIJADDcBAB2cx4I8Dm/u3f6JgYbABoAGwD///z/8///8////////5//5////////8n//z////////////////9//v//////////////////////8/////xz/v//f/7/////////8///Of//P////////////////////8//////////A6gU4P+3VQCrAvwDE/BcWf0BdPUfVKAJo+yjA/LSI3wGADbTKo4RERERERERECcAJwAnAOSGAHdGnDyiyF0V3j5thIcU3Mx4F5V4gtECQ6Nst3P8GKlgGq3BHIdQP+6lv+XFZu9D6Wz8bvQaVODThTgE0xUxljqfyHd+FEzNw0qYGuRGtJPzkFCfaj0nUa+HD+u5fEDeEa9ADVHfpr1HPTC1cE4FFr5XFT2eUqp7p2tXEKhacfJkdxZ9j8+Uxxq5cGei7mm/br9GzcpHguCoAQooECixZR/wtgcAgBEl1ezmiqXlgUYc9jgMgpy8udQ4bS0EACLCBhSgJUkHP/uR/7/qf/fXVTVeVWNRDz/+d3u9PCaIEJeGAxGAJKmRANEENAvsXd+zBLyto6WnnOgUM4mwTy5oEVlOsG0Qw7LcFY9BHr7LIyUDZclq4Y6Qwg1mCXAai5rLW8xCKPkP0YAZohi4ApBWuCUsanURpQigVqCWnHiri8a2WAHWJimW", + "results": "1876 b64ec80b,1876 b64ec80b,checksum_wrong,ERR_ZSTD,1876 b64ec80b,1876 b64ec80b,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,1472 e5639bd6,ERR_ZSTD,1876 b64ec80b,ERR_ZSTD,1876 b64ec80b,ERR_ZSTD" + }, + { + "frame": "KLUv/cQrLAIAAAAAAAD8AgCaBfgAjhEREREREREQDAAMAAwAfj+dcnc36sLGdIkbywc3HacSpqeY/Z0UGKgYhWYEGTRWdqQRJ1+aZL1VtltbJwEEoBQogEAB/xsY6B8TMFdWAX8PIHMALoA2htotEHwCAHsEtAAKAAoACgBbw3bhfi1Cm/oBxWDFMP+E3XJWAdAd/0PSAbnORwERqJOHpUgeNB8FiHDie/kuHhBI8f9z4ufCZQ6IryHFEWHKGYgXTgEQAAAUCRwAACldAAAAAAAAAAsAAEkGvOwr", + "results": "556 70816d4e,556 70816d4e,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,242 58b68857,ERR_ZSTD,556 70816d4e,ERR_ZSTD,556 70816d4e,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ+wgYA0qQCADoEzACOERERERERERAJAAkACQCOtsIFU5BNQhQvMSPDljWBKBN120JEsmI6oRlMfJqG/lJ/igEHqBAwgYlBkUVLBxB26QPADzqgBTrNC0Wf7fT7AuEBAG+arG/0VLCEOQd8Jaxbngd+94ehRAvQFptZfjgZrgPXHpEvFP+ZNesmZJ101+X6OgMuMwG+1yzT0TEyDcIcK+Y=", + "results": "365 13c09b09,365 13c09b09,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,365 13c09b09,365 13c09b09,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,328 3738326,ERR_ZSTD,365 13c09b09,ERR_ZSTD,365 13c09b09,ERR_ZSTD" + }, + { + "frame": "KLUv/QQLWRMAp3QqC48IE/ZuFUkcaUoZ7PmkBhek+eJyUqU+GGLKLdSDsR52dYVEibrYO42AeDJzuCWCLmjssQKJd0cf66oOn5H6wVHqgXOPmHh46BCEtpvlmrSla+v2SzUQEUoQUEbnsNN7mnOaaHtsNnww1R06DsXI7Ixl2IV4O3ZG12DRX2Irw+pJTxCJh5dnGbfdjQywJU2ZgZYn2F2e0pu2pm3nGTBK8Jq0mjiJxbgtXFqyq+n9xNpPzFAP+CJmqAzkaDR5RFYgtbYtYGl1NBNRWgHIyCEA1KX5a0aXzAle3aWi4QuuyGax6G0JUL5hOAuk6I8v82zXLFFMC8eEL9NAkn9gT2LwzyXMe2n6l2YIZMijKy9O7NO5nB63D/eo/x8WanDErs8ZF46ZZ9VOMUx3hOUPszktyFKUgTjdOaIKV1yB2NkJojMDDJfx9PC1dq/0/CPA1o9JxnXPUXLlxQ4sG4w1Y1EjtPmDBgc6+66Q/WNGWPfilbu6X54h9kAV47gWXuGFiUfM8R9V5PIIFTvcBiLxD2foyA4ACAtvO3T4zUUgpv4Mhh2hrcuVbK4U7O8eRN2FnUc9bE0UChTleYXIHGA3h2AHUbkhXuU24L1ejlM4259E+l7PK0BNfMo2A4K5x9ru/6SQZWmj6Hi/9n6/iz7bI5+b1CcVG6+hjuscdsfl19MLUdnLdiAg3lLiksw9AEP+AIZsA/F3isABLwe6KX7Ppv9rtxQXuQchVxuuRJffXGi5VMV7EvaIbKq0hpFnKcp6GrbSVf+hyhwvakEjrxvhPUMufssJIJdrwJTfyb3AxOlJJInnCPMI8AkUNmZqzsQ=", + "results": "619 303423ca,619 303423ca,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,465 afc3c79,ERR_ZSTD,619 303423ca,ERR_ZSTD,619 303423ca,ERR_ZSTD" + }, + { + "frame": "KLUv/cQBJAEAAAAAAAAAAAC0AAA5uQMoE9CV1R/UKqBVAf4BAGASQ16uvAUAFAjBV7oyFL1SNVcdfx6uajzfCOKpRTj/iAnQKPtgBMN4to8teKdNKyhk6WMmkEAJ/DhFTsakv+ui+PLGSAzvaC4hvxs52ioCYhR8Gd9F9X/vrtecciEQG6brlIRVnTYand1J4kpaHzK0jGUYraTf66ZULUER5uBzvecw09U82E6khpISqFDMTGCVFpbdahCmVDuAVE0t1g4Q+pFkxHoIG7Wlvt/R7E4zgHnbJ6rfwmQCgxJGshbERAAAYdkBVAAFFDAAAAABAAABZ0sf", + "results": "292 228895d0,292 228895d0,292 228895d0,292 228895d0,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,24 d65cde7a,ERR_ZSTD,292 228895d0,ERR_ZSTD,292 228895d0,ERR_ZSTD" + }, + { + "frame": "KLUv/cQ1PwIAAAAAAAAAAAC9BQBqGQwBBvA5SM7foQ4ADQAOAH/y//////////////8B///////////////n//8/n/8///////////8D///8////////z///fzioEaVjztAEE4nIbEFaDSCGIEOpHuAP4I++UFDVBy6zQCqeGz0sObzEVsOks54GugGmYTtQMB0z2gMmPJfT79LqjuGPxF0BjxZOh8xJPV2b+vuSSgwdaTIwPY2+wIv+vxPAObpTCoTHnll1lxxjRgGbmj5B", + "results": "575 ebde8e3e,575 ebde8e3e,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,575 ebde8e3e,575 ebde8e3e,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,575 ebde8e3e,ERR_ZSTD,575 ebde8e3e,ERR_ZSTD" + }, + { + "frame": "KLUv/cRS6gMAAAAAAABEAwBosNgWMryd2hhDCpevtkKosA8whhhG7QFAe2Agz4FDT9HbdF2Ij0gTEN1BiCCxndR6ipKkXRgEuKAbR5sAFNH0Ndldvm/7TF1AU4h9MhLm59rQOBiIC75JzVxWrkSlDebkQBQtqiTxFqQIAIKdPY4REREREREREKuniaq7VFB9l1xAgmyJlnXYDc2wf4B8tu+X/4Y9b3SQAAL292jyDIRZM49Ubsav3QOkxdNV0WnE/9R5aMLbYqkzgiZ3YLQYNbu+zWUDGnbRPUlqea9q84GMSJ+rnJiNRl+NCg/hU0yyaYiPIe58NFdJz5hT73mF16UQL9wdehHoiMHauRxH8K801Xo1ZBR2oaEVc9MLsi/9miKqFIHuJWkrPefhvE2ezDeqrZ2kr/phrmPv+lRDspTvVMv56zPNpmUZUqgFwxA8KNYvRoms+my+mf51PUBz7TGoDBR5NfWUnVxMnF+MVOdRIR6oAQQoE/CwBvwBFDCgwMD7D4EAa0bATJ4lsADIU2QBACKIBgjwOf9CzztBAfv////99/+//////////9//ApQU4C9gwM9/CAKPywQgxAEAwPdxBdbtUX/fy8LbOm2uSYSjJzEnVpsHeA2ksFKQPICiiWbaAwARtuNRnaGagC78haD07IadrQPRAQC85QEOBwcPCg4MsLCwsDKooxnNyskvCozCZAsDHLcFBAgPRA3zGd4KODwLDk7+1gwLCe0CCQpR3wkKLHH6/Q==", + "results": "1002 666d59b1,1002 666d59b1,1002 666d59b1,1002 666d59b1,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,741 60b8501a,ERR_ZSTD,1002 666d59b1,ERR_ZSTD,1002 666d59b1,ERR_ZSTD" + }, + { + "frame": "KLUv/QRQ9AEAAoMJG4AlSTIcc4bE1BlIj4zouRgjgDIETdDvzkjcCZ99+5/fmrCJrQMEKBMga8AfFICrAf4BRG2IgB8W2Q4cAAA5LAA0AQBw5r/VXS/ShTr4Lk5hQWkFqBCjzPMQLn0QOtG7vAcqpYADGQWlE0QAAAifAVQABxuIIAAA0NDQBQEAALMC2Vo=", + "results": "171 3d2ec0b2,171 3d2ec0b2,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,73 9e7af546,ERR_ZSTD,171 3d2ec0b2,ERR_ZSTD,171 3d2ec0b2,ERR_ZSTD" + }, + { + "frame": "KLUv/QQMWAAAUHH5X5kSDxMFQpsCQACPKjcAbOUVAAKwYo4REREREREREPjtS6yt/u+zohG+n9k08Kv/7L/Y+2+qlHr5aZjajfyt8sMJhVfn0n0zQd6Oju9A5e2PqNWNb/WgJchdpZfPXGvwDX5SGfMQe9AsA6KrYIjVAZTId+mf7bKkmXdO8SyesIQny4VTtXQt5Jdu2L4v+pZFdvxRLu/4OngpBTdPEcf7L/CMXbCUehFqjDdulLe4VYTCAZ46SPJjgVb0IzegBUtUMCuMGMtcltXadNQbHNr41qViG6hajCRdFdGzZ3lHPMcgIiI56y3C51lM9vyplQemAK1zydjH3JmwgPfezcUiTCPFD8Z9k8YRcHwHVdNVS6k/HJmxlvds0Cr0KgkPwbjFBUPz2mNqvfGLiutdyxYUiaXxvnPJKPKhL0cAhpjBKvt89In5gFdP7jeoFcDRLvCEN/3Q4mrDxlED7ERDTNkYytObf1IKw+Hw+QeysP1yFp/ZAKQQuQNLKyUXy4DV/y3xTbQPmJVHmok9Gzbrnuu/DYp4WdTBPV14O9/JkfQBUgGAgSgwSigyUvnAlm0O8aQUuKSB6HD0nmIHWZGADkA4ORJHWFhJ9tJy5DvnKXme9C6YAEg75hNLIUwQhlfYGmDQB+cCYyyt6t4vWFLjSeiArG7qawaByuI88cbllGoVDR2XFK2BGD6TtY4qHEkRixzAjuMIuUEGrJmu7grkNrTBF64lZjyaOhpah3XGL4KldKQGkW9BRKxv4dF38/3pesYM+qd1oagfWPhPeLVi6POA9BO8oYWEKbwgiPZQlkuaejOE1LaBTt7g1LEGQmmEWZYoDBgBUvw/+1prOYgfZMfea0UTNERg/VIwFyEjBvrS1ylgV52MBJnRApNhEEms0lIUjyugql0KiEfumYEoMXGXMZpxrMXcjpvwlK4i2VigjJ+txRHy5ICUSRBvIvj3C4fTI3Bd", + "results": "5138 cf7af287,5138 cf7af287,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,3824 9445837,ERR_ZSTD,5138 cf7af287,ERR_ZSTD,5138 cf7af287,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ+sgsAHwAAADgBAF9F8mTnozQh7hvomZ5ahCMn+1j0AxWs7oPk1fIxlT+pQB8fHx8fHwgAANgAAAAIAABsuAAAHx8fHx8fHx0dHx8fHx0/qUAfHx8fHx0QAABEPgAAACkAADoXct2Aw/rpLg==", + "results": "445 cb41d271,445 cb41d271,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,430 bc19cb1c,ERR_ZSTD,445 cb41d271,ERR_ZSTD,445 cb41d271,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ4SAAAyCvSNdqdTXLQxAEAegW0ABKwJekMK6jImOAUxiinmiQBy5UFAAUABQA7/3YHA5l7c6Mu+48zrwHn3LhPBQFUGQMJlAMSAQCocQEAmoc8fgwWWx0fCoHPjqB6iw345F4dssRifWPgwFuvANvSgr3NOABTrTH8nCylzyUUTMw=", + "results": "188 33992c7a,188 33992c7a,9 bb7810e,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,158 5a8470f1,ERR_ZSTD,188 33992c7a,ERR_ZSTD,188 33992c7a,ERR_ZSTD" + }, + { + "frame": "KLUv/YQVPAEAAMwDALZGFiCQKembDfs/SHc/9n9Odz/2f05p8d+dSVq8MxMkLbJbIg0ADAALAHelBiVJ1pl1IPTQTqtfzlQtWdoio5O5pdoHL6wpcE99s9eFz2y3YVJjVaTHlAm2kksRCIBQYkoFVdlqdwGT8gTcawHrEUIBYM9zoi5YFBjtAwAihhYnkCdJB+uIgaXzLr/ac34gtnTe5Vd7zg8oJui8y6/2nB9QTND9ai4Ex3s1HkXKR4t+ORXZ1V5cB5nSVd7Q4x2ybykpDIgZNNwsvqQAwWChlR0pM4lSN41umLUKiNCCqliZ9ntw1Dt9BGJLAeJgQTSGiAXaIBgyJp/rA5T+85I=", + "results": "316 607b1dee,316 607b1dee,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,174 5b061cb9,ERR_ZSTD,316 607b1dee,ERR_ZSTD,316 607b1dee,ERR_ZSTD" + }, + { + "frame": "KLUv/YQaBwsAABQDAGoFKAEgkCVJOvPK97LDx5zbkbGdlj52dtCZdrt+ubNXMLcjdcwJAAcACgBz8j+K6G3qggTzlk+Pq1Q3roEiHlQ6yQlkBoMWlWkbCEE7DQMoEzBXVn8UgID/ARZ4GoARwAQKVAwA2idEBY4REREREREREFEAUQBRAJLNGoZGALbJ1szKbRklazXxkM+TTdkVeRYTBmPJ77XVy+pvINRB0ohVbTlv7d8ScuG57Dvk08eX8fYyexUiQ7tuQjXKTKroPQYATLo7RFPVAVGXEV2J0ll6xQxITFpw1a9GXJ2vzgGOH8TwhvRBRyvICb+2vej9V2yJjJnI8rgufMsDgjbBJVPqJvccQtTmk/I+8rJ4xQ8s7pplmEdhQ4ohAZA0pbwX6Dv6oFWdywSquid0bpBESrflXKLZUY8hdpnjPDwSCffHwNxx7xJiTpLp/mdWFeh2nyDBenVkcC+DyXr90VyR58ofWoJzttkrwPdmASq8eUxAj9LpGWZBhqnCWznL4CHf2rE8w4ccAfwSh8+MYBd2lFjBse+R8srPW83dI0RCYlU5rgpUt94MS8ZiHdbivJtmqr5IIVlM2AZrvhQSiICCmZixeGYyWwdg9gENVvkaX1GJZFkuXaWY4X5Eid4RzXm+xE/nfD8mdrCoK9PVbEMxDZQLAFo0+AEH8Dlm/10jCBwAHAAcAPn/5/P/P//////5/////////+n8////P////w/z/P//P/+fP8//+f8///////////////Pn/z8/z//M////n/////+c//n/////8z///+c//3/m//9//v8//5z/D///P/P////////z/3/+5////wdgqJEgpBQjSIAQGSKhSTQcMEYkZKb2ATE9d0IlhWU7dvQL3W5dzED7kWD0VOKsYaJOLErLyzTI2SGQFtST3xYrv7+W3M+VlFJ7MUBMaYHAgddQC1hwXA0RxIGRPV2aRk7IoVpO2CRtVIIVnwu+A48NA6XeGTKafj1yxuLyOIGHTrxRXdjw0xtDX5IRt0Csq2WiCZZcSIjR7L0+SlekIXxzi6RniZYnateh3mD94axrACg1wEcB94uF+0zdQqYqnOhJ0QZxAQUys7r2IaHM4TtxejcsUd3QPNDBAbdsyMZt/6NZCig34EOFmerlFgknec41BgAeCoAYAI4REREREREREBUAFQAVAEaDPU8WUeeVcdX7fsH5ttXCabD/HuRnYPOcBaUF1olWSVIpgUDFwhfWGo1nd/KqX9/pGzVaLHBbw1Uy+0aJEhvcCL8vrKMsVxiXjD1Sp+3CYlQBIKARrXJKVYVa2PoGQEqsMR/CBKgDgVCRgHsUoDRpXLhoxNqLJXwEN9PlqZDyRURMPJ3QnUnENCym91BxtocelHTmkqEWOEP9EgwDoLkXkGtY08bKdBeTzKYDcFGdBwlrgSSS", + "results": "2823 af736cc,2823 af736cc,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,808 5c2cac83,ERR_ZSTD,2823 af736cc,ERR_ZSTD,2823 af736cc,ERR_ZSTD" + }, + { + "frame": "KLUv/YRa4AAAAIoAANndAgASQwMF4A94gVH///////8DLqiBPg7tZxAoYpDaA0B9OIIS78n2wQ4sNgrY1c+AZNAYBP5RaoFA9i3TvTpKxwoILM0F/XsTSbkxacK/S2KxBHCmZlA8FKpN2a4TZWeEWQ==", + "results": "224 2ee8b580,224 2ee8b580,17 c5258290,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,17 c5258290,ERR_ZSTD,224 2ee8b580,ERR_ZSTD,224 2ee8b580,ERR_ZSTD" + }, + { + "frame": "KLUv/QRPXA8AOg/UAArg6YCVigDggtEBCQAJAAkAzf+//Xf9//99v/+/v+337/8f/7P9//////0H///p//+/338VgNmIwFhNWgNwEhIlHa5l/ka8kbasESz6ujDpEaqrZNxfYUOCNgoPLSZd3ZLCgyZeg7Qg9IIC5CuErivSqZUg2LU5IZqN8PwiEkwI4QRiQlQQB2DjQMVjCuqEDAkWZJMRK3nVRJhAWAp0XWghEymCa7eBi0JDzYdWBhHqRhC2NmNpNorwGBAPT6iFTbaJdjdRUIIP7VqWRe79KqBNuat9gUs1chdCKU3y7vdf1luS8e4UUOqAMBy0TWbcvAZBnQQwvcZbwt7Tl/ArWCxBqpVTIv2a0sUjsYY1QEwL0Eb/CMuHTHudCeJ3XEzra42AyGrBQcUQ67UGbiEoJSgSSk6bncGnlhLca91Kjx/VmtXwbuq37WYQa55h2m2RVmh7Y6PBLUtGbYQZkbCFbXSjmSXJcNg6hXFWe7cL246FCa1HNpQNYWJk67TsNhbPIPa9CXfew5NTC7kLhGH7UBb4xVR4sMhM+yCtA6ZBFx550Mkon5n2kAUHJrmh/h0M9ByJCmIC4D54geFAbWB0RAOKZOGpEI4CbyVzldKiVFWBqjqdRa7o3KYAfEeFmVczKHLNzPlUnAy0AQBuB4AKAA7QZTgAAAhEBFWmAJDoAQUABQAGANv/1/4P7/X9+h+/7/vdoQH/62/+AwFUFwoPXCGFDACaL+wDGaAVFCSNAVZV16c2AOA9GQDwfg8AeC+tKt4yADkANgCZiICO2BBDLt/66o3Kxtv2JWW1QB6NeoaBCnDfRDZZhCrfhGfKF2LWoHqYjFV7Me58E/vMcxwYhB1oMTpBEeCYgaDfdiNir0LTJXZDJwIDI9hU7HMCLYMcCSCw7CCjIgIuRJOL22dT58FrMrcZ1GG3F8qLVFcHAI9qYBTb6oGwiIpgdtgG/H4MpkUbg3cEGDdHhdgjznyeEjJCH84mFAS4F7/9GCYkCwc+7gQHZhkSJgKKLKocYZQkSJAk3sDJFdjWIFhiNygaQlogVj3puIZ4BCuAc6l2jSK4bTAkqLGYESqBMUZohIZBFU7tATBIIBKOvQMxFCExyggMMcYIjAAZgTLGBBJMMLEnE24DFAKYJ1UnDL3TedBiwLGmIbYE1Ya5EnAf3CMbdcIIJEbYkT+jGivSNvw/WZqatknrGAYRILHaRp7iQyvtIwswv4b1vICBjBA64Yt1QaHUih22IYYgF5titYDF2v6UMo8lTNcFkg==", + "results": "3280 ef966a16,3280 ef966a16,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,1430 d208aa94,ERR_ZSTD,3280 ef966a16,ERR_ZSTD,3280 ef966a16,ERR_ZSTD" + }, + { + "frame": "KLUv/URQrgAAAABtBACaFTgBD8AlHf//////7QDxN/C0Gg4ADwAOAL9/9szu7/+/tr7//v8DN+v/857/87dX99bv9lMP1/2zs35+7//7733+/wvfvb7+//f/j6//dXc/FaiQJJpIJpSwappzMEYwzBkfwFjHcAGS14oGyyNtZf0rox9BJdlH/kPuAWb4VKR9bnSPlVFV4y9aeq6Z6imu", + "results": "430 26ba970a,430 26ba970a,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,430 26ba970a,430 26ba970a,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,430 26ba970a,ERR_ZSTD,430 26ba970a,ERR_ZSTD" + } +] diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 08e315d2765e..855dc8dd5fee 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -12,6 +12,7 @@ import { afterAll, beforeAll, describe, expect, it } from "bun:test"; import { bunEnv, bunExe, isASAN, isLinux, libcPathForDlopen, rss } from "harness"; import { readFileSync } from "node:fs"; import zlib from "node:zlib"; +import frames from "./zstd-frames-fixture.json"; import path from "path"; // A hand-written empty frame: magic, a descriptor without a content size (so it goes through the @@ -1348,6 +1349,16 @@ describe("the output is that of zstd 1.5.7", () => { } }); + it("in a stream with workers, after others were given up in the middle of a frame", async () => { + const params = { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 1 }; + for (let i = 0; i < 4; i++) { + const abandoned = zlib.createZstdCompress({ params }); + abandoned.write(corpus); + abandoned.destroy(); + } + expect(zlib.zstdDecompressSync(await stream(params, false)).equals(corpus)).toBe(true); + }); + async function flushed(params: Record, sizes: number[]) { const stream = zlib.createZstdCompress({ params }); const chunks: Buffer[] = []; @@ -1386,3 +1397,37 @@ describe("the output is that of zstd 1.5.7", () => { ); }); }); + +// Frames from `decodecorpus`, zstd's own generator of anything the format allows, most of which no encoder makes, and +// what zstd 1.5.7 makes of each of them, whole and damaged in nine ways, through `node:zlib` and through `Bun`. +describe("frames that no encoder makes decode as with zstd 1.5.7", () => { + const digest = (output: Uint8Array) => `${output.length} ${Bun.hash.xxHash32(output).toString(16)}`; + const attempt = (decompress: (input: Buffer) => Uint8Array, input: Buffer) => { + try { + return digest(decompress(input)); + } catch (error: any) { + return error.code.replace("ZSTD_error_", ""); + } + }; + function* damaged(frame: Buffer) { + for (let k = 1; k <= 6; k++) { + const copy = Buffer.from(frame); + copy[Math.floor((frame.length * k) / 7)] ^= 1 << ((k * 3) % 8); + yield copy; + } + yield frame.subarray(0, Math.floor((frame.length * 3) / 4)); + yield frame.subarray(0, frame.length - 1); + yield Buffer.concat([frame, Buffer.from([0x28])]); + } + + it("whole and damaged", () => { + for (const { frame: base64, results } of frames) { + const frame = Buffer.from(base64, "base64"); + const actual: string[] = []; + for (const input of [frame, ...damaged(frame)]) { + actual.push(attempt(zlib.zstdDecompressSync, input), attempt(Bun.zstdDecompressSync, input)); + } + expect({ frame: base64, results: actual.join(",") }).toEqual({ frame: base64, results }); + } + }); +}); From 80386378d11869b143b50edf78a87a2a549bb469 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:06:08 +0000 Subject: [PATCH 033/199] zstd: fast with two segments goes about it as with one --- src/zstd/compress/fast.rs | 483 +++++++++++++++++++++------------- src/zstd/compress/internal.rs | 45 +++- 2 files changed, 335 insertions(+), 193 deletions(-) diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index e57d2a622e0f..fbed87cd80ac 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -373,6 +373,46 @@ impl<'a> NoDictBlock<'a> { } } +/// What `fast` and `dfast` work out before they look for matches in a block, when the window is two segments. +pub(super) struct ExtDictBlock<'a> { + pub(super) window: Segments<'a>, + /// The index of the first position of the current segment. + pub(super) prefix_start_index: u32, + /// Where the block starts in it. + pub(super) istart: u32, + pub(super) rep: SavedRep, +} + +impl<'a> ExtDictBlock<'a> { + /// `None` if nothing of the older segment is within reach. + pub(super) fn new( + ms: &MatchState, + rep: &[u32; ZSTD_REP_NUM], + src: &BlockSrc<'a>, + ) -> Result> { + let dict_start_index = + ms.get_lowest_match_index(src.index_of(src.end), ms.c_params.window_log); + let prefix_start_index = ms.window.dict_limit.max(dict_start_index); + if prefix_start_index == dict_start_index { + return Ok(None); + } + let start_index = src.index_of(src.start); + let older = (src.dict_buf) + .sub(src.dict_pos_of(dict_start_index)..src.dict_pos_of(prefix_start_index))?; + // As far as the start of the older segment is too far too. + let max_rep = start_index.wrapping_sub(dict_start_index); + Ok(Some(Self { + window: Segments { + near: Near::new(src.buf.sub(src.pos_of(prefix_start_index)..src.end)?)?, + older: Near::new(older)?, + }, + prefix_start_index, + istart: start_index - prefix_start_index, + rep: SavedRep::new(rep, max_rep.saturating_sub(1)), + })) + } +} + /// Where `fast` and `dfast` put the matches they find in a block, when there is no dictionary and the window is one /// segment. All of it is read and written once per match, so that the search for the next one has the registers. pub(super) struct MatchSink<'a, 'b> { @@ -400,6 +440,18 @@ impl<'a, 'b> MatchSink<'a, 'b> { } } + /// For a block that starts at the position `anchor` of `near`, which nothing is read before. + pub(super) fn with_near(seq_store: &'b mut SeqStore, near: Near<'a>, anchor: u32) -> Self { + Self { + seq_store: SeqSink::new(seq_store), + near, + older: near, + older_len: 0, + anchor, + failed: false, + } + } + /// How many bytes there are after the last match. pub(super) fn finish(self) -> Result { if self.failed { @@ -455,6 +507,85 @@ impl<'a, 'b> MatchSink<'a, 'b> { } } +/// What can be matched when that is two segments: the current one, and before it either the older one of the window or +/// the content of a dictionary. A place is a position in the two put end to end. +#[derive(Clone, Copy)] +pub(super) struct Segments<'a> { + /// The current segment, up to the end of the block. + pub(super) near: Near<'a>, + /// The other one, as far as it is within reach. + pub(super) older: Near<'a>, +} + +/// Where a place is. +#[derive(Clone, Copy)] +struct Spot<'a> { + segment: Near<'a>, + pos: u32, +} + +impl<'a> Segments<'a> { + #[inline(always)] + fn spot(self, place: u32) -> Spot<'a> { + let older_len = self.older.len(); + if place < older_len { + Spot { + segment: self.older, + pos: place, + } + } else { + Spot { + segment: self.near, + pos: place - older_len, + } + } + } + + /// The place of a position of the current segment. + #[inline(always)] + pub(super) const fn place_of(self, pos: u32) -> u32 { + pos.wrapping_add(self.older.len()) + } + + #[inline(always)] + pub(super) fn read8(self, place: u32) -> Option { + let spot = self.spot(place); + spot.segment.read8(spot.pos) + } + + /// The 4 bytes at `place`, if they are in one segment. + #[inline(always)] + pub(super) fn read32(self, place: u32) -> Option { + let spot = self.spot(place); + spot.segment.read32(spot.pos) + } + + /// The 8 bytes at `place`, if they are in one segment. + #[inline(always)] + pub(super) fn read64(self, place: u32) -> Option { + let spot = self.spot(place); + spot.segment.read64(spot.pos) + } + + /// `ZSTD_count_2segments`: how long what is at the position `ip` is the same as what is at `place`. + #[inline(always)] + pub(super) fn count(self, ip: u32, place: u32) -> u32 { + let spot = self.spot(place); + let len = self.near.count_in(ip, spot.segment, spot.pos); + if place.wrapping_add(len) != self.older.len() { + return len; + } + len + self.near.count(ip + len, 0) + } + + /// How many bytes before the position `ip`, down to `anchor`, are the same as those before `place` in its segment. + #[inline(always)] + pub(super) fn count_backwards(self, ip: u32, anchor: u32, place: u32) -> u32 { + let spot = self.spot(place); + (self.near).count_backwards_in(ip, spot.segment, spot.pos, (ip - anchor).min(spot.pos)) + } +} + /// What `fast` looks for matches with, when there is no dictionary and the window is one segment. struct Finder { /// The index of the first position. @@ -592,6 +723,144 @@ impl Finder { } } +impl Finder { + /// `ZSTD_compressBlock_fast_extDict_generic`, which goes about it as [`Self::find_matches`] does. + #[inline(never)] + fn find_matches_ext_dict( + &mut self, + hash_table: &mut [u32], + sink: &mut MatchSink<'_, '_>, + window: Segments<'_>, + ) { + const K_STEP_INCR: u32 = 1 << (K_SEARCH_STRENGTH - 1); + let Self { + base_index, + step_size, + hlog, + first: mut ip0, + mut rep_offset1, + mut rep_offset2, + } = *self; + let near = window.near; + let older_len = window.older.len(); + // The index of the first place. + let origin_index = base_index.wrapping_sub(older_len); + if hash_table.is_empty() { + return; + } + let Some(mut bytes0) = near.read64(ip0) else { + return; + }; + + 'block: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + let mut ip2 = ip0 + step; + let mut current0 = base_index.wrapping_add(ip0); + + // Ends with where 4 bytes are the same as earlier, and with the next position if it is still to be indexed. + let (ip, place, back, off_base, next) = loop { + let Some(ahead) = near.window::<{ HASH_READ_SIZE + 2 }>(ip2) else { + break 'block; + }; + let idx0 = table_swap(hash_table, hash_of::(bytes0, hlog), current0); + + // The 4 bytes have to be in one segment, with a byte before them. + let rep_place = window.place_of(ip2).wrapping_sub(rep_offset1); + if window.read32(rep_place) == Some(le32_at::<0, _>(ahead)) + && older_len.wrapping_sub(rep_place) >= 4 + && rep_offset1 > 0 + { + table_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); + let back = + u32::from(near.read8(ip2 - 1) == window.read8(rep_place.wrapping_sub(1))); + break (ip2, rep_place, back, REPCODE1_TO_OFFBASE, u32::MAX); + } + let mut place = idx0.wrapping_sub(origin_index); + let mut next = u32::MAX; + if window.read32(place) == Some(bytes0 as u32) { + table_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); + } else { + let bytes1 = bytes0 >> 8; + current0 += 1; + let idx1 = table_swap(hash_table, hash_of::(bytes1, hlog), current0); + place = idx1.wrapping_sub(origin_index); + if window.read32(place) != Some(bytes1 as u32) { + current0 = base_index.wrapping_add(ip2); + bytes0 = le64_at::<0, _>(ahead); + ip2 += step; + if ip2 >= next_step { + core::hint::cold_path(); + step += 1; + next_step += K_STEP_INCR; + } + continue; + } + next = ip2; + } + let ip = current0.wrapping_sub(base_index); + rep_offset2 = rep_offset1; + rep_offset1 = window.place_of(ip) - place; + let back = window.count_backwards(ip, sink.anchor, place); + break (ip, place, back, offset_to_offbase(rep_offset1), next); + }; + + ip0 = ip + 4 + window.count(ip + 4, place + 4); + sink.store(ip - back, ip0, off_base); + + if next < ip0 + && let Some(bytes) = near.read64(next) + { + table_put( + hash_table, + hash_of::(bytes, hlog), + base_index.wrapping_add(next), + ); + } + + // `ip0` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { + break 'block; + }; + if let Some(to_insert) = near.read64((current0 + 2).wrapping_sub(base_index)) { + table_put(hash_table, hash_of::(to_insert, hlog), current0 + 2); + } + table_put( + hash_table, + hash_of::(le64_at::<0, _>(around), hlog), + base_index.wrapping_add(ip0 - 2), + ); + bytes0 = le64_at::<2, _>(around); + + loop { + let rep_place = window.place_of(ip0).wrapping_sub(rep_offset2); + if !(window.read32(rep_place) == Some(bytes0 as u32) + && index_overlap_check(older_len, rep_place) + && rep_offset2 > 0) + { + break; + } + let end = ip0 + 4 + window.count(ip0 + 4, rep_place + 4); + core::mem::swap(&mut rep_offset1, &mut rep_offset2); + table_put( + hash_table, + hash_of::(bytes0, hlog), + base_index.wrapping_add(ip0), + ); + sink.store(ip0, end, REPCODE1_TO_OFFBASE); + ip0 = end; + let Some(bytes) = near.read64(ip0) else { + break 'block; + }; + bytes0 = bytes; + } + } + + self.rep_offset1 = rep_offset1; + self.rep_offset2 = rep_offset2; + } +} + /// `ZSTD_compressBlock_fast` pub(crate) fn compress_block_fast( ms: &mut MatchState, @@ -846,183 +1115,6 @@ pub(crate) fn compress_block_fast_dict_match_state( } } -/// `ZSTD_compressBlock_fast_extDict_generic` -fn compress_block_fast_ext_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - seg: &TwoSegments<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << (K_SEARCH_STRENGTH - 1); - - let hlog = ms.c_params.hash_log; - let step_size = least_step(&ms.c_params) as usize + 1; - let mut anchor = seg.istart; - let mut ip0 = seg.istart; - let SavedRep { - mut offset_1, - mut offset_2, - offset_saved1, - offset_saved2, - } = SavedRep::with_ext_dict(rep, src, seg); - - // Whether there is a match for the position `ip` at `idx`, which is from the table. - let match_found = |ip: usize, idx: u32| -> Result { - let candidate = seg.locate(idx); - Ok(idx >= seg.dict_start_index - && read_le32(candidate.buf, candidate.pos)? == read_le32(seg.buf, ip)?) - }; - let hash_table = ms.hash_table.as_mut_slice(); - - 'start: loop { - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - - let mut ip1 = ip0 + 1; - let mut ip2 = ip0 + step; - let mut ip3 = ip2 + 1; - if ip3 >= seg.ilimit { - break 'start; - } - - let mut hash0 = hash_ptr(seg.buf, ip0, hlog, MLS)?; - let mut hash1 = hash_ptr(seg.buf, ip1, hlog, MLS)?; - let mut idx = hash_table.at(hash0)?; - let mut current0; - - // Ends with the start and the length so far of a repeat match, or with `None` for a match at `idx`. - let rep_match = loop { - let current2 = src.index_of(ip2); - let rep_index = current2.wrapping_sub(offset_1); - let mut rep_match = seg.locate(rep_index); - // The 4 bytes have to be in one segment, with a byte before them. - let rep_found = ((seg.prefix_start_index.wrapping_sub(rep_index) >= 4) - & (offset_1 > 0)) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(seg.buf, ip2)?; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if rep_found { - ip0 = ip2; - let back = usize::from( - seg.buf.at(ip0 - 1)? == rep_match.buf.at(rep_match.pos.wrapping_sub(1))?, - ); - ip0 -= back; - rep_match.pos = rep_match.pos.wrapping_sub(back); - break Some((rep_match, 4 + back)); - } - - if match_found(ip0, idx)? { - break None; - } - - idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(seg.buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip3; - - current0 = src.index_of(ip0); - hash_table.put(hash0, current0)?; - - if match_found(ip0, idx)? { - break None; - } - - idx = hash_table.at(hash1)?; - hash0 = hash1; - hash1 = hash_ptr(seg.buf, ip2, hlog, MLS)?; - - ip0 = ip1; - ip1 = ip2; - ip2 = ip0 + step; - ip3 = ip1 + step; - - if ip2 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - if ip3 >= seg.ilimit { - break 'start; - } - }; - - let (offcode, match0, mut m_length) = match rep_match { - Some((match0, m_length)) => (REPCODE1_TO_OFFBASE, match0, m_length), - None => { - let offset = current0.wrapping_sub(idx); - let mut match0 = seg.locate(idx); - offset_2 = offset_1; - offset_1 = offset; - let back = - count_backwards(seg.buf, ip0, anchor, match0.buf, match0.pos, match0.start); - ip0 -= back; - match0.pos -= back; - (offset_to_offbase(offset), match0, 4 + back) - } - }; - - m_length += count_2segments( - seg.buf, - ip0 + m_length, - match0.buf, - match0.pos + m_length, - seg.iend, - match0.end, - seg.prefix_start, - )?; - seq_store.store_seq(ip0 - anchor, seg.buf, anchor, offcode, m_length)?; - ip0 += m_length; - anchor = ip0; - - if ip1 < ip0 { - hash_table.put(hash1, src.index_of(ip1))?; - } - - if ip0 <= seg.ilimit { - hash_table.put( - hash_ptr(seg.buf, src.pos_of(current0 + 2), hlog, MLS)?, - current0 + 2, - )?; - hash_table.put( - hash_ptr(seg.buf, ip0 - 2, hlog, MLS)?, - src.index_of(ip0 - 2), - )?; - - while ip0 <= seg.ilimit { - let rep_index2 = src.index_of(ip0).wrapping_sub(offset_2); - let rep_match2 = seg.locate(rep_index2); - if !((index_overlap_check(seg.prefix_start_index, rep_index2) & (offset_2 > 0)) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(seg.buf, ip0)?) - { - break; - } - let rep_length2 = count_2segments( - seg.buf, - ip0 + 4, - rep_match2.buf, - rep_match2.pos + 4, - seg.iend, - rep_match2.end, - seg.prefix_start, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, seg.buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_table.put(hash_ptr(seg.buf, ip0, hlog, MLS)?, src.index_of(ip0))?; - ip0 += rep_length2; - anchor = ip0; - } - } - } - - restore_rep(rep, offset_1, offset_2, offset_saved1, offset_saved2); - Ok(seg.iend - anchor) -} - /// `ZSTD_compressBlock_fast_extDict` pub(crate) fn compress_block_fast_ext_dict( ms: &mut MatchState, @@ -1030,17 +1122,34 @@ pub(crate) fn compress_block_fast_ext_dict( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { - let Some(seg) = &TwoSegments::with_ext_dict(ms, src)? else { - return Ok(src.end - src.start); - }; - // Nothing of the older segment is within reach. - if seg.prefix_start_index == seg.dict_start_index { + let Some(block) = ExtDictBlock::new(ms, rep, src)? else { + // Nothing of the older segment is within reach. return compress_block_fast(ms, seq_store, rep, src); - } + }; + let hlog = ms.c_params.hash_log; + let mut finder = Finder { + base_index: block.prefix_start_index, + step_size: least_step(&ms.c_params) + 1, + hlog, + first: block.istart, + rep_offset1: block.rep.offset_1, + rep_offset2: block.rep.offset_2, + }; + let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); + let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; match ms.c_params.min_match { - 5 => compress_block_fast_ext_dict_generic::<5>(ms, seq_store, rep, src, seg), - 6 => compress_block_fast_ext_dict_generic::<6>(ms, seq_store, rep, src, seg), - 7 => compress_block_fast_ext_dict_generic::<7>(ms, seq_store, rep, src, seg), - _ => compress_block_fast_ext_dict_generic::<4>(ms, seq_store, rep, src, seg), + 5 => finder.find_matches_ext_dict::<5>(hash_table, &mut sink, block.window), + 6 => finder.find_matches_ext_dict::<6>(hash_table, &mut sink, block.window), + 7 => finder.find_matches_ext_dict::<7>(hash_table, &mut sink, block.window), + _ => finder.find_matches_ext_dict::<4>(hash_table, &mut sink, block.window), } + let last_literals = sink.finish()?; + restore_rep( + rep, + finder.rep_offset1, + finder.rep_offset2, + block.rep.offset_saved1, + block.rep.offset_saved2, + ); + Ok(last_literals) } diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 153328170cb2..da616eaed3cc 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -744,6 +744,27 @@ impl<'a> Near<'a> { len } + /// How long what is at `pos` is the same as what is at `other_pos` in `other`, up to the end of either. + #[inline(always)] + pub(crate) fn count_in(self, pos: u32, other: Near<'_>, other_pos: u32) -> u32 { + let mut len = 0; + while let (Some(bytes), Some(other_bytes)) = + (self.chunk16(pos + len), other.chunk16(other_pos + len)) + { + let diff = u128::from_le_bytes(*bytes) ^ u128::from_le_bytes(*other_bytes); + if diff != 0 { + return len + (diff.trailing_zeros() >> 3); + } + len += 16; + } + while let Some(byte) = self.read8(pos + len) + && other.read8(other_pos + len) == Some(byte) + { + len += 1; + } + len + } + /// How many bytes before `pos`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. #[inline(always)] pub(crate) fn count_backwards( @@ -754,25 +775,37 @@ impl<'a> Near<'a> { match_low: u32, ) -> u32 { let most = (pos - anchor).min(match_pos - match_low); + self.count_backwards_in(pos, self, match_pos, most) + } + + /// How many bytes before `pos` are the same as those before `other_pos` in `other`, up to `most`. + #[inline(always)] + pub(crate) fn count_backwards_in( + self, + pos: u32, + other: Near<'_>, + other_pos: u32, + most: u32, + ) -> u32 { // Whether there is anything to compare is as good as random: it takes no branch to find out. - if let (Some(bytes), Some(match_bytes)) = ( + if let (Some(bytes), Some(other_bytes)) = ( self.read64(pos.wrapping_sub(8)), - self.read64(match_pos.wrapping_sub(8)), + other.read64(other_pos.wrapping_sub(8)), ) { - let same = (bytes ^ match_bytes).leading_zeros() >> 3; + let same = (bytes ^ other_bytes).leading_zeros() >> 3; if same < 8 || most <= 8 { return same.min(most); } } - self.count_backwards_bytewise(pos, match_pos, most) + self.count_backwards_bytewise(pos, other, other_pos, most) } #[cold] - fn count_backwards_bytewise(self, pos: u32, match_pos: u32, most: u32) -> u32 { + fn count_backwards_bytewise(self, pos: u32, other: Near<'_>, other_pos: u32, most: u32) -> u32 { let mut len = 0; while len < most && let Some(byte) = self.read8(pos - len - 1) - && self.read8(match_pos - len - 1) == Some(byte) + && other.read8(other_pos - len - 1) == Some(byte) { len += 1; } From 671862eb86fc3b5d72dc2009c3b9b573b16c7374 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:06:50 +0000 Subject: [PATCH 034/199] [autofix.ci] apply automated fixes --- test/js/bun/util/zstd.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 855dc8dd5fee..7d88d9221aec 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -12,8 +12,8 @@ import { afterAll, beforeAll, describe, expect, it } from "bun:test"; import { bunEnv, bunExe, isASAN, isLinux, libcPathForDlopen, rss } from "harness"; import { readFileSync } from "node:fs"; import zlib from "node:zlib"; -import frames from "./zstd-frames-fixture.json"; import path from "path"; +import frames from "./zstd-frames-fixture.json"; // A hand-written empty frame: magic, a descriptor without a content size (so it goes through the // streaming decoder), the window descriptor (the decoder allocates a window of 2^windowLog bytes before From 4a978bc272223d0ecacad9312baf5e50f1d8c59e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:07:59 +0000 Subject: [PATCH 035/199] zstd: dfast with two segments reads the input once per position --- src/zstd/compress/double_fast.rs | 278 +++++++++++++++---------------- 1 file changed, 131 insertions(+), 147 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index d24daa4ec656..8a451a858129 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -1,7 +1,8 @@ //! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. use super::fast::{ - FAST_HASH_FILL_STEP, MatchSink, NoDictBlock, TwoSegments, count_backwards, restore_rep, + ExtDictBlock, FAST_HASH_FILL_STEP, MatchSink, NoDictBlock, Segments, TwoSegments, + count_backwards, restore_rep, }; use super::internal::{ BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, @@ -586,156 +587,123 @@ pub(crate) fn compress_block_double_fast_dict_match_state( } } -/// `ZSTD_compressBlock_doubleFast_extDict_generic` -fn compress_block_double_fast_ext_dict_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - seg: &TwoSegments<'_>, -) -> Result { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let [mut offset_1, mut offset_2, _] = *rep; - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - - let mut anchor = seg.istart; - let mut ip = seg.istart; - - while ip < seg.ilimit { - let h_small = hash_ptr(seg.buf, ip, h_bits_s, MLS)?; - let match_index = hash_small.at(h_small)?; - let match_s = seg.locate(match_index); - - let h_long = hash_ptr(seg.buf, ip, h_bits_l, 8)?; - let match_long_index = hash_long.at(h_long)?; - let match_long = seg.locate(match_long_index); +impl Finder { + /// `ZSTD_compressBlock_doubleFast_extDict_generic` + #[inline(never)] + fn find_matches_ext_dict( + &mut self, + hash_long: &mut [u32], + hash_small: &mut [u32], + sink: &mut MatchSink<'_, '_>, + window: Segments<'_>, + ) { + let Self { + base_index, + h_bits_l, + h_bits_s, + first: mut ip, + mut offset_1, + mut offset_2, + } = *self; + let near = window.near; + let older_len = window.older.len(); + // The index of the first place. + let origin_index = base_index.wrapping_sub(older_len); + if hash_long.is_empty() || hash_small.is_empty() { + return; + } - let curr = src.index_of(ip); - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = seg.locate(rep_index); - hash_long.put(h_long, curr)?; - hash_small.put(h_small, curr)?; - - let m_length; - if (index_overlap_check(seg.prefix_start_index, rep_index) - & (offset_1 <= curr.wrapping_add(1).wrapping_sub(seg.dict_start_index))) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(seg.buf, ip + 1)? - { - m_length = count_2segments( - seg.buf, - ip + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - seg.iend, - rep_match.end, - seg.prefix_start, - )? + 4; - ip += 1; - seq_store.store_seq(ip - anchor, seg.buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - } else { - // The match, how much of it is known, and its offset. `ip` is moved to where it is for. - let (found, known, offset) = if match_long_index > seg.dict_start_index - && read_le64(match_long.buf, match_long.pos)? == read_le64(seg.buf, ip)? - { - (match_long, 8, curr.wrapping_sub(match_long_index)) - } else if match_index > seg.dict_start_index - && read_le32(match_s.buf, match_s.pos)? == read_le32(seg.buf, ip)? + // `ip` is before the last position that can be looked at. + while let Some(here) = near.window::<{ HASH_READ_SIZE + 1 }>(ip) { + let bytes0 = le64_at::<0, _>(here); + let curr = base_index.wrapping_add(ip); + let place_long = + table_swap(hash_long, hash8(bytes0, h_bits_l, 0), curr).wrapping_sub(origin_index); + let place_short = table_swap(hash_small, hash_of::(bytes0, h_bits_s), curr) + .wrapping_sub(origin_index); + + let rep_place = window.place_of(ip + 1).wrapping_sub(offset_1); + if window.read32(rep_place) == Some((bytes0 >> 8) as u32) + && index_overlap_check(older_len, rep_place) + && offset_1 <= window.place_of(ip + 1) { - // A short match: a long one at the next position is preferred. - let h3 = hash_ptr(seg.buf, ip + 1, h_bits_l, 8)?; - let match_index3 = hash_long.at(h3)?; - let match3 = seg.locate(match_index3); - hash_long.put(h3, curr + 1)?; - if match_index3 > seg.dict_start_index - && read_le64(match3.buf, match3.pos)? == read_le64(seg.buf, ip + 1)? + ip += 1; + let end = ip + 4 + window.count(ip + 4, rep_place + 4); + sink.store(ip, end, REPCODE1_TO_OFFBASE); + ip = end; + } else { + // The match, and how much of it is known. `ip` is moved to where it is for. + let (place, known) = if window.read64(place_long) == Some(bytes0) && place_long > 0 { - ip += 1; - (match3, 8, curr.wrapping_add(1).wrapping_sub(match_index3)) + (place_long, 8) + } else if window.read32(place_short) == Some(bytes0 as u32) && place_short > 0 { + // A short match: a long one at the next position is preferred. + let bytes1 = le64_at::<1, _>(here); + let place3 = table_swap(hash_long, hash8(bytes1, h_bits_l, 0), curr + 1) + .wrapping_sub(origin_index); + if window.read64(place3) == Some(bytes1) && place3 > 0 { + ip += 1; + (place3, 8) + } else { + (place_short, 4) + } } else { - (match_s, 4, curr.wrapping_sub(match_index)) - } - } else { - ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; - continue; - }; - let forwards = count_2segments( - seg.buf, - ip + known, - found.buf, - found.pos + known, - seg.iend, - found.end, - seg.prefix_start, - )? + known; - let back = count_backwards(seg.buf, ip, anchor, found.buf, found.pos, found.start); - ip -= back; - m_length = forwards + back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip - anchor, - seg.buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - } - - ip += m_length; - anchor = ip; + ip += ((ip - sink.anchor) >> K_SEARCH_STRENGTH) + 1; + continue; + }; + let end = ip + known + window.count(ip + known, place + known); + let back = window.count_backwards(ip, sink.anchor, place); + offset_2 = offset_1; + offset_1 = window.place_of(ip) - place; + sink.store(ip - back, end, offset_to_offbase(offset_1)); + ip = end; + } - if ip <= seg.ilimit { + // `ip` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip - 2) else { + break; + }; let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(seg.buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put( - hash_ptr(seg.buf, ip - 2, h_bits_l, 8)?, - src.index_of(ip - 2), - )?; - hash_small.put( - hash_ptr(seg.buf, to_insert, h_bits_s, MLS)?, - index_to_insert, - )?; - hash_small.put( - hash_ptr(seg.buf, ip - 1, h_bits_s, MLS)?, - src.index_of(ip - 1), - )?; + if let Some(to_insert) = near.read64(index_to_insert.wrapping_sub(base_index)) { + table_put(hash_long, hash8(to_insert, h_bits_l, 0), index_to_insert); + table_put( + hash_small, + hash_of::(to_insert, h_bits_s), + index_to_insert, + ); + } + table_put( + hash_long, + hash8(le64_at::<0, _>(around), h_bits_l, 0), + base_index.wrapping_add(ip - 2), + ); + table_put( + hash_small, + hash_of::(le64_at::<1, _>(around), h_bits_s), + base_index.wrapping_add(ip - 1), + ); - while ip <= seg.ilimit { - let current2 = src.index_of(ip); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = seg.locate(rep_index2); - if !((index_overlap_check(seg.prefix_start_index, rep_index2) - & (offset_2 <= current2.wrapping_sub(seg.dict_start_index))) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(seg.buf, ip)?) + while let Some(bytes) = near.read64(ip) { + let rep_place = window.place_of(ip).wrapping_sub(offset_2); + if !(window.read32(rep_place) == Some(bytes as u32) + && index_overlap_check(older_len, rep_place) + && offset_2 <= window.place_of(ip)) { break; } - let rep_length2 = count_2segments( - seg.buf, - ip + 4, - rep_match2.buf, - rep_match2.pos + 4, - seg.iend, - rep_match2.end, - seg.prefix_start, - )? + 4; + let end = ip + 4 + window.count(ip + 4, rep_place + 4); core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, seg.buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_small.put(hash_ptr(seg.buf, ip, h_bits_s, MLS)?, current2)?; - hash_long.put(hash_ptr(seg.buf, ip, h_bits_l, 8)?, current2)?; - ip += rep_length2; - anchor = ip; + let index = base_index.wrapping_add(ip); + table_put(hash_small, hash_of::(bytes, h_bits_s), index); + table_put(hash_long, hash8(bytes, h_bits_l, 0), index); + sink.store(ip, end, REPCODE1_TO_OFFBASE); + ip = end; } } - } - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(seg.iend - anchor) + self.offset_1 = offset_1; + self.offset_2 = offset_2; + } } /// `ZSTD_compressBlock_doubleFast_extDict` @@ -745,17 +713,33 @@ pub(crate) fn compress_block_double_fast_ext_dict( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { - let Some(seg) = &TwoSegments::with_ext_dict(ms, src)? else { - return Ok(src.end - src.start); - }; - // Nothing of the older segment is within reach. - if seg.prefix_start_index == seg.dict_start_index { + let Some(block) = ExtDictBlock::new(ms, rep, src)? else { + // Nothing of the older segment is within reach. return compress_block_double_fast(ms, seq_store, rep, src); - } + }; + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let [offset_1, offset_2, _] = *rep; + let mut finder = Finder { + base_index: block.prefix_start_index, + h_bits_l, + h_bits_s, + first: block.istart, + offset_1, + offset_2, + }; + let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); + let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; + let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; match ms.c_params.min_match { - 5 => compress_block_double_fast_ext_dict_generic::<5>(ms, seq_store, rep, src, seg), - 6 => compress_block_double_fast_ext_dict_generic::<6>(ms, seq_store, rep, src, seg), - 7 => compress_block_double_fast_ext_dict_generic::<7>(ms, seq_store, rep, src, seg), - _ => compress_block_double_fast_ext_dict_generic::<4>(ms, seq_store, rep, src, seg), + 5 => finder.find_matches_ext_dict::<5>(hash_long, hash_small, &mut sink, block.window), + 6 => finder.find_matches_ext_dict::<6>(hash_long, hash_small, &mut sink, block.window), + 7 => finder.find_matches_ext_dict::<7>(hash_long, hash_small, &mut sink, block.window), + _ => finder.find_matches_ext_dict::<4>(hash_long, hash_small, &mut sink, block.window), } + let last_literals = sink.finish()?; + let [rep0, rep1, _] = rep; + *rep0 = finder.offset_1; + *rep1 = finder.offset_2; + Ok(last_literals) } From a393a00fa96fd69bc303d2b3e01632f0f147b41f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:10:29 +0000 Subject: [PATCH 036/199] zstd: fast with a dictionary's match state reads the input once per position --- src/zstd/compress/double_fast.rs | 10 +- src/zstd/compress/fast.rs | 406 ++++++++++++++++--------------- 2 files changed, 213 insertions(+), 203 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 8a451a858129..e2631fdc7da9 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -622,12 +622,11 @@ impl Finder { let place_short = table_swap(hash_small, hash_of::(bytes0, h_bits_s), curr) .wrapping_sub(origin_index); - let rep_place = window.place_of(ip + 1).wrapping_sub(offset_1); - if window.read32(rep_place) == Some((bytes0 >> 8) as u32) - && index_overlap_check(older_len, rep_place) + if window.read32_before(ip + 1, offset_1) == Some((bytes0 >> 8) as u32) && offset_1 <= window.place_of(ip + 1) { ip += 1; + let rep_place = window.place_of(ip) - offset_1; let end = ip + 4 + window.count(ip + 4, rep_place + 4); sink.store(ip, end, REPCODE1_TO_OFFBASE); ip = end; @@ -684,13 +683,12 @@ impl Finder { ); while let Some(bytes) = near.read64(ip) { - let rep_place = window.place_of(ip).wrapping_sub(offset_2); - if !(window.read32(rep_place) == Some(bytes as u32) - && index_overlap_check(older_len, rep_place) + if !(window.read32_before(ip, offset_2) == Some(bytes as u32) && offset_2 <= window.place_of(ip)) { break; } + let rep_place = window.place_of(ip) - offset_2; let end = ip + 4 + window.count(ip + 4, rep_place + 4); core::mem::swap(&mut offset_1, &mut offset_2); let index = base_index.wrapping_add(ip); diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index fbed87cd80ac..e62a353f31cf 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -4,7 +4,7 @@ use super::internal::{ BlockSrc, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, count, count_2segments, hash_of, hash_ptr, - index_overlap_check, le32_at, le64_at, offset_to_offbase, table_put, table_swap, + index_overlap_check, le32_at, le64_at, offset_to_offbase, table_get, table_put, table_swap, write_tagged_index, }; use crate::common::ZSTD_REP_NUM; @@ -567,6 +567,17 @@ impl<'a> Segments<'a> { spot.segment.read64(spot.pos) } + /// The 4 bytes that are `offset` before the position `pos`, if they are in one segment. Which one that is does not + /// change much from one time to the next. + #[inline(always)] + pub(super) fn read32_before(self, pos: u32, offset: u32) -> Option { + let at = pos.wrapping_sub(offset); + match self.near.read32(at) { + Some(bytes) => Some(bytes), + None => self.older.read32(self.place_of(at)), + } + } + /// `ZSTD_count_2segments`: how long what is at the position `ip` is the same as what is at `place`. #[inline(always)] pub(super) fn count(self, ip: u32, place: u32) -> u32 { @@ -767,8 +778,8 @@ impl Finder { // The 4 bytes have to be in one segment, with a byte before them. let rep_place = window.place_of(ip2).wrapping_sub(rep_offset1); - if window.read32(rep_place) == Some(le32_at::<0, _>(ahead)) - && older_len.wrapping_sub(rep_place) >= 4 + if window.read32_before(ip2, rep_offset1) == Some(le32_at::<0, _>(ahead)) + && rep_place != older_len && rep_offset1 > 0 { table_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); @@ -832,14 +843,8 @@ impl Finder { ); bytes0 = le64_at::<2, _>(around); - loop { + while window.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) && rep_offset2 > 0 { let rep_place = window.place_of(ip0).wrapping_sub(rep_offset2); - if !(window.read32(rep_place) == Some(bytes0 as u32) - && index_overlap_check(older_len, rep_place) - && rep_offset2 > 0) - { - break; - } let end = ip0 + 4 + window.count(ip0 + 4, rep_place + 4); core::mem::swap(&mut rep_offset1, &mut rep_offset2); table_put( @@ -900,201 +905,187 @@ pub(crate) fn compress_block_fast( Ok(last_literals) } -/// `ZSTD_compressBlock_fast_dictMatchState_generic` -fn compress_block_fast_dict_match_state_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - seg: &TwoSegments<'_>, -) -> Result { - const K_STEP_INCR: usize = 1 << K_SEARCH_STRENGTH; +/// What `fast` and `dfast` work out before they look for matches in a block, when a dictionary has its own match state. +pub(super) struct DictBlock<'a> { + /// The current segment, and the content of the dictionary. + pub(super) window: Segments<'a>, + /// The index of the first position of the current segment. + pub(super) prefix_start_index: u32, + /// The index that the start of the content has in the dictionary's tables. + pub(super) dict_start_index: u32, + /// Where the block starts in the current segment. + pub(super) istart: u32, +} - let hlog = ms.c_params.hash_log; - let step_size = least_step(&ms.c_params) as usize; - let [mut offset_1, mut offset_2, _] = *rep; - let hash_table = ms.hash_table.as_mut_slice(); - let dms = src.dms.as_ref().ok_or(BUG)?; - let dict_hash_table = dms.ms.hash_table.as_slice(); - let dict_h_bits = dms.ms.tagged_hash_log(); - - let mut anchor = seg.istart; - let mut ip0 = seg.first(); - let mut ip1 = seg.istart + step_size; - - 'search: while ip1 <= seg.ilimit { - let mut hash0 = hash_ptr(seg.buf, ip0, hlog, MLS)?; - - let dict_hash_and_tag0 = hash_ptr(seg.buf, ip0, dict_h_bits, MLS)?; - let mut dict_match_index_and_tag = - dict_hash_table.at(dict_hash_and_tag0 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let mut dict_tags_match = - compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag0); - - let mut match_index = hash_table.at(hash0)?; - let mut curr = src.index_of(ip0); - let mut step = step_size; - let mut next_step = ip0 + K_STEP_INCR; - - let m_length = loop { - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = seg.locate(rep_index); - let hash1 = hash_ptr(seg.buf, ip1, hlog, MLS)?; - let dict_hash_and_tag1 = hash_ptr(seg.buf, ip1, dict_h_bits, MLS)?; - hash_table.put(hash0, curr)?; - - if index_overlap_check(seg.prefix_start_index, rep_index) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(seg.buf, ip0 + 1)? - { - let m_length = count_2segments( - seg.buf, - ip0 + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - seg.iend, - rep_match.end, - seg.prefix_start, - )? + 4; - ip0 += 1; - seq_store.store_seq( - ip0 - anchor, - seg.buf, - anchor, - REPCODE1_TO_OFFBASE, - m_length, - )?; - break m_length; - } +impl<'a> DictBlock<'a> { + pub(super) fn new(src: &BlockSrc<'a>, prefix_start_index: u32) -> Result { + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_start_index = dms.ms.window.dict_limit; + Ok(Self { + window: Segments { + near: Near::new(src.buf.sub(src.pos_of(prefix_start_index)..src.end)?)?, + older: Near::new(dms.buf.sub(dms.pos_of(dict_start_index)..dms.end)?)?, + }, + prefix_start_index, + dict_start_index, + istart: src.index_of(src.start).wrapping_sub(prefix_start_index), + }) + } - if dict_tags_match { - let dict_match_index = dict_match_index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match = seg.dict_pos_of(dict_match_index); - // A match in the dictionary is only taken if there is no candidate in the prefix. - if dict_match_index > seg.dict_start_index - && read_le32(seg.dict_buf, dict_match)? == read_le32(seg.buf, ip0)? - && match_index <= seg.prefix_start_index - { - let offset = curr - .wrapping_sub(dict_match_index) - .wrapping_sub(seg.dict_index_delta); - let mut m_length = count_2segments( - seg.buf, - ip0 + 4, - seg.dict_buf, - dict_match + 4, - seg.iend, - seg.dict_end, - seg.prefix_start, - )? + 4; - let back = count_backwards( - seg.buf, - ip0, - anchor, - seg.dict_buf, - dict_match, - seg.dict_start, - ); - ip0 -= back; - m_length += back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip0 - anchor, - seg.buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - break m_length; - } - } + /// The first position to look at: not `istart` if there is nothing before it to match. + pub(super) const fn first(&self) -> u32 { + self.istart + (self.window.place_of(self.istart) == 0) as u32 + } +} - let match_pos = src.pos_of(match_index); - if match4_found(seg.buf, ip0, match_pos, match_index, seg.prefix_start_index)? { - let offset = ip0.wrapping_sub(match_pos) as u32; - let mut m_length = count(seg.buf, ip0 + 4, seg.buf, match_pos + 4, seg.iend)? + 4; - let back = - count_backwards(seg.buf, ip0, anchor, seg.buf, match_pos, seg.prefix_start); - ip0 -= back; - m_length += back; - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip0 - anchor, - seg.buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - break m_length; - } +/// A table of a dictionary, where entries have tags. +#[derive(Clone, Copy)] +pub(super) struct TaggedTable<'a> { + table: &'a [u32], + /// How many bits to hash to, tag included. + h_bits: u32, + /// The index that the first place has in the table. + start_index: u32, +} - dict_match_index_and_tag = - dict_hash_table.at(dict_hash_and_tag1 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - dict_tags_match = - compare_packed_tags(dict_match_index_and_tag as usize, dict_hash_and_tag1); - match_index = hash_table.at(hash1)?; +impl<'a> TaggedTable<'a> { + pub(super) fn new(table: &'a [u32], h_bits: u32, start_index: u32) -> Result { + Ok(Self { + table: table.sub(..1 << (h_bits - ZSTD_SHORT_CACHE_TAG_BITS))?, + h_bits, + start_index, + }) + } - if ip1 >= next_step { - step += 1; - next_step += K_STEP_INCR; - } - ip0 = ip1; - ip1 += step; - if ip1 > seg.ilimit { - break 'search; - } + /// The place that the table has for the `MLS` first bytes of `bytes`. `None` if the tag tells that it is for + /// something else, or if it is the first place, which stands for none. + #[inline(always)] + pub(super) fn get(self, bytes: u64) -> Option { + let hash_and_tag = hash_of::(bytes, self.h_bits); + let index_and_tag = table_get(self.table, hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS); + let index = index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + (compare_packed_tags(index_and_tag as usize, hash_and_tag) && index > self.start_index) + .then(|| index - self.start_index) + } +} - curr = src.index_of(ip0); - hash0 = hash1; +impl Finder { + /// `ZSTD_compressBlock_fast_dictMatchState_generic` + #[inline(never)] + fn find_matches_dict( + &mut self, + hash_table: &mut [u32], + sink: &mut MatchSink<'_, '_>, + window: Segments<'_>, + dict_table: TaggedTable<'_>, + ) { + const K_STEP_INCR: u32 = 1 << K_SEARCH_STRENGTH; + let Self { + base_index, + step_size, + hlog, + first: mut ip0, + mut rep_offset1, + mut rep_offset2, + } = *self; + let near = window.near; + if hash_table.is_empty() || dict_table.table.is_empty() { + return; + } + let Some(mut bytes0) = near.read64(ip0) else { + return; }; + let mut ip1 = sink.anchor + step_size; + + 'block: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + let mut curr; + + // One turn per position. Ends with the place of a match, or with `None` once a repeat is stored. + let place = loop { + let Some(bytes1) = near.read64(ip1) else { + break 'block; + }; + curr = base_index.wrapping_add(ip0); + let match_index = table_swap(hash_table, hash_of::(bytes0, hlog), curr); + + if window.read32_before(ip0 + 1, rep_offset1) == Some((bytes0 >> 8) as u32) { + ip0 += 1; + let rep_place = window.place_of(ip0).wrapping_sub(rep_offset1); + let end = ip0 + 4 + window.count(ip0 + 4, rep_place + 4); + sink.store(ip0, end, REPCODE1_TO_OFFBASE); + ip0 = end; + break None; + } - ip0 += m_length; - anchor = ip0; - - if ip0 <= seg.ilimit { - hash_table.put( - hash_ptr(seg.buf, src.pos_of(curr + 2), hlog, MLS)?, - curr + 2, - )?; - hash_table.put( - hash_ptr(seg.buf, ip0 - 2, hlog, MLS)?, - src.index_of(ip0 - 2), - )?; - - while ip0 <= seg.ilimit { - let current2 = src.index_of(ip0); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = seg.locate(rep_index2); - if !(index_overlap_check(seg.prefix_start_index, rep_index2) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(seg.buf, ip0)?) + // A match in the dictionary is only taken if there is no candidate in the prefix. + if let Some(place) = dict_table.get::(bytes0) + && window.older.read32(place) == Some(bytes0 as u32) + && match_index <= base_index { - break; + break Some(place); } - let rep_length2 = count_2segments( - seg.buf, - ip0 + 4, - rep_match2.buf, - rep_match2.pos + 4, - seg.iend, - rep_match2.end, - seg.prefix_start, - )? + 4; - core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, seg.buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_table.put(hash_ptr(seg.buf, ip0, hlog, MLS)?, current2)?; - ip0 += rep_length2; - anchor = ip0; + let match_pos = match_index.wrapping_sub(base_index); + if near.diff64(match_pos, bytes0) as u32 == 0 { + break Some(window.place_of(match_pos)); + } + + if ip1 >= next_step { + core::hint::cold_path(); + step += 1; + next_step += K_STEP_INCR; + } + ip0 = ip1; + ip1 += step; + bytes0 = bytes1; + }; + + if let Some(place) = place { + let end = ip0 + 4 + window.count(ip0 + 4, place + 4); + let back = window.count_backwards(ip0, sink.anchor, place); + rep_offset2 = rep_offset1; + rep_offset1 = window.place_of(ip0) - place; + sink.store(ip0 - back, end, offset_to_offbase(rep_offset1)); + ip0 = end; + } + + // `ip0` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { + break 'block; + }; + if let Some(to_insert) = near.read64((curr + 2).wrapping_sub(base_index)) { + table_put(hash_table, hash_of::(to_insert, hlog), curr + 2); + } + table_put( + hash_table, + hash_of::(le64_at::<0, _>(around), hlog), + base_index.wrapping_add(ip0 - 2), + ); + bytes0 = le64_at::<2, _>(around); + + while window.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) { + let rep_place = window.place_of(ip0).wrapping_sub(rep_offset2); + let end = ip0 + 4 + window.count(ip0 + 4, rep_place + 4); + core::mem::swap(&mut rep_offset1, &mut rep_offset2); + table_put( + hash_table, + hash_of::(bytes0, hlog), + base_index.wrapping_add(ip0), + ); + sink.store(ip0, end, REPCODE1_TO_OFFBASE); + ip0 = end; + let Some(bytes) = near.read64(ip0) else { + break 'block; + }; + bytes0 = bytes; } + ip1 = ip0 + step_size; } - ip1 = ip0 + step_size; + self.rep_offset1 = rep_offset1; + self.rep_offset2 = rep_offset2; } - - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(seg.iend - anchor) } /// `ZSTD_compressBlock_fast_dictMatchState` @@ -1104,15 +1095,36 @@ pub(crate) fn compress_block_fast_dict_match_state( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { - let Some(seg) = &TwoSegments::with_dict_match_state(src, ms.window.dict_limit)? else { - return Ok(src.end - src.start); + let block = DictBlock::new(src, ms.window.dict_limit)?; + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_table = TaggedTable::new( + &dms.ms.hash_table, + dms.ms.tagged_hash_log(), + block.dict_start_index, + )?; + let hlog = ms.c_params.hash_log; + let [rep_offset1, rep_offset2, _] = *rep; + let mut finder = Finder { + base_index: block.prefix_start_index, + step_size: least_step(&ms.c_params), + hlog, + first: block.first(), + rep_offset1, + rep_offset2, }; + let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); + let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; match ms.c_params.min_match { - 5 => compress_block_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src, seg), - 6 => compress_block_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src, seg), - 7 => compress_block_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src, seg), - _ => compress_block_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src, seg), + 5 => finder.find_matches_dict::<5>(hash_table, &mut sink, block.window, dict_table), + 6 => finder.find_matches_dict::<6>(hash_table, &mut sink, block.window, dict_table), + 7 => finder.find_matches_dict::<7>(hash_table, &mut sink, block.window, dict_table), + _ => finder.find_matches_dict::<4>(hash_table, &mut sink, block.window, dict_table), } + let last_literals = sink.finish()?; + let [rep0, rep1, _] = rep; + *rep0 = finder.rep_offset1; + *rep1 = finder.rep_offset2; + Ok(last_literals) } /// `ZSTD_compressBlock_fast_extDict` From f3eb4d3b0145df1e2b7cee9ef21c73a17bae4be9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:14:19 +0000 Subject: [PATCH 037/199] zstd: dfast with a dictionary's match state reads the input once per position --- src/zstd/compress/double_fast.rs | 431 ++++++++++++------------------- src/zstd/compress/fast.rs | 267 +++---------------- src/zstd/compress/internal.rs | 8 - 3 files changed, 206 insertions(+), 500 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index e2631fdc7da9..a17de3efc4ad 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -1,18 +1,17 @@ //! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. use super::fast::{ - ExtDictBlock, FAST_HASH_FILL_STEP, MatchSink, NoDictBlock, Segments, TwoSegments, - count_backwards, restore_rep, + DictBlock, ExtDictBlock, FAST_HASH_FILL_STEP, MatchSink, NoDictBlock, Segments, TaggedTable, + restore_rep, }; use super::internal::{ BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, - compare_packed_tags, count, count_2segments, hash_of, hash_ptr, hash8, index_overlap_check, - le64_at, offset_to_offbase, table_get, table_put, table_swap, write_tagged_index, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, hash_of, hash_ptr, + hash8, le64_at, offset_to_offbase, table_get, table_put, table_swap, write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; -use crate::util::{BUG, SliceExt, read_le32, read_le64}; +use crate::util::{BUG, SliceExt}; /// `ZSTD_fillDoubleHashTableForCDict` fn fill_double_hash_table_for_cdict( @@ -274,18 +273,16 @@ pub(crate) fn compress_block_double_fast( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { - let Some(block) = NoDictBlock::new(ms, rep, src)? else { - return Ok(src.end - src.start); - }; + let block = NoDictBlock::new(ms, rep, src)?; let h_bits_l = ms.c_params.hash_log; let h_bits_s = ms.c_params.chain_log; let mut finder = Finder { base_index: block.prefix_lowest_index, h_bits_l, h_bits_s, - first: block.first - block.prefix_lowest, - offset_1: block.offset_1, - offset_2: block.offset_2, + first: block.first, + offset_1: block.rep.offset_1, + offset_2: block.rep.offset_2, }; let mut sink = MatchSink::new(seq_store, &block); let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; @@ -296,275 +293,151 @@ pub(crate) fn compress_block_double_fast( 7 => finder.find_matches::<7>(hash_long, hash_small, &mut sink), _ => finder.find_matches::<4>(hash_long, hash_small, &mut sink), } - let last_literals = sink.finish()?; + let last_literals = sink.last_literals()?; restore_rep( rep, finder.offset_1, finder.offset_2, - block.offset_saved1, - block.offset_saved2, + block.rep.offset_saved1, + block.rep.offset_saved2, ); Ok(last_literals) } -/// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` -fn compress_block_double_fast_dict_match_state_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - seg: &TwoSegments<'_>, -) -> Result { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let [mut offset_1, mut offset_2, _] = *rep; - let hash_long = ms.hash_table.as_mut_slice(); - let hash_small = ms.chain_table.as_mut_slice(); - let dms = src.dms.as_ref().ok_or(BUG)?; - let dict_hash_long = dms.ms.hash_table.as_slice(); - let dict_hash_small = dms.ms.chain_table.as_slice(); - let dict_h_bits_l = dms.ms.tagged_hash_log(); - let dict_h_bits_s = dms.ms.tagged_chain_log(); - - let mut anchor = seg.istart; - let mut ip = seg.first(); - - 'search: while ip < seg.ilimit { - let h2 = hash_ptr(seg.buf, ip, h_bits_l, 8)?; - let h = hash_ptr(seg.buf, ip, h_bits_s, MLS)?; - let dict_hash_and_tag_l = hash_ptr(seg.buf, ip, dict_h_bits_l, 8)?; - let dict_hash_and_tag_s = hash_ptr(seg.buf, ip, dict_h_bits_s, MLS)?; - let dict_match_index_and_tag_l = - dict_hash_long.at(dict_hash_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_match_index_and_tag_s = - dict_hash_small.at(dict_hash_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_tags_match_l = - compare_packed_tags(dict_match_index_and_tag_l as usize, dict_hash_and_tag_l); - let dict_tags_match_s = - compare_packed_tags(dict_match_index_and_tag_s as usize, dict_hash_and_tag_s); - let curr = src.index_of(ip); - let match_index_l = hash_long.at(h2)?; - let mut match_index_s = hash_small.at(h)?; - let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1); - let rep_match = seg.locate(rep_index); - hash_long.put(h2, curr)?; - hash_small.put(h, curr)?; - - let m_length = 'match_stored: { - if index_overlap_check(seg.prefix_start_index, rep_index) - && read_le32(rep_match.buf, rep_match.pos)? == read_le32(seg.buf, ip + 1)? - { - let m_length = count_2segments( - seg.buf, - ip + 1 + 4, - rep_match.buf, - rep_match.pos + 4, - seg.iend, - rep_match.end, - seg.prefix_start, - )? + 4; - ip += 1; - seq_store.store_seq(ip - anchor, seg.buf, anchor, REPCODE1_TO_OFFBASE, m_length)?; - break 'match_stored m_length; - } +impl Finder { + /// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` + #[inline(never)] + fn find_matches_dict( + &mut self, + hash_long: &mut [u32], + hash_small: &mut [u32], + sink: &mut MatchSink<'_, '_>, + window: Segments<'_>, + dict_tables: &DictTables<'_>, + ) { + let Self { + base_index, + h_bits_l, + h_bits_s, + first: mut ip, + mut offset_1, + mut offset_2, + } = *self; + let near = window.near; + if hash_long.is_empty() || hash_small.is_empty() { + return; + } - // The offset and the length of a match, which `ip` is moved to the start of. - let (offset, m_length) = 'match_found: { - let match_long = src.pos_of(match_index_l); - if match_index_l >= seg.prefix_start_index - && read_le64(seg.buf, match_long)? == read_le64(seg.buf, ip)? - { - let m_length = count(seg.buf, ip + 8, seg.buf, match_long + 8, seg.iend)? + 8; - let offset = ip.wrapping_sub(match_long) as u32; - let back = - count_backwards(seg.buf, ip, anchor, seg.buf, match_long, seg.prefix_start); - ip -= back; - break 'match_found (offset, m_length + back); - } else if dict_tags_match_l { - let dict_match_index_l = - dict_match_index_and_tag_l >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match_l = seg.dict_pos_of(dict_match_index_l); - if dict_match_index_l > seg.dict_start_index - && read_le64(seg.dict_buf, dict_match_l)? == read_le64(seg.buf, ip)? + // `ip` is before the last position that can be looked at. + 'search: while let Some(here) = near.window::<{ HASH_READ_SIZE + 1 }>(ip) { + let bytes0 = le64_at::<0, _>(here); + let curr = base_index.wrapping_add(ip); + let match_long = + table_swap(hash_long, hash8(bytes0, h_bits_l, 0), curr).wrapping_sub(base_index); + let match_index_s = table_swap(hash_small, hash_of::(bytes0, h_bits_s), curr); + + if window.read32_before(ip + 1, offset_1) == Some((bytes0 >> 8) as u32) { + ip += 1; + let rep_place = window.place_of(ip).wrapping_sub(offset_1); + let end = ip + 4 + window.count(ip + 4, rep_place + 4); + sink.store(ip, end, REPCODE1_TO_OFFBASE); + ip = end; + } else { + // The place of the match, and how much of it is known. `ip` is moved to where it is for. + let (place, known) = 'found: { + if near.diff64(match_long, bytes0) == 0 { + break 'found (window.place_of(match_long), 8); + } + if let Some(place) = dict_tables.long.get::<8>(bytes0) + && window.older.read64(place) == Some(bytes0) { - let m_length = count_2segments( - seg.buf, - ip + 8, - seg.dict_buf, - dict_match_l + 8, - seg.iend, - seg.dict_end, - seg.prefix_start, - )? + 8; - let offset = curr - .wrapping_sub(dict_match_index_l) - .wrapping_sub(seg.dict_index_delta); - let back = count_backwards( - seg.buf, - ip, - anchor, - seg.dict_buf, - dict_match_l, - seg.dict_start, - ); - ip -= back; - break 'match_found (offset, m_length + back); + break 'found (place, 8); } - } - let short_found = if match_index_s > seg.prefix_start_index { - read_le32(seg.buf, src.pos_of(match_index_s))? == read_le32(seg.buf, ip)? - } else if dict_tags_match_s { - let dict_match_index_s = - dict_match_index_and_tag_s >> ZSTD_SHORT_CACHE_TAG_BITS; - match_index_s = dict_match_index_s.wrapping_add(seg.dict_index_delta); - dict_match_index_s > seg.dict_start_index - && read_le32(seg.dict_buf, seg.dict_pos_of(dict_match_index_s))? - == read_le32(seg.buf, ip)? - } else { - false - }; - if !short_found { - ip += ((ip - anchor) >> K_SEARCH_STRENGTH) + 1; - continue 'search; - } + // The dictionary is only looked up if there is no candidate in the prefix. + let short = if match_index_s > base_index { + Some(window.place_of(match_index_s - base_index)) + } else { + dict_tables.short.get::(bytes0) + }; + let Some(place_short) = + short.filter(|&place| window.read32(place) == Some(bytes0 as u32)) + else { + ip += ((ip - sink.anchor) >> K_SEARCH_STRENGTH) + 1; + continue 'search; + }; - // A short match: a long one at the next position is preferred. - let hl3 = hash_ptr(seg.buf, ip + 1, h_bits_l, 8)?; - let dict_hash_and_tag_l3 = hash_ptr(seg.buf, ip + 1, dict_h_bits_l, 8)?; - let match_index_l3 = hash_long.at(hl3)?; - let dict_match_index_and_tag_l3 = - dict_hash_long.at(dict_hash_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS)?; - let dict_tags_match_l3 = - compare_packed_tags(dict_match_index_and_tag_l3 as usize, dict_hash_and_tag_l3); - let match_l3 = src.pos_of(match_index_l3); - hash_long.put(hl3, curr + 1)?; - - if match_index_l3 >= seg.prefix_start_index - && read_le64(seg.buf, match_l3)? == read_le64(seg.buf, ip + 1)? - { - let m_length = count(seg.buf, ip + 9, seg.buf, match_l3 + 8, seg.iend)? + 8; - ip += 1; - let offset = ip.wrapping_sub(match_l3) as u32; - let back = - count_backwards(seg.buf, ip, anchor, seg.buf, match_l3, seg.prefix_start); - ip -= back; - break 'match_found (offset, m_length + back); - } else if dict_tags_match_l3 { - let dict_match_index_l3 = - dict_match_index_and_tag_l3 >> ZSTD_SHORT_CACHE_TAG_BITS; - let dict_match_l3 = seg.dict_pos_of(dict_match_index_l3); - if dict_match_index_l3 > seg.dict_start_index - && read_le64(seg.dict_buf, dict_match_l3)? == read_le64(seg.buf, ip + 1)? + // A short match: a long one at the next position is preferred. + let bytes1 = le64_at::<1, _>(here); + let match_long1 = table_swap(hash_long, hash8(bytes1, h_bits_l, 0), curr + 1) + .wrapping_sub(base_index); + if near.read64(match_long1) == Some(bytes1) { + ip += 1; + break 'found (window.place_of(match_long1), 8); + } + if let Some(place) = dict_tables.long.get::<8>(bytes1) + && window.older.read64(place) == Some(bytes1) { - let m_length = count_2segments( - seg.buf, - ip + 1 + 8, - seg.dict_buf, - dict_match_l3 + 8, - seg.iend, - seg.dict_end, - seg.prefix_start, - )? + 8; ip += 1; - let offset = curr - .wrapping_add(1) - .wrapping_sub(dict_match_index_l3) - .wrapping_sub(seg.dict_index_delta); - let back = count_backwards( - seg.buf, - ip, - anchor, - seg.dict_buf, - dict_match_l3, - seg.dict_start, - ); - ip -= back; - break 'match_found (offset, m_length + back); + break 'found (place, 8); } - } + (place_short, 4) + }; + let end = ip + known + window.count(ip + known, place + known); + let back = window.count_backwards(ip, sink.anchor, place); + offset_2 = offset_1; + offset_1 = window.place_of(ip) - place; + sink.store(ip - back, end, offset_to_offbase(offset_1)); + ip = end; + } - let match_s = seg.locate(match_index_s); - let m_length = count_2segments( - seg.buf, - ip + 4, - match_s.buf, - match_s.pos + 4, - seg.iend, - match_s.end, - seg.prefix_start, - )? + 4; - let offset = curr.wrapping_sub(match_index_s); - let back = - count_backwards(seg.buf, ip, anchor, match_s.buf, match_s.pos, match_s.start); - ip -= back; - (offset, m_length + back) + // `ip` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip - 2) else { + break; }; - - offset_2 = offset_1; - offset_1 = offset; - seq_store.store_seq( - ip - anchor, - seg.buf, - anchor, - offset_to_offbase(offset), - m_length, - )?; - m_length - }; - - ip += m_length; - anchor = ip; - - if ip <= seg.ilimit { let index_to_insert = curr + 2; - let to_insert = src.pos_of(index_to_insert); - hash_long.put(hash_ptr(seg.buf, to_insert, h_bits_l, 8)?, index_to_insert)?; - hash_long.put( - hash_ptr(seg.buf, ip - 2, h_bits_l, 8)?, - src.index_of(ip - 2), - )?; - hash_small.put( - hash_ptr(seg.buf, to_insert, h_bits_s, MLS)?, - index_to_insert, - )?; - hash_small.put( - hash_ptr(seg.buf, ip - 1, h_bits_s, MLS)?, - src.index_of(ip - 1), - )?; - - while ip <= seg.ilimit { - let current2 = src.index_of(ip); - let rep_index2 = current2.wrapping_sub(offset_2); - let rep_match2 = seg.locate(rep_index2); - if !(index_overlap_check(seg.prefix_start_index, rep_index2) - && read_le32(rep_match2.buf, rep_match2.pos)? == read_le32(seg.buf, ip)?) - { - break; - } - let rep_length2 = count_2segments( - seg.buf, - ip + 4, - rep_match2.buf, - rep_match2.pos + 4, - seg.iend, - rep_match2.end, - seg.prefix_start, - )? + 4; + if let Some(to_insert) = near.read64(index_to_insert.wrapping_sub(base_index)) { + table_put(hash_long, hash8(to_insert, h_bits_l, 0), index_to_insert); + table_put( + hash_small, + hash_of::(to_insert, h_bits_s), + index_to_insert, + ); + } + table_put( + hash_long, + hash8(le64_at::<0, _>(around), h_bits_l, 0), + base_index.wrapping_add(ip - 2), + ); + table_put( + hash_small, + hash_of::(le64_at::<1, _>(around), h_bits_s), + base_index.wrapping_add(ip - 1), + ); + + while let Some(bytes) = near.read64(ip) + && window.read32_before(ip, offset_2) == Some(bytes as u32) + { + let rep_place = window.place_of(ip).wrapping_sub(offset_2); + let end = ip + 4 + window.count(ip + 4, rep_place + 4); core::mem::swap(&mut offset_1, &mut offset_2); - seq_store.store_seq(0, seg.buf, anchor, REPCODE1_TO_OFFBASE, rep_length2)?; - hash_small.put(hash_ptr(seg.buf, ip, h_bits_s, MLS)?, current2)?; - hash_long.put(hash_ptr(seg.buf, ip, h_bits_l, 8)?, current2)?; - ip += rep_length2; - anchor = ip; + let index = base_index.wrapping_add(ip); + table_put(hash_small, hash_of::(bytes, h_bits_s), index); + table_put(hash_long, hash8(bytes, h_bits_l, 0), index); + sink.store(ip, end, REPCODE1_TO_OFFBASE); + ip = end; } } + + self.offset_1 = offset_1; + self.offset_2 = offset_2; } +} - let [rep0, rep1, _] = rep; - *rep0 = offset_1; - *rep1 = offset_2; - Ok(seg.iend - anchor) +/// The tables of a dictionary. +struct DictTables<'a> { + long: TaggedTable<'a>, + short: TaggedTable<'a>, } /// `ZSTD_compressBlock_doubleFast_dictMatchState` @@ -576,15 +449,45 @@ pub(crate) fn compress_block_double_fast_dict_match_state( ) -> Result { let prefix_lowest_index = ms.get_lowest_prefix_index(src.index_of(src.end), ms.c_params.window_log); - let Some(seg) = &TwoSegments::with_dict_match_state(src, prefix_lowest_index)? else { - return Ok(src.end - src.start); + let block = DictBlock::new(src, prefix_lowest_index)?; + let dms = src.dms.as_ref().ok_or(BUG)?; + let tables = &DictTables { + long: TaggedTable::new( + &dms.ms.hash_table, + dms.ms.tagged_hash_log(), + block.dict_start_index, + )?, + short: TaggedTable::new( + &dms.ms.chain_table, + dms.ms.tagged_chain_log(), + block.dict_start_index, + )?, }; + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let [offset_1, offset_2, _] = *rep; + let mut finder = Finder { + base_index: block.prefix_start_index, + h_bits_l, + h_bits_s, + first: block.first(), + offset_1, + offset_2, + }; + let sink = &mut MatchSink::with_near(seq_store, block.window.near, block.istart); + let long = ms.hash_table.sub_mut(..1 << h_bits_l)?; + let small = ms.chain_table.sub_mut(..1 << h_bits_s)?; match ms.c_params.min_match { - 5 => compress_block_double_fast_dict_match_state_generic::<5>(ms, seq_store, rep, src, seg), - 6 => compress_block_double_fast_dict_match_state_generic::<6>(ms, seq_store, rep, src, seg), - 7 => compress_block_double_fast_dict_match_state_generic::<7>(ms, seq_store, rep, src, seg), - _ => compress_block_double_fast_dict_match_state_generic::<4>(ms, seq_store, rep, src, seg), + 5 => finder.find_matches_dict::<5>(long, small, sink, block.window, tables), + 6 => finder.find_matches_dict::<6>(long, small, sink, block.window, tables), + 7 => finder.find_matches_dict::<7>(long, small, sink, block.window, tables), + _ => finder.find_matches_dict::<4>(long, small, sink, block.window, tables), } + let last_literals = sink.last_literals()?; + let [rep0, rep1, _] = rep; + *rep0 = finder.offset_1; + *rep1 = finder.offset_2; + Ok(last_literals) } impl Finder { @@ -735,7 +638,7 @@ pub(crate) fn compress_block_double_fast_ext_dict( 7 => finder.find_matches_ext_dict::<7>(hash_long, hash_small, &mut sink, block.window), _ => finder.find_matches_ext_dict::<4>(hash_long, hash_small, &mut sink, block.window), } - let last_literals = sink.finish()?; + let last_literals = sink.last_literals()?; let [rep0, rep1, _] = rep; *rep0 = finder.offset_1; *rep1 = finder.offset_2; diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index e62a353f31cf..e57cc584d77d 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -3,161 +3,15 @@ use super::internal::{ BlockSrc, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, - ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, count, count_2segments, hash_of, hash_ptr, - index_overlap_check, le32_at, le64_at, offset_to_offbase, table_get, table_put, table_swap, - write_tagged_index, + ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, hash_of, hash_ptr, le32_at, le64_at, + offset_to_offbase, table_get, table_put, table_swap, write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; -use crate::util::{BUG, SliceExt, read_le32}; +use crate::util::{BUG, SliceExt}; pub(super) const FAST_HASH_FILL_STEP: u32 = 3; -/// Where an index leads, when it can be in one of two segments. -#[derive(Clone, Copy)] -pub(super) struct Located<'a> { - /// The buffer that the segment is in. - pub(super) buf: &'a [u8], - /// The position of the index in `buf`. - pub(super) pos: usize, - /// Where the segment starts in `buf`. - pub(super) start: usize, - /// Where it ends. - pub(super) end: usize, -} - -/// What `fast` and `dfast` work out before they look for matches in a block, when indices lead to two segments: the -/// current one, and below `prefix_start_index` either the older one of the window or the dictionary's. -pub(super) struct TwoSegments<'a> { - /// The buffer of the current segment, up to the end of the block. - pub(super) buf: &'a [u8], - base: usize, - pub(super) istart: usize, - pub(super) iend: usize, - /// The last position that [`HASH_READ_SIZE`] bytes can be read at. - pub(super) ilimit: usize, - pub(super) prefix_start_index: u32, - pub(super) prefix_start: usize, - pub(super) dict_buf: &'a [u8], - dict_base: usize, - /// What to subtract from an index below `prefix_start_index` to get an index of the other segment. - pub(super) dict_index_delta: u32, - pub(super) dict_start_index: u32, - pub(super) dict_start: usize, - pub(super) dict_end: usize, -} - -impl<'a> TwoSegments<'a> { - /// The other segment is the dictionary's, which has its own match state. `None` if the block is too short to look - /// for anything. - pub(super) fn with_dict_match_state( - src: &BlockSrc<'a>, - prefix_start_index: u32, - ) -> Result> { - let dms = src.dms.as_ref().ok_or(BUG)?; - let dict_start_index = dms.ms.window.dict_limit; - let Some(ilimit) = src.end.checked_sub(HASH_READ_SIZE) else { - return Ok(None); - }; - Ok(Some(Self { - buf: src.buf.sub(..src.end)?, - base: src.base, - istart: src.start, - iend: src.end, - ilimit, - prefix_start_index, - prefix_start: src.pos_of(prefix_start_index), - dict_buf: dms.buf, - dict_base: dms.base, - dict_index_delta: prefix_start_index - .wrapping_sub(dms.end.wrapping_sub(dms.base) as u32), - dict_start_index, - dict_start: dms.pos_of(dict_start_index), - dict_end: dms.end, - })) - } - - /// The other segment is the older one of the window, as far as it is within reach. `None` if the block is too short - /// to look for anything. - pub(super) fn with_ext_dict(ms: &MatchState, src: &BlockSrc<'a>) -> Result> { - let low_limit = ms.get_lowest_match_index(src.index_of(src.end), ms.c_params.window_log); - let prefix_start_index = ms.window.dict_limit.max(low_limit); - let Some(ilimit) = src.end.checked_sub(HASH_READ_SIZE) else { - return Ok(None); - }; - Ok(Some(Self { - buf: src.buf.sub(..src.end)?, - base: src.base, - istart: src.start, - iend: src.end, - ilimit, - prefix_start_index, - prefix_start: src.pos_of(prefix_start_index), - dict_buf: src.dict_buf, - dict_base: src.dict_base, - dict_index_delta: 0, - dict_start_index: low_limit, - dict_start: src.dict_pos_of(low_limit), - dict_end: src.dict_pos_of(prefix_start_index), - })) - } - - /// The first position to look at, with a dictionary: not `istart` if there is nothing before it to match. - pub(super) const fn first(&self) -> usize { - let dict_and_prefix_length = self - .istart - .wrapping_sub(self.prefix_start) - .wrapping_add(self.dict_end) - .wrapping_sub(self.dict_start) as u32; - self.istart + (dict_and_prefix_length == 0) as usize - } - - /// The position in `dict_buf` of an index of the other segment. - #[inline(always)] - pub(super) const fn dict_pos_of(&self, index: u32) -> usize { - self.dict_base.wrapping_add(index as usize) - } - - #[inline(always)] - pub(super) const fn locate(&self, index: u32) -> Located<'a> { - if index < self.prefix_start_index { - Located { - buf: self.dict_buf, - pos: self.dict_pos_of(index.wrapping_sub(self.dict_index_delta)), - start: self.dict_start, - end: self.dict_end, - } - } else { - Located { - buf: self.buf, - pos: self.base.wrapping_add(index as usize), - start: self.prefix_start, - end: self.iend, - } - } - } -} - -/// How many bytes before `ip`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. -#[inline(always)] -pub(super) fn count_backwards( - buf: &[u8], - ip: usize, - anchor: usize, - match_buf: &[u8], - match_pos: usize, - match_low: usize, -) -> usize { - let before_ip = buf.get(anchor..ip).unwrap_or_default(); - let before_match = match_buf.get(match_low..match_pos).unwrap_or_default(); - before_ip - .iter() - .rev() - .zip(before_match.iter().rev()) - .take_while(|(a, b)| a == b) - .count() -} - /// The repeat offsets at the start of a block. Those that reach further than `max_rep` are 0, and saved. pub(super) struct SavedRep { pub(super) offset_1: u32, @@ -185,12 +39,6 @@ impl SavedRep { offset_saved2, } } - - fn with_ext_dict(rep: &[u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, seg: &TwoSegments<'_>) -> Self { - let max_rep = src.index_of(seg.istart).wrapping_sub(seg.dict_start_index); - // As far as `max_rep` is too far too. - Self::new(rep, max_rep.saturating_sub(1)) - } } /// The repeat offsets to leave at the end of a block, at the start of which those that reached too far were saved and @@ -288,19 +136,6 @@ pub(crate) fn fill_hash_table( } } -/// `ZSTD_match4Found_branch`, and `ZSTD_match4Found_cmov`, which only differs in how it avoids reading at an index that -/// is not valid. -#[inline(always)] -fn match4_found( - buf: &[u8], - current: usize, - match_pos: usize, - match_idx: u32, - idx_low_limit: u32, -) -> Result { - Ok(match_idx >= idx_low_limit && read_le32(buf, current)? == read_le32(buf, match_pos)?) -} - /// `targetLength + !targetLength`: how far `fast` goes from one position to the next, before it speeds up. const fn least_step(c_params: &CompressionParameters) -> u32 { c_params.target_length + (c_params.target_length == 0) as u32 @@ -309,67 +144,45 @@ const fn least_step(c_params: &CompressionParameters) -> u32 { /// What `fast` and `dfast` work out before they look for matches in a block, when there is no dictionary and the window /// is one segment. pub(super) struct NoDictBlock<'a> { - /// The input from `base_index`, the lowest index that is read, up to the end of the block. Positions are in this. - pub(super) near: Near<'a>, - pub(super) base_index: u32, - /// The part of it where matches can be, which starts at `prefix_lowest_index`. A lower index, less that one, wraps - /// around to a position past the end. - pub(super) candidates: Near<'a>, + /// The input from `prefix_lowest_index`, the lowest index a match can be at, up to the end of the block. Positions + /// are in this: a lower index, less that one, wraps around to a position past the end. + near: Near<'a>, pub(super) prefix_lowest_index: u32, - pub(super) prefix_lowest: u32, - pub(super) istart: u32, + /// The same with what is before it, as far as a repeat offset reaches: where the window started at the start of the + /// block. + older: Near<'a>, + /// How much that is. + older_len: u32, + istart: u32, /// The first position to look at. pub(super) first: u32, - /// The last position that [`HASH_READ_SIZE`] bytes can be read at. - pub(super) ilimit: usize, - pub(super) offset_1: u32, - pub(super) offset_2: u32, - /// The repeat offsets that reach too far, which are 0 above. - pub(super) offset_saved1: u32, - pub(super) offset_saved2: u32, + pub(super) rep: SavedRep, } impl<'a> NoDictBlock<'a> { - /// `None` if the block is too short to look for anything. pub(super) fn new( ms: &MatchState, rep: &[u32; ZSTD_REP_NUM], src: &BlockSrc<'a>, - ) -> Result> { + ) -> Result { let window_log = ms.c_params.window_log; let start_index = src.index_of(src.start); let prefix_lowest_index = ms.get_lowest_prefix_index(src.index_of(src.end), window_log); let first_index = start_index + u32::from(start_index == prefix_lowest_index); let window_low = ms.get_lowest_prefix_index(first_index, window_log); - // A repeat offset reaches as far as the window did at the start of the block. - let base_index = window_low.min(prefix_lowest_index); - let near = Near::new(src.buf.sub(src.pos_of(base_index)..src.end)?)?; - let prefix_lowest = prefix_lowest_index - base_index; - let Some(ilimit) = near.bytes().len().checked_sub(HASH_READ_SIZE) else { - return Ok(None); - }; - - let SavedRep { - offset_1, - offset_2, - offset_saved1, - offset_saved2, - } = SavedRep::new(rep, first_index.wrapping_sub(window_low)); - Ok(Some(Self { - near, - base_index, - candidates: Near::new(near.bytes().sub(prefix_lowest as usize..)?)?, + let older_index = window_low.min(prefix_lowest_index); + let older = src.buf.sub(src.pos_of(older_index)..src.end)?; + let older_len = prefix_lowest_index - older_index; + Ok(Self { + near: Near::new(older.sub(older_len as usize..)?)?, prefix_lowest_index, - prefix_lowest, - istart: start_index - base_index, - first: first_index - base_index, - ilimit, - offset_1, - offset_2, - offset_saved1, - offset_saved2, - })) + older: Near::new(older)?, + older_len, + istart: start_index - prefix_lowest_index, + first: first_index - prefix_lowest_index, + rep: SavedRep::new(rep, first_index.wrapping_sub(window_low)), + }) } } @@ -432,10 +245,10 @@ impl<'a, 'b> MatchSink<'a, 'b> { pub(super) fn new(seq_store: &'b mut SeqStore, block: &NoDictBlock<'a>) -> Self { Self { seq_store: SeqSink::new(seq_store), - near: block.candidates, - older: block.near, - older_len: block.prefix_lowest, - anchor: block.istart - block.prefix_lowest, + near: block.near, + older: block.older, + older_len: block.older_len, + anchor: block.istart, failed: false, } } @@ -453,7 +266,7 @@ impl<'a, 'b> MatchSink<'a, 'b> { } /// How many bytes there are after the last match. - pub(super) fn finish(self) -> Result { + pub(super) fn last_literals(&self) -> Result { if self.failed { return Err(BUG); } @@ -873,18 +686,16 @@ pub(crate) fn compress_block_fast( rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, ) -> Result { - let Some(block) = NoDictBlock::new(ms, rep, src)? else { - return Ok(src.end - src.start); - }; + let block = NoDictBlock::new(ms, rep, src)?; let hlog = ms.c_params.hash_log; let mut finder = Finder { base_index: block.prefix_lowest_index, // The least step is 2, as two positions are looked at per turn. step_size: least_step(&ms.c_params) + 1, hlog, - first: block.first - block.prefix_lowest, - rep_offset1: block.offset_1, - rep_offset2: block.offset_2, + first: block.first, + rep_offset1: block.rep.offset_1, + rep_offset2: block.rep.offset_2, }; let mut sink = MatchSink::new(seq_store, &block); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; @@ -894,13 +705,13 @@ pub(crate) fn compress_block_fast( 7 => finder.find_matches::<7>(hash_table, &mut sink), _ => finder.find_matches::<4>(hash_table, &mut sink), } - let last_literals = sink.finish()?; + let last_literals = sink.last_literals()?; restore_rep( rep, finder.rep_offset1, finder.rep_offset2, - block.offset_saved1, - block.offset_saved2, + block.rep.offset_saved1, + block.rep.offset_saved2, ); Ok(last_literals) } @@ -1120,7 +931,7 @@ pub(crate) fn compress_block_fast_dict_match_state( 7 => finder.find_matches_dict::<7>(hash_table, &mut sink, block.window, dict_table), _ => finder.find_matches_dict::<4>(hash_table, &mut sink, block.window, dict_table), } - let last_literals = sink.finish()?; + let last_literals = sink.last_literals()?; let [rep0, rep1, _] = rep; *rep0 = finder.rep_offset1; *rep1 = finder.rep_offset2; @@ -1155,7 +966,7 @@ pub(crate) fn compress_block_fast_ext_dict( 7 => finder.find_matches_ext_dict::<7>(hash_table, &mut sink, block.window), _ => finder.find_matches_ext_dict::<4>(hash_table, &mut sink, block.window), } - let last_literals = sink.finish()?; + let last_literals = sink.last_literals()?; restore_rep( rep, finder.rep_offset1, diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index da616eaed3cc..a0930ff340e3 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -7,8 +7,6 @@ use crate::common::{ ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; use crate::error::{ErrorCode, Result}; -use core::hint::{Locality, prefetch_read}; - use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64, write_le24}; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; @@ -711,12 +709,6 @@ impl<'a> Near<'a> { self.0.get(pos..pos + N)?.first_chunk() } - /// A hint that `pos`, which may be anything, is about to be read. - #[inline(always)] - pub(crate) fn prefetch(self, pos: u32) { - prefetch_read(self.0.as_ptr().wrapping_add(pos as usize), Locality::L1); - } - #[inline(always)] pub(crate) fn read8(self, pos: u32) -> Option { self.0.get(pos as usize).copied() From 91049431b7dfe3dde1ebe3dad28eae3d44adb085 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:18:00 +0000 Subject: [PATCH 038/199] zstd: workers share one queue of jobs A job was sent to one worker, chosen in advance, so it waited for that thread to be scheduled even if all the others were idle: seconds, on a machine with more to run than it has cores. The C library has one queue too. --- src/zstd/compress/mt.rs | 45 ++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 4c73cc40c97b..864572177a6f 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -313,11 +313,6 @@ fn worker_main(jobs: &Receiver, cancelled: &AtomicBool) { } } -struct Worker { - jobs: Sender, - thread: JoinHandle<()>, -} - /// `ZSTDMT_jobDescription`, as far as flushing is concerned. struct PendingJob { /// `None` for a job that needs no worker. @@ -385,7 +380,11 @@ impl PendingJob { /// `ZSTDMT_CCtx` pub(crate) struct MtCtx { - workers: Vec, + workers: Vec>, + /// Whichever worker is free takes the next job, so that a job does not wait for one thread to be scheduled. + queue: Sender, + /// The other end, for workers to come. + queue_end: Receiver, cancelled: Arc, /// The jobs from `doneJobID` to `nextJobID`. jobs: VecDeque, @@ -450,8 +449,11 @@ fn compute_overlap_size(params: &CCtxParams) -> usize { impl MtCtx { /// `ZSTDMT_createCCtx_advanced` pub(crate) fn new(nb_workers: u32) -> Result { + let (queue, queue_end) = channel(); let mut mtctx = Self { workers: Vec::new(), + queue, + queue_end, cancelled: Arc::new(AtomicBool::new(false)), jobs: VecDeque::new(), params: CCtxParams::new(0), @@ -475,24 +477,31 @@ impl MtCtx { /// `ZSTDMT_resize`: no job is under way. fn resize(&mut self, nb_workers: usize) -> Result<()> { - for worker in self.workers.drain(nb_workers.min(self.workers.len())..) { - drop(worker.jobs); - let _ = worker.thread.join(); + if nb_workers < self.workers.len() { + self.stop_workers(); } while self.workers.len() < nb_workers { - let (jobs, receiver) = channel(); + let queue_end = self.queue_end.clone(); let cancelled = Arc::clone(&self.cancelled); - let thread = std::thread::Builder::new() + let worker = std::thread::Builder::new() .name("zstd".into()) .stack_size(2 << 20) - .spawn(move || worker_main(&receiver, &cancelled)) + .spawn(move || worker_main(&queue_end, &cancelled)) .map_err(|_| ErrorCode::MemoryAllocation)?; - self.workers.push(Worker { jobs, thread }); + self.workers.push(worker); } self.params.nb_workers = nb_workers as i32; Ok(()) } + fn stop_workers(&mut self) { + // They leave when nothing can be sent to the queue they have. + (self.queue, self.queue_end) = channel(); + for worker in self.workers.drain(..) { + let _ = worker.join(); + } + } + /// `ZSTDMT_waitForAllJobsCompleted` and `ZSTDMT_releaseAllJobResources`, for a frame that is given up. fn abandon_jobs(&mut self) { self.cancelled.store(true, Ordering::SeqCst); @@ -658,10 +667,7 @@ impl MtCtx { serial_out, progress, }; - let worker = (self.workers) - .get(self.next_job_id as usize % self.workers.len().max(1)) - .ok_or(ErrorCode::Generic)?; - worker.jobs.send(job).map_err(|_| ErrorCode::Generic)?; + self.queue.send(job).map_err(|_| ErrorCode::Generic)?; self.jobs.push_back(pending); self.next_job_id += 1; Ok(()) @@ -816,9 +822,6 @@ impl MtCtx { impl Drop for MtCtx { fn drop(&mut self) { self.cancelled.store(true, Ordering::SeqCst); - for worker in self.workers.drain(..) { - drop(worker.jobs); - let _ = worker.thread.join(); - } + self.stop_workers(); } } From a5239f0ef009edcf6c0bd59f5263b2d3175a5a0c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:18:42 +0000 Subject: [PATCH 039/199] zstd: a context keeps no books of the workspace that the C code would have --- src/zstd/compress/cctx.rs | 98 ++++++++++++++++------------ src/zstd/compress/params.rs | 126 ++---------------------------------- 2 files changed, 61 insertions(+), 163 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 4a854de0483b..d0d2d702a346 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -11,16 +11,16 @@ use super::fse::FseRepeat; use super::huf::{self, HufRepeat}; use super::internal::{ BlockSrc, BufId, Buffers, CompressedBlockState, CompressionParameters, DictMatchState, - DictTableLoadMethod, EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, - MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqChunk, SeqStore, Strategy, + DictTableLoadMethod, EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, Match, + MatchState, Optimal, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqChunk, SeqStore, Strategy, TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, min_gain, no_compress_block, offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, }; use super::params::{ - CCtxParams, CParamMode, Format, MatchStateSizes, Workspace, WorkspaceNeeds, - adjust_c_params_internal, allocate_chain_table, cdict_indices_are_tagged, compress_bound, - cycle_log, get_c_params_from_cctx_params, max_nb_seq, resolve_row_match_finder_mode, + CCtxParams, CParamMode, Format, MatchStateSizes, adjust_c_params_internal, + allocate_chain_table, cdict_indices_are_tagged, compress_bound, cycle_log, + get_c_params_from_cctx_params, max_nb_seq, resolve_row_match_finder_mode, row_match_finder_used, }; use super::superblock::compress_super_block; @@ -31,7 +31,7 @@ use crate::common::{ BlockType, LITBITS, LL_FSE_LOG, MAX_LL, MAX_ML, MAX_OFF, MIN_CBLOCK_SIZE, ML_FSE_LOG, OFF_FSE_LOG, ZSTD_BLOCKHEADERSIZE, ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_FRAMEHEADERSIZE_MAX, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGICNUMBER, ZSTD_REP_NUM, - ZSTD_WINDOWLOG_ABSOLUTEMIN, + ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, }; use crate::error::{ErrorCode, Result}; use crate::util::{ @@ -145,13 +145,16 @@ pub(crate) fn reset_table( len: usize, clean: bool, ) -> Result<()> { - if table.len() != len { + if len > table.capacity() { // First, so that both are not allocated at once. *table = Vec::new(); *table = try_zeroed(len)?; - } else if clean { - table.fill(T::default()); + return Ok(()); + } + if clean { + table.clear(); } + table.resize(len, T::default()); Ok(()) } @@ -192,8 +195,6 @@ fn reset_match_state( } let row_log = c_params.search_log.clamp(4, 6); ms.row_hash_log = c_params.hash_log - row_log; - } else { - ms.tag_table = Vec::new(); } if for_who == ResetTarget::CCtx && c_params.strat() >= Strategy::BtOpt { @@ -203,15 +204,34 @@ fn reset_match_state( reset_table(&mut ms.opt.off_code_freq, MAX_OFF as usize + 1, false)?; reset_table(&mut ms.opt.match_table, ZSTD_OPT_SIZE, false)?; reset_table(&mut ms.opt.price_table, ZSTD_OPT_SIZE, false)?; - } else { - ms.opt.match_table = Vec::new(); - ms.opt.price_table = Vec::new(); } ms.c_params = *c_params; Ok(()) } +/// How many bytes the tables of a context take. +fn tables_size(c_params: &CompressionParameters, use_row_match_finder: ParamSwitch) -> usize { + let sizes = MatchStateSizes::new(c_params, use_row_match_finder, true); + let mut size = (sizes.h_size + sizes.chain_size + sizes.h3_size) * size_of::(); + if row_match_finder_used(c_params.strat(), use_row_match_finder) { + size += sizes.h_size; + } + if c_params.strat() >= Strategy::BtOpt { + size += ZSTD_OPT_SIZE * (size_of::() + size_of::()); + } + size +} + +/// How many bytes the tables of `ms` hold on to, in use or not. +fn tables_capacity(ms: &MatchState) -> usize { + (ms.hash_table.capacity() + ms.chain_table.capacity() + ms.hash_table3.capacity()) + * size_of::() + + ms.tag_table.capacity() + + ms.opt.match_table.capacity() * size_of::() + + ms.opt.price_table.capacity() * size_of::() +} + /// `ZSTD_indexTooCloseToMax`: starting a frame this close to the limit would mean bringing indices down during it. fn index_too_close_to_max(w: &Window) -> bool { const ZSTD_INDEXOVERFLOW_MARGIN: usize = 16 << 20; @@ -766,9 +786,8 @@ pub(crate) struct FrameCompressor { pub(crate) applied_params: CCtxParams, pub(crate) dict_id: u32, pub(crate) dict_content_size: usize, - workspace: Workspace, - /// See [`Self::for_worker`]. - room_for_any_block: bool, + /// For how many frames in a row the tables have been much larger than it takes. + oversized_duration: usize, pub(crate) block_size_max: usize, pub(crate) pledged_src_size_plus_one: u64, pub(crate) consumed_src_size: u64, @@ -814,8 +833,7 @@ impl FrameCompressor { applied_params: CCtxParams::new(0), dict_id: 0, dict_content_size: 0, - workspace: Workspace::new(), - room_for_any_block: false, + oversized_duration: 0, block_size_max: 0, pledged_src_size_plus_one: 0, consumed_src_size: 0, @@ -833,15 +851,8 @@ impl FrameCompressor { } } - /// For the jobs of one worker. The size of a job is all it is told about the size of the input, and a workspace that - /// is too small for a job a little larger than those before would be replaced, which clears the tables. The C code - /// hardly gets there, as small jobs all go to the context that had the first job of the frame, which did not know - /// the size. Which context a job goes to is a matter of timing there, so it cannot show in the output. pub(crate) fn for_worker() -> Self { - Self { - room_for_any_block: true, - ..Self::new() - } + Self::new() } /// `ZSTD_resetCCtx_internal` @@ -881,24 +892,29 @@ impl FrameCompressor { let index_too_close = index_too_close_to_max(&self.match_state.window); let dict_too_big = loaded_dict_size > ZSTD_CHUNKSIZE_MAX as usize; - let mut needs_index_reset = if index_too_close || dict_too_big || !self.initialized { + // Whether indices go on or start again does not show: what the tables hold from before is too low to be valid. + let needs_index_reset = if index_too_close || dict_too_big || !self.initialized { IndexResetPolicy::Reset } else { IndexResetPolicy::Continue }; - if self.workspace.reset(&WorkspaceNeeds { - c_params: ¶ms.c_params, - ldm_params: ¶ms.ldm_params, - use_row_match_finder: params.use_row_match_finder, - buff_in_size: sizes.buff_in_size, - buff_out_size: sizes.buff_out_size, - block_size: if self.room_for_any_block { - ZSTD_BLOCKSIZE_MAX.min(1 << params.c_params.window_log) - } else { - block_size - }, - }) { - needs_index_reset = IndexResetPolicy::Reset; + + // `ZSTD_cwksp_check_wasteful`: a context does not hold on for long to much more than it uses. + let needed = tables_size(¶ms.c_params, params.use_row_match_finder); + if tables_capacity(&self.match_state) < needed.saturating_mul(ZSTD_WORKSPACETOOLARGE_FACTOR) + { + self.oversized_duration = 0; + } else if self.oversized_duration < ZSTD_WORKSPACETOOLARGE_MAXDURATION { + self.oversized_duration += 1; + } else { + self.oversized_duration = 0; + let ms = &mut self.match_state; + ms.hash_table = Vec::new(); + ms.chain_table = Vec::new(); + ms.hash_table3 = Vec::new(); + ms.tag_table = Vec::new(); + ms.opt.match_table = Vec::new(); + ms.opt.price_table = Vec::new(); } self.match_state.c_params = params.c_params; diff --git a/src/zstd/compress/params.rs b/src/zstd/compress/params.rs index d062b759acb6..24f8420ec93a 100644 --- a/src/zstd/compress/params.rs +++ b/src/zstd/compress/params.rs @@ -3,15 +3,13 @@ use core::ops::RangeInclusive; use super::internal::{ - CompressionParameters, LdmParams, ParamSwitch, Strategy, ZSTD_OPT_SIZE, - ZSTD_SHORT_CACHE_TAG_BITS, + CompressionParameters, LdmParams, ParamSwitch, Strategy, ZSTD_SHORT_CACHE_TAG_BITS, }; use super::lazy::ZSTD_ROW_HASH_TAG_BITS; -use super::{ldm, row_match_finder_supported}; +use super::row_match_finder_supported; use crate::common::{ - LITBITS, MAX_LL, MAX_ML, MAX_OFF, WILDCOPY_OVERLENGTH, ZSTD_BLOCKSIZE_MAX, - ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WINDOWLOG_LIMIT_DEFAULT, - ZSTD_WINDOWLOG_MAX, ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, + ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_WINDOWLOG_ABSOLUTEMIN, + ZSTD_WINDOWLOG_LIMIT_DEFAULT, ZSTD_WINDOWLOG_MAX, }; use crate::error::{ErrorCode, Result}; use crate::util::highbit32; @@ -622,25 +620,6 @@ pub const fn compress_bound(src_size: usize) -> usize { src_size + (src_size >> 8) + margin } -// ── the size of the workspace ──────────────────────────────────────────────── -// -// The C code carves all the tables and buffers of a context out of one allocation. A context only goes on counting -// indexes from where the previous frame left them as long as that allocation is kept, so this keeps its books. - -const SIZEOF_COMPRESSED_BLOCK_STATE: usize = 5632; -const TMP_WORKSPACE_SIZE: usize = 8920; -const SIZEOF_SEQ_DEF: usize = 8; -const SIZEOF_MATCH: usize = 8; -const SIZEOF_OPTIMAL: usize = 28; -const SIZEOF_RAW_SEQ: usize = 12; -const SIZEOF_LDM_ENTRY: usize = 8; -const ZSTD_CWKSP_ALIGNMENT_BYTES: usize = 64; - -/// `ZSTD_cwksp_aligned64_alloc_size` -const fn aligned64(size: usize) -> usize { - size.next_multiple_of(ZSTD_CWKSP_ALIGNMENT_BYTES) -} - /// `ZSTD_maxNbSeq` pub(crate) const fn max_nb_seq(block_size: usize, min_match: u32) -> usize { block_size / if min_match == 3 { 3 } else { 4 } @@ -678,103 +657,6 @@ impl MatchStateSizes { } } -/// The books of a `ZSTD_cwksp`, as offsets from its start, which is taken to be aligned. -pub(crate) struct Workspace { - size: usize, - object_end: usize, - table_end: usize, - alloc_start: usize, - oversized_duration: usize, -} - -/// What a context needs room for. -pub(crate) struct WorkspaceNeeds<'a> { - pub(crate) c_params: &'a CompressionParameters, - pub(crate) ldm_params: &'a LdmParams, - pub(crate) use_row_match_finder: ParamSwitch, - pub(crate) buff_in_size: usize, - pub(crate) buff_out_size: usize, - pub(crate) block_size: usize, -} - -impl WorkspaceNeeds<'_> { - /// The tables, which the C code allocates from the bottom. - fn tables(&self) -> usize { - let sizes = MatchStateSizes::new(self.c_params, self.use_row_match_finder, true); - (sizes.chain_size + sizes.h_size + sizes.h3_size) * size_of::() - } - - /// Everything else, which the C code allocates from the top. - fn allocs(&self) -> usize { - let max_nb_seq = max_nb_seq(self.block_size, self.c_params.min_match); - let mut total = 0; - if row_match_finder_used(self.c_params.strat(), self.use_row_match_finder) { - total += aligned64(1 << self.c_params.hash_log); - } - if self.c_params.strat() >= Strategy::BtOpt { - total += aligned64((MAX_ML as usize + 1) * size_of::()) - + aligned64((MAX_LL as usize + 1) * size_of::()) - + aligned64((MAX_OFF as usize + 1) * size_of::()) - + aligned64((1 << LITBITS) * size_of::()) - + aligned64(ZSTD_OPT_SIZE * SIZEOF_MATCH) - + aligned64(ZSTD_OPT_SIZE * SIZEOF_OPTIMAL); - } - total += aligned64(max_nb_seq * SIZEOF_SEQ_DEF); - if self.ldm_params.enable_ldm == ParamSwitch::Enable { - total += aligned64((1usize << self.ldm_params.hash_log) * SIZEOF_LDM_ENTRY) - + aligned64( - ldm::get_max_nb_seq(*self.ldm_params, self.block_size) * SIZEOF_RAW_SEQ, - ) - + (1usize << (self.ldm_params.hash_log - self.ldm_params.bucket_size_log)); - } - total - + (self.block_size + WILDCOPY_OVERLENGTH) - + self.buff_in_size - + self.buff_out_size - + 3 * max_nb_seq - } - - /// `ZSTD_estimateCCtxSize_usingCCtxParams_internal` - fn needed_space(&self) -> usize { - let objects = TMP_WORKSPACE_SIZE + 2 * SIZEOF_COMPRESSED_BLOCK_STATE; - let slack_space = ZSTD_CWKSP_ALIGNMENT_BYTES * 2; - objects + self.tables() + self.allocs() + slack_space - } -} - -impl Workspace { - pub(crate) const fn new() -> Self { - Self { - size: 0, - object_end: 0, - table_end: 0, - alloc_start: 0, - oversized_duration: 0, - } - } - - /// What `ZSTD_resetCCtx_internal` does to the workspace. Returns whether it made a new one. - pub(crate) fn reset(&mut self, needs: &WorkspaceNeeds<'_>) -> bool { - let needed_space = needs.needed_space(); - self.oversized_duration += 1; - let too_small = self.size < needed_space; - let wasteful = self.alloc_start - self.table_end - >= needed_space.saturating_mul(ZSTD_WORKSPACETOOLARGE_FACTOR) - && self.oversized_duration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; - let resize = too_small || wasteful; - if resize { - self.size = needed_space; - self.object_end = aligned64(TMP_WORKSPACE_SIZE + 2 * SIZEOF_COMPRESSED_BLOCK_STATE); - self.oversized_duration = 0; - } - self.table_end = self.object_end + needs.tables(); - self.alloc_start = - (self.size - self.size % ZSTD_CWKSP_ALIGNMENT_BYTES).saturating_sub(needs.allocs()); - self.alloc_start = self.alloc_start.max(self.table_end); - resize - } -} - const fn cp( window_log: u32, chain_log: u32, From df51c0fc94073bd7295b4ea3079248c3d69da7c7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:18:42 +0000 Subject: [PATCH 040/199] zstd: a Huffman table is made from the symbols that occur, and its tree without branches --- src/zstd/compress/huf.rs | 244 +++++++++++++++++++++------------------ 1 file changed, 131 insertions(+), 113 deletions(-) diff --git a/src/zstd/compress/huf.rs b/src/zstd/compress/huf.rs index 25db9d506b29..73d985e53945 100644 --- a/src/zstd/compress/huf.rs +++ b/src/zstd/compress/huf.rs @@ -109,25 +109,27 @@ fn set_value(elt: &mut u64, value: u64) { } } -/// Gives each symbol its code. `ct` has the number of bits of each, at most `table_log`, and `nb_per_rank` has how -/// many symbols there are of each number of bits. -fn set_values( - ct: &mut [u64], +/// The first code of each number of bits up to `table_log`, given how many symbols there are of each: the longest codes +/// have the lowest values. +fn first_values( nb_per_rank: &[u16; RANKS], table_log: u32, -) -> Result<()> { +) -> Result<[u16; RANKS]> { let mut val_per_rank = [0u16; RANKS]; - // The first code of each rank: the longest codes have the lowest values. let mut min = 0u16; for n in (1..=table_log as usize).rev() { val_per_rank.put(n, min)?; min = min.wrapping_add(nb_per_rank.at(n)?) >> 1; } - for elt in ct { - let value = val_per_rank.at_mut(get_nb_bits(*elt) as usize)?; - set_value(elt, u64::from(*value)); - *value = value.wrapping_add(1); - } + Ok(val_per_rank) +} + +/// Gives `elt`, which has its number of bits, the next code of that many bits. +#[inline(always)] +fn set_next_value(elt: &mut u64, val_per_rank: &mut [u16]) -> Result<()> { + let value = val_per_rank.at_mut(get_nb_bits(*elt) as usize)?; + set_value(elt, u64::from(*value)); + *value = value.wrapping_add(1); Ok(()) } @@ -268,7 +270,10 @@ pub(crate) fn read_ctable( *elt = u64::from(nb_bits); *nb_per_rank.at_mut(usize::from(nb_bits))? += 1; } - set_values(ct, &nb_per_rank, table_log)?; + let mut val_per_rank = first_values(&nb_per_rank, table_log)?; + for elt in ct { + set_next_value(elt, &mut val_per_rank)?; + } Ok(HufCTableStats { size: stats.size, @@ -286,27 +291,19 @@ struct NodeElt { nb_bits: u8, } -/// The first node that is not a leaf. -const STARTNODE: i32 = HUF_SYMBOLVALUE_MAX as i32 + 1; - -/// `huffNodeTable`: the leaves from 0 on, sorted from the most frequent symbol to the rarest, the other nodes from -/// [`STARTNODE`] on, and a sentinel at -1. A number that is out of the table wraps around, which spares the checks. -struct HuffNodes([NodeElt; 2 * (HUF_SYMBOLVALUE_MAX as usize + 1)]); +/// The leaves of `huffNodeTable`, sorted from the most frequent symbol to the rarest. What there is at -1, or anywhere +/// else out of the table, is all zeros. +struct HuffNodes([NodeElt; HUF_SYMBOLVALUE_MAX as usize + 1]); impl HuffNodes { - #[inline(always)] - const fn index(n: i32) -> usize { - n.wrapping_add(1) as usize & (2 * (HUF_SYMBOLVALUE_MAX as usize + 1) - 1) - } - #[inline(always)] fn get(&self, n: i32) -> NodeElt { - self.0.get(Self::index(n)).copied().unwrap_or_default() + self.0.get(n as usize).copied().unwrap_or_default() } #[inline(always)] fn set(&mut self, n: i32, node: NodeElt) { - if let Some(slot) = self.0.get_mut(Self::index(n)) { + if let Some(slot) = self.0.get_mut(n as usize) { *slot = node; } } @@ -321,33 +318,12 @@ impl HuffNodes { u32::from(self.get(n).nb_bits) } - #[inline(always)] - fn set_count(&mut self, n: i32, count: u32) { - if let Some(node) = self.0.get_mut(Self::index(n)) { - node.count = count; - } - } - - #[inline(always)] - fn set_parent(&mut self, n: i32, parent: i32) { - if let Some(node) = self.0.get_mut(Self::index(n)) { - node.parent = parent as u16; - } - } - #[inline(always)] fn set_nb_bits(&mut self, n: i32, nb_bits: u32) { - if let Some(node) = self.0.get_mut(Self::index(n)) { + if let Some(node) = self.0.get_mut(n as usize) { node.nb_bits = nb_bits as u8; } } - - /// The nodes from 0 on. - #[inline(always)] - fn leaves(&self) -> &[NodeElt] { - let [_, nodes @ ..] = &self.0; - nodes - } } /// `HUF_setMaxHeight`: makes no code longer than `target_nb_bits`, which is not 0 and is enough for a code for every @@ -544,26 +520,49 @@ fn simple_quick_sort(arr: &mut HuffNodes, mut low: i32, mut high: i32) { } } -/// `HUF_sort`: makes the first nodes, which are all zeros, the symbols by decreasing count. At most 256 counts. -fn sort(huff_node: &mut HuffNodes, count: &[u32]) -> Result<()> { +/// The symbols whose count is not 0, from the lowest to the highest. +struct PresentSymbols { + symbols: [u8; HUF_SYMBOLVALUE_MAX as usize + 1], + len: usize, +} + +impl PresentSymbols { + fn get(&self) -> &[u8] { + self.symbols.get(..self.len).unwrap_or_default() + } +} + +/// `HUF_sort`: makes the first nodes, which are all zeros, the symbols by decreasing count, but for those whose count +/// is 0, which would come last. At most 256 counts. +fn sort(huff_node: &mut HuffNodes, count: &[u32]) -> Result { let mut rank_position = [RankPos::default(); RANK_POSITION_TABLE_SIZE]; + let mut present = PresentSymbols { + symbols: [0; _], + len: 0, + }; // A bucket starts after the symbols of all the buckets above it. - for &c in count { - rank_position.at_mut(get_index(c))?.base += 1; + let mut highest = 0; + for (symbol, &c) in count.iter().enumerate() { + present.symbols.put(present.len, symbol as u8)?; + present.len += usize::from(c != 0); + let index = get_index(c); + rank_position.at_mut(index)?.base += 1; + highest = highest.max(index); } let mut above = 0u16; - for rank in rank_position.iter_mut().rev() { + for rank in rank_position.sub_mut(..=highest)?.iter_mut().rev() { rank.base += above; rank.curr = rank.base; above = rank.base; } - for (n, &c) in count.iter().enumerate() { + for &symbol in present.get() { + let c = count.at(usize::from(symbol))?; let rank = rank_position.at_mut(get_index(c) + 1)?; let node = NodeElt { count: c, - byte: n as u8, + byte: symbol, ..NodeElt::default() }; huff_node.set(i32::from(rank.curr), node); @@ -571,84 +570,102 @@ fn sort(huff_node: &mut HuffNodes, count: &[u32]) -> Result<()> { } let shared = RANK_POSITION_DISTINCT_COUNT_CUTOFF as usize..RANK_POSITION_TABLE_SIZE - 1; - for rank in rank_position.sub(shared)? { + for rank in rank_position + .sub(shared)? + .iter() + .take_while(|rank| rank.curr != 0) + { if rank.curr > rank.base + 1 { simple_quick_sort(huff_node, i32::from(rank.base), i32::from(rank.curr) - 1); } } - Ok(()) + Ok(present) } -/// `HUF_buildTree`: `huff_node` is sorted, and some count is not 0. Sets the depth of every leaf, and returns the -/// rarest symbol whose count is not 0. -fn build_tree(huff_node: &mut HuffNodes, max_symbol_value: u32) -> Result { - let mut symbols = (0..=max_symbol_value as i32).rev(); - let non_null_rank = symbols.find(|&n| huff_node.count(n) != 0).ok_or(BUG)?; - let mut low_s = non_null_rank; - let mut node_nb = STARTNODE; - let node_root = node_nb + low_s - 1; - let mut low_n = node_nb; - - let mut join = |huff_node: &mut HuffNodes, n1: i32, n2: i32| { - let count = huff_node.count(n1).wrapping_add(huff_node.count(n2)); - huff_node.set_count(node_nb, count); - huff_node.set_parent(n1, node_nb); - huff_node.set_parent(n2, node_nb); - node_nb += 1; - }; +/// `HUF_buildTree`: sets the depth of `leaves`, which are sorted by decreasing count. There are at most 256. +fn build_tree(leaves: &mut [NodeElt]) { + /// The count of a parent that is not made yet, which is more than any count, + const NO_PARENT: u32 = 1 << 30; + /// and what there is past the last leaf, which is more than that. + const NO_LEAF: u32 = 1 << 31; + type PerParent = [T; HUF_SYMBOLVALUE_MAX as usize + 1]; + #[inline(always)] + fn at(table: &PerParent, n: u8) -> T { + table.get(usize::from(n)).copied().unwrap_or_default() + } + #[inline(always)] + fn put(table: &mut PerParent, n: u8, value: T) { + if let Some(slot) = table.get_mut(usize::from(n)) { + *slot = value; + } + } - join(huff_node, low_s, low_s - 1); - low_s -= 2; - for n in STARTNODE + 1..=node_root { - huff_node.set_count(n, 1 << 30); - } - // The sentinel is never the smaller of the two. - huff_node.set_count(-1, 1 << 31); - - // Join the two nodes of the lowest counts, which are the next leaves or the next parents. - for _ in STARTNODE + 1..=node_root { - let mut lowest = |huff_node: &HuffNodes| { - if huff_node.count(low_s) < huff_node.count(low_n) { - low_s -= 1; - low_s + 1 - } else { - low_n += 1; - low_n - 1 - } + // The parents, in the order they are made, which is by increasing count. + let nb_parents = leaves.len().saturating_sub(1) as u8; + let mut counts: PerParent = [NO_PARENT; _]; + let mut parents: PerParent = [0; _]; + + // Join the two nodes of the lowest counts, which are among the next two leaves and the next two parents. With the + // same count, a parent goes first. + let mut low_s = leaves.len().wrapping_sub(1); + let mut low_n = 0u8; + for parent in 0..nb_parents { + let leaf = |n: usize| leaves.get(n).map_or(NO_LEAF, |leaf| leaf.count); + let (s0, s1) = (leaf(low_s), leaf(low_s.wrapping_sub(1))); + let (n0, n1) = (at(&counts, low_n), at(&counts, low_n.wrapping_add(1))); + let first_is_leaf = s0 < n0; + let second_is_leaf = if first_is_leaf { s1 < n0 } else { s0 < n1 }; + let first = if first_is_leaf { s0 } else { n0 }; + let second = match (first_is_leaf, second_is_leaf) { + (true, true) => s1, + (true, false) => n0, + (false, true) => s0, + (false, false) => n1, }; - let n1 = lowest(huff_node); - let n2 = lowest(huff_node); - join(huff_node, n1, n2); + put(&mut counts, parent, first.wrapping_add(second)); + // The nodes that are not taken this time get their parent when they are. + for n in [low_s, low_s.wrapping_sub(1)] { + if let Some(leaf) = leaves.get_mut(n) { + leaf.parent = u16::from(parent); + } + } + put(&mut parents, low_n, parent); + put(&mut parents, low_n.wrapping_add(1), parent); + let nb_leaves = u8::from(first_is_leaf) + u8::from(second_is_leaf); + low_s = low_s.wrapping_sub(usize::from(nb_leaves)); + low_n = low_n.wrapping_add(2 - nb_leaves); } // The parents come after their children, so that this goes from the root down. - huff_node.set_nb_bits(node_root, 0); - for n in (STARTNODE..node_root).rev().chain(0..=non_null_rank) { - let parent = i32::from(huff_node.get(n).parent); - huff_node.set_nb_bits(n, huff_node.nb_bits(parent) + 1); + let mut depths: PerParent = [0; _]; + for n in (0..nb_parents.saturating_sub(1)).rev() { + let depth = at(&depths, at(&parents, n)).wrapping_add(1); + put(&mut depths, n, depth); + } + for leaf in leaves { + leaf.nb_bits = at(&depths, leaf.parent as u8).wrapping_add(1); } - - Ok(non_null_rank) } -/// `HUF_buildCTableFromTree`: no leaf is deeper than `max_nb_bits`, which is at most 12. +/// `HUF_buildCTableFromTree`: `leaves` are the symbols `present`, none deeper than `max_nb_bits`, which is at most 12. fn build_ctable_from_tree( ctable: &mut HufCTable, - huff_node: &[NodeElt], - non_null_rank: i32, + leaves: &[NodeElt], + present: &[u8], max_symbol_value: u32, max_nb_bits: u32, ) -> Result<()> { - let alphabet_size = max_symbol_value as usize + 1; - let ct = ctable.elements_mut().sub_mut(..alphabet_size)?; + let ct = ctable.elements_mut(); + ct.sub_mut(..=max_symbol_value as usize)?.fill(0); let mut nb_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 1]; - for node in huff_node.sub(..=non_null_rank as usize)? { + for node in leaves { *nb_per_rank.at_mut(usize::from(node.nb_bits))? += 1; - } - for node in huff_node.sub(..alphabet_size)? { ct.put(usize::from(node.byte), u64::from(node.nb_bits))?; } - set_values(ct, &nb_per_rank, max_nb_bits)?; + let mut val_per_rank = first_values(&nb_per_rank, max_nb_bits)?; + for &symbol in present { + set_next_value(ct.at_mut(usize::from(symbol))?, &mut val_per_rank)?; + } ctable.write_header(max_nb_bits, max_symbol_value); Ok(()) } @@ -669,16 +686,17 @@ pub(crate) fn build_ctable( } let mut huff_node = HuffNodes([NodeElt::default(); _]); - sort(&mut huff_node, count.sub(..=max_symbol_value as usize)?)?; - let non_null_rank = build_tree(&mut huff_node, max_symbol_value)?; + let present = sort(&mut huff_node, count.sub(..=max_symbol_value as usize)?)?; + let non_null_rank = present.len.checked_sub(1).ok_or(BUG)?; + build_tree(huff_node.0.sub_mut(..present.len)?); max_nb_bits = set_max_height(&mut huff_node, non_null_rank as u32, max_nb_bits)?; if max_nb_bits > HUF_TABLELOG_MAX { return Err(ErrorCode::Generic); } build_ctable_from_tree( ctable, - huff_node.leaves(), - non_null_rank, + huff_node.0.sub(..present.len)?, + present.get(), max_symbol_value, max_nb_bits, )?; From 74929d0b1e0aa5f4a0a036f754da14fa003fe246 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:20:49 +0000 Subject: [PATCH 041/199] zstd: less between the end of a match and the next look at the table, in fast --- src/zstd/compress/fast.rs | 30 +++++++++++++++++------------- src/zstd/compress/internal.rs | 11 ++++------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index e57cc584d77d..315f84402782 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -457,8 +457,8 @@ impl Finder { let mut ip2 = ip0 + step; let mut current0 = base_index.wrapping_add(ip0); - // Ends with where 4 bytes are the same as earlier. - let (ip, back, off_base) = loop { + // Ends with where a match starts and ends. + let (start, end, off_base) = loop { // There is a position after `ip2` to look at, and that one is before the last that can be. let Some(ahead) = near.window::<{ HASH_READ_SIZE + 2 }>(ip2) else { break 'block; @@ -470,7 +470,8 @@ impl Finder { { table_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); let back = u32::from(sink.same_byte_before(ip2, rep_offset1)); - break (ip2, back, REPCODE1_TO_OFFBASE); + let end = ip2 + 4 + sink.count_before(ip2 + 4, rep_offset1); + break (ip2 - back, end, REPCODE1_TO_OFFBASE); } let mut match0 = idx0.wrapping_sub(base_index); if near.read32(match0) == Some(bytes0 as u32) { @@ -504,26 +505,29 @@ impl Finder { let ip = current0.wrapping_sub(base_index); rep_offset2 = rep_offset1; rep_offset1 = ip - match0; + let end = ip + 4 + near.count(ip + 4, match0 + 4); let back = near.count_backwards(ip, sink.anchor, match0, 0); - break (ip, back, offset_to_offbase(rep_offset1)); + break (ip - back, end, offset_to_offbase(rep_offset1)); }; - ip0 = ip + 4 + sink.count_before(ip + 4, rep_offset1); - sink.store(ip - back, ip0, off_base); + ip0 = end; + sink.store(start, end, off_base); // `ip0` is not past the last position that can be looked at. - let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { + let Some(bytes) = near.read64(ip0) else { break 'block; }; + bytes0 = bytes; if let Some(to_insert) = near.read64((current0 + 2).wrapping_sub(base_index)) { table_put(hash_table, hash_of::(to_insert, hlog), current0 + 2); } - table_put( - hash_table, - hash_of::(le64_at::<0, _>(around), hlog), - base_index.wrapping_add(ip0 - 2), - ); - bytes0 = le64_at::<2, _>(around); + if let Some(before) = near.read64(ip0 - 2) { + table_put( + hash_table, + hash_of::(before, hlog), + base_index.wrapping_add(ip0 - 2), + ); + } while rep_offset2 > 0 && sink.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) { let end = ip0 + 4 + sink.count_before(ip0 + 4, rep_offset2); diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index a0930ff340e3..b91baa5e55dc 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -805,18 +805,16 @@ impl<'a> Near<'a> { } } -/// `table[hash]`. The size of `table` is a power of 2 that `hash` is less than, which the mask tells the compiler. +/// `table[hash]`. `hash` is less than the size of `table`: to check that delays nothing, which a mask would. #[inline(always)] pub(crate) fn table_get(table: &[u32], hash: usize) -> u32 { - let cell = table.get(hash & table.len().wrapping_sub(1)); - cell.copied().unwrap_or(0) + table.get(hash).copied().unwrap_or(0) } /// `table[hash] = index`, likewise. #[inline(always)] pub(crate) fn table_put(table: &mut [u32], hash: usize, index: u32) { - let mask = table.len().wrapping_sub(1); - if let Some(cell) = table.get_mut(hash & mask) { + if let Some(cell) = table.get_mut(hash) { *cell = index; } } @@ -824,8 +822,7 @@ pub(crate) fn table_put(table: &mut [u32], hash: usize, index: u32) { /// `table[hash] = index`, likewise. Returns what was there. #[inline(always)] pub(crate) fn table_swap(table: &mut [u32], hash: usize, index: u32) -> u32 { - let mask = table.len().wrapping_sub(1); - match table.get_mut(hash & mask) { + match table.get_mut(hash) { Some(cell) => core::mem::replace(cell, index), None => 0, } From 7a59b4e24fe44c641870cbd3e65e8e8d56ea862f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:24:40 +0000 Subject: [PATCH 042/199] zstd: less between the end of a match and the next look at the tables, and no branch on where an index leads in dfast with two segments --- src/zstd/compress/double_fast.rs | 12 ++++++---- src/zstd/compress/fast.rs | 40 +++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index a17de3efc4ad..351fe76a3655 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -222,7 +222,11 @@ impl Finder { ip = end; // `ip` is not past the last position that can be looked at. - let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip - 2) else { + let Some(bytes) = near.read64(ip) else { + break 'block; + }; + bytes0 = bytes; + let Some(around) = near.window::<{ 1 + HASH_READ_SIZE }>(ip - 2) else { break 'block; }; let index_to_insert = curr + 2; @@ -244,7 +248,6 @@ impl Finder { hash_of::(le64_at::<1, _>(around), h_bits_s), base_index.wrapping_add(ip - 1), ); - bytes0 = le64_at::<2, _>(around); while offset_2 > 0 && sink.read32_before(ip, offset_2) == Some(bytes0 as u32) { let end = ip + 4 + sink.count_before(ip + 4, offset_2); @@ -535,10 +538,9 @@ impl Finder { ip = end; } else { // The match, and how much of it is known. `ip` is moved to where it is for. - let (place, known) = if window.read64(place_long) == Some(bytes0) && place_long > 0 - { + let (place, known) = if window.diff64_after_first(place_long, bytes0) == 0 { (place_long, 8) - } else if window.read32(place_short) == Some(bytes0 as u32) && place_short > 0 { + } else if window.diff64_after_first(place_short, bytes0) as u32 == 0 { // A short match: a long one at the next position is preferred. let bytes1 = le64_at::<1, _>(here); let place3 = table_swap(hash_long, hash8(bytes1, h_bits_l, 0), curr + 1) diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 315f84402782..10be55aa4c62 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -380,6 +380,14 @@ impl<'a> Segments<'a> { spot.segment.read64(spot.pos) } + /// What differs between `bytes` and the 8 bytes at `place`: 0 if nothing does. Not 0 in the lower half if they are + /// not in one segment, or if `place` is the first one, which takes no branch to find out. + #[inline(always)] + pub(super) fn diff64_after_first(self, place: u32, bytes: u64) -> u64 { + let spot = self.spot(place); + spot.segment.diff64(spot.pos, bytes) | u64::from(place == 0) + } + /// The 4 bytes that are `offset` before the position `pos`, if they are in one segment. Which one that is does not /// change much from one time to the next. #[inline(always)] @@ -647,18 +655,20 @@ impl Finder { } // `ip0` is not past the last position that can be looked at. - let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { + let Some(bytes) = near.read64(ip0) else { break 'block; }; + bytes0 = bytes; if let Some(to_insert) = near.read64((current0 + 2).wrapping_sub(base_index)) { table_put(hash_table, hash_of::(to_insert, hlog), current0 + 2); } - table_put( - hash_table, - hash_of::(le64_at::<0, _>(around), hlog), - base_index.wrapping_add(ip0 - 2), - ); - bytes0 = le64_at::<2, _>(around); + if let Some(before) = near.read64(ip0 - 2) { + table_put( + hash_table, + hash_of::(before, hlog), + base_index.wrapping_add(ip0 - 2), + ); + } while window.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) && rep_offset2 > 0 { let rep_place = window.place_of(ip0).wrapping_sub(rep_offset2); @@ -866,18 +876,20 @@ impl Finder { } // `ip0` is not past the last position that can be looked at. - let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { + let Some(bytes) = near.read64(ip0) else { break 'block; }; + bytes0 = bytes; if let Some(to_insert) = near.read64((curr + 2).wrapping_sub(base_index)) { table_put(hash_table, hash_of::(to_insert, hlog), curr + 2); } - table_put( - hash_table, - hash_of::(le64_at::<0, _>(around), hlog), - base_index.wrapping_add(ip0 - 2), - ); - bytes0 = le64_at::<2, _>(around); + if let Some(before) = near.read64(ip0 - 2) { + table_put( + hash_table, + hash_of::(before, hlog), + base_index.wrapping_add(ip0 - 2), + ); + } while window.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) { let rep_place = window.place_of(ip0).wrapping_sub(rep_offset2); From 129ede982689094f08ae55e4ab053f3b44b01742 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:24:53 +0000 Subject: [PATCH 043/199] zstd: how many bits a sequence takes is read from the bits themselves --- src/zstd/compress/block_entropy.rs | 5 +- src/zstd/compress/sequences.rs | 143 +++++++++++++++-------------- src/zstd/compress/superblock.rs | 2 +- 3 files changed, 79 insertions(+), 71 deletions(-) diff --git a/src/zstd/compress/block_entropy.rs b/src/zstd/compress/block_entropy.rs index 619c0fb19658..73847b74a829 100644 --- a/src/zstd/compress/block_entropy.rs +++ b/src/zstd/compress/block_entropy.rs @@ -228,7 +228,7 @@ fn entropy_compress_seq_store_internal( next_entropy.fse.copy_from(&prev_entropy.fse); return Ok(op); } - let last_count_size = { + let (last_count_size, many_bits) = { let seq_head = op; op += 1; let mut histograms = seq_to_codes(store, chunk)?; @@ -248,7 +248,7 @@ fn entropy_compress_seq_store_internal( + ((stats.ml_type as u32) << 2)) as u8, )?; op += stats.size; - stats.last_count_size + (stats.last_count_size, histograms.many_bits) }; let range = chunk.seq_start..chunk.seq_end; @@ -257,6 +257,7 @@ fn entropy_compress_seq_store_internal( &next_entropy.fse, store.codes.sub(range.clone())?, store.sequences.sub(range)?, + many_bits, )?; op += bitstream_size; // Decoders up to 1.3.4 fail on a table header with fewer than 4 bytes after it. diff --git a/src/zstd/compress/sequences.rs b/src/zstd/compress/sequences.rs index 1edbb720290b..960e32bdd833 100644 --- a/src/zstd/compress/sequences.rs +++ b/src/zstd/compress/sequences.rs @@ -1,5 +1,7 @@ //! `zstd_compress_sequences.c`: how the sequences of a block are encoded. +use core::num::NonZero; + use super::fse::{ self, FSE_NCOUNTBOUND, FseCTable, FseRepeat, FseSymbolTransform, PredefinedCTable, }; @@ -464,21 +466,17 @@ pub(crate) struct SeqCodes { pub(crate) ll: Code, pub(crate) of: Code, pub(crate) ml: Code, - /// How many `extra_bits` are, or [`Self::TOO_MANY`]. - nb_extra_bits: u8, } impl SeqCodes { /// With more than this on top of the bits that leave the three states, a sequence may not fit the container. const MAX_EXTRA_BITS: u32 = 64 - 7 - (LL_FSE_LOG + ML_FSE_LOG + OFF_FSE_LOG) - 1; - const TOO_MANY: u8 = 0x80; /// `None` if there are too many extra bits. #[inline(always)] fn new(ll: LengthCode, ml: LengthCode, off_base: u32) -> Option { let of = highbit32(off_base); - let nb_extra_bits = ll.nb_bits + ml.nb_bits + of; - if nb_extra_bits > Self::MAX_EXTRA_BITS { + if ll.nb_bits + ml.nb_bits + of > Self::MAX_EXTRA_BITS { return None; } // The highest bit of `off_base`, which the code stands for, is the bit set above the others. @@ -488,17 +486,6 @@ impl SeqCodes { ll: ll.code, of: Code::new(of), ml: ml.code, - nb_extra_bits: nb_extra_bits as u8, - }) - } - - fn new_any(ll: LengthCode, ml: LengthCode, off_base: u32) -> Self { - Self::new(ll, ml, off_base).unwrap_or_else(|| Self { - extra_bits: 1, - ll: ll.code, - of: Code::new(highbit32(off_base)), - ml: ml.code, - nb_extra_bits: Self::TOO_MANY, }) } @@ -509,7 +496,20 @@ impl SeqCodes { self.ll = codes.ll; self.of = codes.of; self.ml = codes.ml; - self.nb_extra_bits = codes.nb_extra_bits; + } + + fn ll_bits(self) -> u32 { + u32::from(LL_BITS.get(self.ll as usize).copied().unwrap_or(0)) + } + + fn ml_bits(self) -> u32 { + u32::from(ML_BITS.get(self.ml as usize).copied().unwrap_or(0)) + } + + /// Whether `extra_bits` lacks the extra bits. + fn has_many_bits(self) -> bool { + self.extra_bits == 1 + && self.ll_bits() + self.ml_bits() + self.of as u32 > Self::MAX_EXTRA_BITS } } @@ -542,6 +542,8 @@ pub(crate) struct SeqHistograms { pub(crate) ll: CodeCounts, pub(crate) of: CodeCounts, pub(crate) ml: CodeCounts, + /// Whether some sequence has too many extra bits to be written at once. + pub(crate) many_bits: bool, } impl SeqHistograms { @@ -552,11 +554,24 @@ impl SeqHistograms { count_code(&mut self.ml, codes.ml, 1); } + /// The codes of any sequence, which are not counted. + fn codes_of(&mut self, ll: LengthCode, ml: LengthCode, off_base: u32) -> SeqCodes { + SeqCodes::new(ll, ml, off_base).unwrap_or_else(|| { + self.many_bits = true; + SeqCodes { + extra_bits: 1, + ll: ll.code, + of: Code::new(highbit32(off_base)), + ml: ml.code, + } + }) + } + /// For a sequence that takes more than the tables of [`seq_to_codes`]. #[cold] #[inline(never)] fn add_unusual(&mut self, seq: SeqDef) -> SeqCodes { - let codes = SeqCodes::new_any( + let codes = self.codes_of( LL_CODES.get_any(seq.lit_length, LL_DELTA_CODE), ML_CODES.get_any(seq.ml_base, ML_DELTA_CODE), seq.off_base, @@ -578,6 +593,7 @@ pub(crate) fn seq_to_codes( ll: [0; 64], of: [0; 64], ml: [0; 64], + many_bits: false, }; for (codes, seq) in codes.iter_mut().zip(sequences) { if let (Some(ll), Some(ml)) = (LL_CODES.get(seq.lit_length), ML_CODES.get(seq.ml_base)) @@ -609,19 +625,23 @@ pub(crate) fn seq_to_codes( bits: u32::from(bits), }; count_code(counts, length.code, 1); - *codes = SeqCodes::new_any(ll, ml, seq.off_base); + *codes = histograms.codes_of(ll, ml, seq.off_base); } histograms } -/// Bits on their way to the stream: the first to be written are the lowest. +/// Bits on their way to the stream, with a bit set above them: the first to be written are the lowest. #[derive(Clone, Copy)] -struct SeqBits { - bits: u64, - nb_bits: u32, -} +struct SeqBits(u64); impl SeqBits { + /// The lowest `nb_bits` of `bits`, which is less than 32. + #[inline(always)] + fn new(bits: u32, nb_bits: u32) -> Self { + let above = 1 << (nb_bits & 31); + Self((u64::from(bits) & (above - 1)) | above) + } + /// `FSE_encodeSymbol`: the bits that leave the state go below the others. #[inline(always)] fn encode_below(&mut self, table: &FseCTable, state: &mut u32, code: Code) { @@ -629,10 +649,9 @@ impl SeqBits { let nb_bits = state.wrapping_add(transform.delta_nb_bits) >> 16; let kept = state.wrapping_shr(nb_bits); // Taking `kept` from what comes above the state cancels the bits that stay in it. - self.bits = (self.bits.wrapping_sub(u64::from(kept))) + self.0 = (self.0.wrapping_sub(u64::from(kept))) .wrapping_shl(nb_bits) .wrapping_add(u64::from(*state)); - self.nb_bits += nb_bits; let next = kept.wrapping_add_signed(transform.delta_find_state) as usize; let state_table = &table.state_table; *state = u32::from((state_table.get(next & (state_table.len() - 1)).copied()).unwrap_or(0)); @@ -655,8 +674,8 @@ struct SeqStream<'a> { } impl SeqStream<'_> { - /// `BIT_addBits` and `BIT_flushBits`: `seq` makes at most 56 bits, with a bit set above them. `None` if the output is - /// too small, which is found out first so that there is less to keep while `seq` is at work. + /// `BIT_addBits` and `BIT_flushBits`: `seq` makes at most 56 bits. `None` if the output is too small, which is found + /// out first so that there is less to keep while `seq` is at work. #[inline(always)] fn write_with(&mut self, seq: impl FnOnce() -> SeqBits) -> Option<()> { let pos = self.bit_pos >> 3; @@ -664,25 +683,18 @@ impl SeqStream<'_> { return None; } let out: &mut [u8; 8] = self.dst.get_mut(pos..)?.first_chunk_mut()?; - let seq = seq(); + let seq = NonZero::new(seq().0)?; let [partial, ..] = *out; // The lowest of the new bits replaces the bit set. - let bits = seq - .bits - .wrapping_sub(1) - .wrapping_shl(self.bit_pos as u32 & 7); + let bits = (seq.get() - 1).wrapping_shl(self.bit_pos as u32 & 7); *out = bits.wrapping_add(u64::from(partial)).to_le_bytes(); - self.bit_pos += seq.nb_bits as usize; + self.bit_pos += seq.ilog2() as usize; Some(()) } #[inline(always)] fn write(&mut self, bits: u32, nb_bits: u32) -> Option<()> { - let above = 1 << (nb_bits & 31); - self.write_with(|| SeqBits { - bits: (u64::from(bits) & (above - 1)) | above, - nb_bits, - }) + self.write_with(|| SeqBits::new(bits, nb_bits)) } } @@ -690,12 +702,9 @@ impl SeqStream<'_> { struct Encoded { states: SeqStates, bit_pos: usize, - /// How many sequences are left. - nb_left: usize, } -/// From the last of `codes` towards the first. It stops after one whose extra bits are too many, with -/// [`SeqCodes::TOO_MANY`] bits too many in the position. `None` if the output is too small. +/// From the last of `codes` to the first. `None` if the output is too small. /// /// On its own, so that nothing else competes for the registers. #[inline(never)] @@ -712,13 +721,9 @@ fn encode_run( match_length: states.match_length, offset_bits: states.offset_bits, }; - let mut codes = codes.iter(); - while let Some(seq_codes) = codes.next_back() { + for seq_codes in codes.iter().rev() { stream.write_with(|| { - let mut seq = SeqBits { - bits: u64::from(seq_codes.extra_bits), - nb_bits: u32::from(seq_codes.nb_extra_bits), - }; + let mut seq = SeqBits(u64::from(seq_codes.extra_bits)); seq.encode_below( &tables.litlength_c_table, &mut states.lit_length, @@ -736,14 +741,10 @@ fn encode_run( ); seq })?; - if seq_codes.nb_extra_bits == SeqCodes::TOO_MANY { - break; - } } Some(Encoded { states, bit_pos: stream.bit_pos, - nb_left: codes.len(), }) } @@ -753,6 +754,7 @@ fn encode_sequences_into( tables: &FseCTables, codes: &[SeqCodes], sequences: &[SeqDef], + many_bits: bool, ) -> Option { *dst.first_mut()? = 1; let mut stream = SeqStream { dst, bit_pos: 0 }; @@ -762,31 +764,34 @@ fn encode_sequences_into( match_length: (tables.matchlength_c_table).first_state(last_codes.ml as u8), offset_bits: tables.offcode_c_table.first_state(last_codes.of as u8), }; - let mut nb_left = codes.len() - 1; + // The sequence whose extra bits are to be written: the last one, which has nothing else to write, + let mut apart = codes.len() - 1; loop { - // The extra bits of the last sequence, which has nothing else to write, or of the one that the run stopped at. - let (seq_codes, seq) = (codes.get(nb_left)?, sequences.get(nb_left)?); - let ll_bits = LL_BITS.get(seq_codes.ll as usize).copied()?; - let ml_bits = ML_BITS.get(seq_codes.ml as usize).copied()?; - stream.write(u32::from(seq.lit_length), u32::from(ll_bits))?; - stream.write(u32::from(seq.ml_base), u32::from(ml_bits))?; + let (seq_codes, seq) = (codes.get(apart)?, sequences.get(apart)?); + stream.write(u32::from(seq.lit_length), seq_codes.ll_bits())?; + stream.write(u32::from(seq.ml_base), seq_codes.ml_bits())?; stream.write(seq.off_base, seq_codes.of as u32)?; - let before = nb_left; + // then each that has too many. + let left = codes.get(..apart)?; + let next = if many_bits { + left.iter().rposition(|codes| codes.has_many_bits()) + } else { + None + }; let run = encode_run( stream.dst, stream.bit_pos, tables, states, - codes.get(..nb_left)?, + left.get(next.unwrap_or(0)..)?, )?; states = run.states; stream.bit_pos = run.bit_pos; - nb_left = run.nb_left; - if nb_left == before || codes.get(nb_left)?.nb_extra_bits != SeqCodes::TOO_MANY { - break; + match next { + Some(next) => apart = next, + None => break, } - stream.bit_pos -= usize::from(SeqCodes::TOO_MANY); } // `FSE_flushCState` @@ -797,15 +802,17 @@ fn encode_sequences_into( Some(stream.bit_pos + 1) } -/// `ZSTD_encodeSequences`: `codes` are those of `sequences`, of which there is at least one. +/// `ZSTD_encodeSequences`: `codes` are those of `sequences`, of which there is at least one. Without `many_bits`, none +/// has too many extra bits to be written at once. pub(crate) fn encode_sequences( dst: &mut [u8], tables: &FseCTables, codes: &[SeqCodes], sequences: &[SeqDef], + many_bits: bool, ) -> Result { let capacity = dst.len(); - encode_sequences_into(dst, tables, codes, sequences) + encode_sequences_into(dst, tables, codes, sequences, many_bits) // The stream has to end before the last position that a container can be written to. .filter(|nb_bits| (nb_bits >> 3) + size_of::() < capacity) .map(|nb_bits| nb_bits.div_ceil(8)) diff --git a/src/zstd/compress/superblock.rs b/src/zstd/compress/superblock.rs index d95d8d0f9b18..c6bb28bab0b5 100644 --- a/src/zstd/compress/superblock.rs +++ b/src/zstd/compress/superblock.rs @@ -161,7 +161,7 @@ fn compress_sub_block_sequences( )?; } - let bitstream_size = encode_sequences(dst.sub_mut(op..)?, fse_tables, codes, sequences)?; + let bitstream_size = encode_sequences(dst.sub_mut(op..)?, fse_tables, codes, sequences, true)?; op += bitstream_size; // Decoders up to 1.3.4 fail on a table header with fewer than 4 bytes after it, if write_entropy From ef448e6a8a2d2ddc09619616c9f38cf7e333a69a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:27:31 +0000 Subject: [PATCH 044/199] zstd: fast copies up to 32 literals without looking at their number --- src/zstd/compress/double_fast.rs | 18 +++++++++--------- src/zstd/compress/fast.rs | 18 +++++++++--------- src/zstd/compress/internal.rs | 13 +++++++------ 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 351fe76a3655..74e968e3c907 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -167,7 +167,7 @@ impl Finder { { ip += 1; let end = ip + 4 + sink.count_before(ip + 4, offset_1); - sink.store(ip, end, REPCODE1_TO_OFFBASE); + sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); break end; } @@ -216,7 +216,7 @@ impl Finder { ); } let back = near.count_backwards(ip, sink.anchor, match0, 0); - sink.store(ip - back, end, offset_to_offbase(offset_1)); + sink.store::<16>(ip - back, end, offset_to_offbase(offset_1)); break end; }; ip = end; @@ -255,7 +255,7 @@ impl Finder { let index = base_index.wrapping_add(ip); table_put(hash_small, hash_of::(bytes0, h_bits_s), index); table_put(hash_long, hash8(bytes0, h_bits_l, 0), index); - sink.store(ip, end, REPCODE1_TO_OFFBASE); + sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); ip = end; let Some(bytes) = near.read64(ip) else { break 'block; @@ -343,7 +343,7 @@ impl Finder { ip += 1; let rep_place = window.place_of(ip).wrapping_sub(offset_1); let end = ip + 4 + window.count(ip + 4, rep_place + 4); - sink.store(ip, end, REPCODE1_TO_OFFBASE); + sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); ip = end; } else { // The place of the match, and how much of it is known. `ip` is moved to where it is for. @@ -390,7 +390,7 @@ impl Finder { let back = window.count_backwards(ip, sink.anchor, place); offset_2 = offset_1; offset_1 = window.place_of(ip) - place; - sink.store(ip - back, end, offset_to_offbase(offset_1)); + sink.store::<16>(ip - back, end, offset_to_offbase(offset_1)); ip = end; } @@ -427,7 +427,7 @@ impl Finder { let index = base_index.wrapping_add(ip); table_put(hash_small, hash_of::(bytes, h_bits_s), index); table_put(hash_long, hash8(bytes, h_bits_l, 0), index); - sink.store(ip, end, REPCODE1_TO_OFFBASE); + sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); ip = end; } } @@ -534,7 +534,7 @@ impl Finder { ip += 1; let rep_place = window.place_of(ip) - offset_1; let end = ip + 4 + window.count(ip + 4, rep_place + 4); - sink.store(ip, end, REPCODE1_TO_OFFBASE); + sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); ip = end; } else { // The match, and how much of it is known. `ip` is moved to where it is for. @@ -559,7 +559,7 @@ impl Finder { let back = window.count_backwards(ip, sink.anchor, place); offset_2 = offset_1; offset_1 = window.place_of(ip) - place; - sink.store(ip - back, end, offset_to_offbase(offset_1)); + sink.store::<16>(ip - back, end, offset_to_offbase(offset_1)); ip = end; } @@ -599,7 +599,7 @@ impl Finder { let index = base_index.wrapping_add(ip); table_put(hash_small, hash_of::(bytes, h_bits_s), index); table_put(hash_long, hash8(bytes, h_bits_l, 0), index); - sink.store(ip, end, REPCODE1_TO_OFFBASE); + sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); ip = end; } } diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 10be55aa4c62..10b4c217d0ad 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -300,11 +300,11 @@ impl<'a, 'b> MatchSink<'a, 'b> { /// Stores the match from `ip` to `end`, with the literals before it. #[inline(always)] - pub(super) fn store(&mut self, ip: u32, end: u32, off_base: u32) { + pub(super) fn store(&mut self, ip: u32, end: u32, off_base: u32) { let anchor = self.anchor; self.anchor = end; if (self.seq_store) - .store_seq(ip - anchor, self.near, anchor, off_base, end - ip) + .store_seq::(ip - anchor, self.near, anchor, off_base, end - ip) .is_err() { core::hint::cold_path(); @@ -519,7 +519,7 @@ impl Finder { }; ip0 = end; - sink.store(start, end, off_base); + sink.store::<32>(start, end, off_base); // `ip0` is not past the last position that can be looked at. let Some(bytes) = near.read64(ip0) else { @@ -545,7 +545,7 @@ impl Finder { hash_of::(bytes0, hlog), base_index.wrapping_add(ip0), ); - sink.store(ip0, end, REPCODE1_TO_OFFBASE); + sink.store::<32>(ip0, end, REPCODE1_TO_OFFBASE); ip0 = end; let Some(bytes) = near.read64(ip0) else { break 'block; @@ -642,7 +642,7 @@ impl Finder { }; ip0 = ip + 4 + window.count(ip + 4, place + 4); - sink.store(ip - back, ip0, off_base); + sink.store::<32>(ip - back, ip0, off_base); if next < ip0 && let Some(bytes) = near.read64(next) @@ -679,7 +679,7 @@ impl Finder { hash_of::(bytes0, hlog), base_index.wrapping_add(ip0), ); - sink.store(ip0, end, REPCODE1_TO_OFFBASE); + sink.store::<32>(ip0, end, REPCODE1_TO_OFFBASE); ip0 = end; let Some(bytes) = near.read64(ip0) else { break 'block; @@ -839,7 +839,7 @@ impl Finder { ip0 += 1; let rep_place = window.place_of(ip0).wrapping_sub(rep_offset1); let end = ip0 + 4 + window.count(ip0 + 4, rep_place + 4); - sink.store(ip0, end, REPCODE1_TO_OFFBASE); + sink.store::<32>(ip0, end, REPCODE1_TO_OFFBASE); ip0 = end; break None; } @@ -871,7 +871,7 @@ impl Finder { let back = window.count_backwards(ip0, sink.anchor, place); rep_offset2 = rep_offset1; rep_offset1 = window.place_of(ip0) - place; - sink.store(ip0 - back, end, offset_to_offbase(rep_offset1)); + sink.store::<32>(ip0 - back, end, offset_to_offbase(rep_offset1)); ip0 = end; } @@ -900,7 +900,7 @@ impl Finder { hash_of::(bytes0, hlog), base_index.wrapping_add(ip0), ); - sink.store(ip0, end, REPCODE1_TO_OFFBASE); + sink.store::<32>(ip0, end, REPCODE1_TO_OFFBASE); ip0 = end; let Some(bytes) = near.read64(ip0) else { break 'block; diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index b91baa5e55dc..6fbab725601c 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -435,9 +435,10 @@ impl<'a> SeqSink<'a> { } } - /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `src` from `literals` on. + /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `src` from `literals` on. Up to `SHORT` of them, which + /// is at most [`WILDCOPY_OVERLENGTH`], that many bytes are copied whatever their number. #[inline(always)] - pub(crate) fn store_seq( + pub(crate) fn store_seq( &mut self, lit_length: u32, src: Near<'_>, @@ -446,12 +447,12 @@ impl<'a> SeqSink<'a> { match_length: u32, ) -> Result<()> { let lit_len = self.lit_len as usize; - let room = self.lit_buffer.get_mut(lit_len..lit_len + 16); + let room = self.lit_buffer.get_mut(lit_len..lit_len + SHORT); match ( - src.chunk16(literals), - room.and_then(<[u8]>::first_chunk_mut::<16>), + src.window::(literals), + room.and_then(<[u8]>::first_chunk_mut::), ) { - (Some(from), Some(to)) if lit_length <= 16 => *to = *from, + (Some(from), Some(to)) if lit_length as usize <= SHORT => *to = *from, _ => self.lit_buffer.write_at( lit_len, src.bytes().span(literals as usize, lit_length as usize)?, From 226e9d6d9da2cb9484ecc8ab18b042bcd1b59be6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:32:32 +0000 Subject: [PATCH 045/199] zstd: speed up the binary tree match finders and the optimal parser The ways down the trees wait for memory, so the hash cell, the root and what is below it are hinted at a few positions ahead, and both children at each step. Candidates are compared 16 bytes at a time, which also tells which of the two is smaller. The match finders are no longer instantiated per minimum match and dictionary mode: what a block does not change is gathered once in `Trees` or `Dubt`. The optimal parser keeps one array per member of `ZSTD_optimal_t`, so that the lengths of a match are tried on vectors, and the price of a symbol is worked out when its frequency changes rather than each time it is asked for. --- src/zstd/compress/cctx.rs | 4 +- src/zstd/compress/internal.rs | 602 ++++++++++- src/zstd/compress/lazy.rs | 641 +++++------- src/zstd/compress/opt.rs | 1775 +++++++++++++++------------------ 4 files changed, 1633 insertions(+), 1389 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index e4c1da17d745..e27ea4ab83e4 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -218,10 +218,10 @@ fn reset_match_state( reset_table(&mut ms.opt.match_length_freq, MAX_ML as usize + 1, false)?; reset_table(&mut ms.opt.off_code_freq, MAX_OFF as usize + 1, false)?; reset_struct_table(&mut ms.opt.match_table, ZSTD_OPT_SIZE, false)?; - reset_struct_table(&mut ms.opt.price_table, ZSTD_OPT_SIZE, false)?; + ms.opt.price_table.resize(ZSTD_OPT_SIZE)?; } else { ms.opt.match_table = Vec::new(); - ms.opt.price_table = Vec::new(); + ms.opt.price_table.resize(0)?; } ms.c_params = *c_params; diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 0ae2205bd39e..de3310cbefa3 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1,5 +1,7 @@ //! `zstd_compress_internal.h`: what the parts of the compressor share. +use core::hint::{Locality, prefetch_read}; + use super::fse::{FseCTable, FseRepeat}; use super::huf::{HufCTable, HufRepeat}; use crate::common::{ @@ -7,7 +9,7 @@ use crate::common::{ ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; use crate::error::{ErrorCode, Result}; -use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64, write_le24}; +use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64, try_zeroed, write_le24}; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; pub(crate) const HASH_READ_SIZE: usize = 8; @@ -715,6 +717,73 @@ impl<'a> Near<'a> { len } + /// [`Self::count`] of positions that are known to have `from` bytes in common, and which one is before the other in + /// the order of what follows them. Only the sum of `match_pos` and `from` has to be a position. + #[inline(always)] + pub(crate) fn compare(self, pos: u32, match_pos: u32, from: u32) -> Common { + let mut len = match self.compare_chunks(pos, match_pos, from) { + Ok(common) => return common, + Err(len) => len, + }; + while let Some(byte) = self.read8(pos.wrapping_add(len)) + && let Some(match_byte) = self.read8(match_pos.wrapping_add(len)) + { + if byte != match_byte { + return Common::followed_by(len, match_byte, byte); + } + len += 1; + } + Common::up_to_the_end(len) + } + + /// [`Self::compare`], 16 bytes at a time as long as there are as many. Fails with how many bytes are known to be in + /// common when there are not. + #[inline(always)] + pub(crate) fn compare_chunks( + self, + pos: u32, + match_pos: u32, + from: u32, + ) -> core::result::Result { + self.compare_chunks_with(pos, self, match_pos, from) + } + + /// [`Self::compare_chunks`], for a match that is at `match_pos` in `other`. + #[inline(always)] + pub(crate) fn compare_chunks_with( + self, + pos: u32, + other: Self, + match_pos: u32, + from: u32, + ) -> core::result::Result { + let mut len = from; + while let (Some(bytes), Some(match_bytes)) = ( + self.chunk16(pos.wrapping_add(len)), + other.chunk16(match_pos.wrapping_add(len)), + ) { + // A comparison of vectors. + let mut differ = 0u32; + for (n, (byte, match_byte)) in bytes.iter().zip(match_bytes).enumerate() { + differ |= u32::from(byte != match_byte) << n; + } + if differ != 0 { + let first = differ.trailing_zeros(); + let byte = bytes.get(first as usize & 15).copied().unwrap_or(0); + let match_byte = match_bytes.get(first as usize & 15).copied().unwrap_or(0); + return Ok(Common::followed_by(len + first, match_byte, byte)); + } + len += 16; + } + Err(len) + } + + /// A hint that `pos`, which may be anything, is about to be read. + #[inline(always)] + pub(crate) fn prefetch(self, pos: u32) { + prefetch_read(self.0.as_ptr().wrapping_add(pos as usize), Locality::L1); + } + /// How many bytes before `pos`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. #[inline(always)] pub(crate) fn count_backwards( @@ -751,6 +820,433 @@ impl<'a> Near<'a> { } } +/// What a position and a match for it have in common. +#[derive(Clone, Copy)] +pub(crate) struct Common { + pub(crate) len: u32, + /// Whether the match is before the position in the order of what follows them. `None` if the input ends before + /// they differ. + pub(crate) match_is_smaller: Option, +} + +impl Common { + #[inline(always)] + pub(crate) const fn followed_by(len: u32, match_byte: u8, byte: u8) -> Self { + Self { + len, + match_is_smaller: Some(match_byte < byte), + } + } + + #[inline(always)] + pub(crate) const fn up_to_the_end(len: u32) -> Self { + Self { + len, + match_is_smaller: None, + } + } +} + +/// A table of binary trees of positions, in the order of what follows them: for each position, the roots of the smaller +/// ones and of the larger ones below it. Its size is a power of 2. +pub(crate) type Nodes = [[u32; 2]]; + +/// `chainTable`, as the strategies that use binary trees see it. +#[inline(always)] +pub(crate) fn bt_nodes(chain_table: &mut [u32]) -> &mut Nodes { + let (nodes, _) = chain_table.as_chunks_mut(); + nodes +} + +/// What is below `index`, the smaller ones first. +#[inline(always)] +pub(crate) fn bt_children(bt: &Nodes, index: u32) -> [u32; 2] { + let node = bt.get(index as usize & bt.len().wrapping_sub(1)); + node.copied().unwrap_or_default() +} + +/// Sets what [`bt_children`] returns. +#[inline(always)] +pub(crate) fn bt_set_children(bt: &mut Nodes, index: u32, children: [u32; 2]) { + let mask = bt.len().wrapping_sub(1); + if let Some(node) = bt.get_mut(index as usize & mask) { + *node = children; + } +} + +/// A hint that [`bt_children`] of `index` is about to be read. +#[inline(always)] +pub(crate) fn bt_prefetch(bt: &Nodes, index: u32) { + let node = index as usize & bt.len().wrapping_sub(1); + prefetch_read(bt.as_ptr().wrapping_add(node), Locality::L1); +} + +/// The way down a tree for a position that becomes its root: what is met on the way goes to its left or to its right. +pub(crate) struct Descent { + /// The positions from there down have been overwritten in the table. + bt_low: u32, + /// Where the next smaller position goes, as twice the position that it goes below, plus 1 for its larger side. And + /// the next larger one. + smaller_slot: usize, + larger_slot: usize, + /// What all the smaller positions from here on are sure to have in common with the new root, and the larger ones. + common_smaller: u32, + common_larger: u32, +} + +impl Descent { + #[inline(always)] + pub(crate) const fn new(bt: &Nodes, curr: u32) -> Self { + Self::down_to(curr, bt_low(bt, curr)) + } + + /// [`Self::new`], which does not go below the positions from `bt_low` down. + #[inline(always)] + pub(crate) const fn down_to(curr: u32, bt_low: u32) -> Self { + let slot = (curr as usize) << 1; + Self { + bt_low, + smaller_slot: slot, + larger_slot: slot + 1, + common_smaller: 0, + common_larger: 0, + } + } + + /// What the next position is sure to have in common with the new root. + #[inline(always)] + pub(crate) fn common(&self) -> u32 { + self.common_smaller.min(self.common_larger) + } + + /// Goes past `match_index`, which has `len` bytes in common with the new root and `children` below it. Returns the + /// next position, if the table still has it. + #[inline(always)] + pub(crate) fn step( + &mut self, + bt: &mut Nodes, + match_index: u32, + children: [u32; 2], + len: u32, + match_is_smaller: bool, + ) -> Option { + let slots = bt.as_flattened_mut(); + let slot = (match_index as usize) << 1; + let [smaller_child, larger_child] = children; + if match_is_smaller { + table_put(slots, self.smaller_slot, match_index); + self.common_smaller = len; + if match_index <= self.bt_low { + // This slot keeps its value: only the other one is cleared. + self.smaller_slot = self.larger_slot; + return None; + } + // What is larger than it may be smaller than the new root. + self.smaller_slot = slot + 1; + Some(larger_child) + } else { + table_put(slots, self.larger_slot, match_index); + self.common_larger = len; + if match_index <= self.bt_low { + self.larger_slot = self.smaller_slot; + return None; + } + self.larger_slot = slot; + Some(smaller_child) + } + } + + /// Nothing more goes to the left or to the right of the new root. + #[inline(always)] + pub(crate) fn end(&self, bt: &mut Nodes) { + let slots = bt.as_flattened_mut(); + table_put(slots, self.smaller_slot, 0); + table_put(slots, self.larger_slot, 0); + } +} + +/// The positions from there down have been overwritten in the table when `curr` is added. +#[inline(always)] +pub(crate) const fn bt_low(bt: &Nodes, curr: u32) -> u32 { + curr.saturating_sub(bt.len().wrapping_sub(1) as u32) +} + +/// What the match finders that use binary trees read. +#[derive(Clone, Copy)] +pub(crate) struct TreeInput<'a> { + /// The current segment, up to where matches end. A position in it is an index minus `dict_limit`. + pub(crate) near: Near<'a>, + pub(crate) dict_limit: u32, + /// The older segment. + pub(crate) dict: Prefix<'a>, + /// Whether there is enough of it for hints at what is about to be read to be worth giving. + pub(crate) far: bool, +} + +impl<'a> TreeInput<'a> { + /// `prefix` is the current segment, which starts at the index `dict_limit`, and `dict` the older one. + pub(crate) fn new( + prefix: Prefix<'a>, + dict: Prefix<'a>, + dict_limit: u32, + iend: usize, + ) -> Result { + let near = Near::new(prefix.buf.sub(prefix.pos_of(dict_limit)..iend)?)?; + Ok(Self { + near, + dict_limit, + dict, + far: near.len() > 1 << 16, + }) + } + + /// `ZSTD_count_2segments`, for the position `ip` of `near` and a match that starts in another segment. + #[inline(always)] + pub(crate) fn count_2segments( + &self, + ip: u32, + match_buf: &[u8], + match_pos: usize, + match_end: usize, + ) -> Result { + let buf = self.near.bytes(); + let len = count_2segments( + buf, + ip as usize, + match_buf, + match_pos, + buf.len(), + match_end, + 0, + )?; + Ok(len as u32) + } + + /// [`Self::count_2segments`] for a match at `match_index`, in the older segment. + #[inline(always)] + pub(crate) fn count_ext_dict(&self, ip: u32, match_index: u32) -> Result { + self.count_2segments( + ip, + self.dict.buf, + self.dict.pos_of(match_index), + self.dict.pos_of(self.dict_limit), + ) + } + + /// [`Near::compare`] wherever the match is, for what [`Near::compare_chunks`] leaves. + #[cold] + #[inline(never)] + fn compare_rest(&self, ip: u32, match_index: u32, from: u32) -> Result { + // A position of the older segment, as one of the current segment, only makes sense with a length that brings + // it there. + if u64::from(match_index) + u64::from(from) >= u64::from(self.dict_limit) { + let match_pos = match_index.wrapping_sub(self.dict_limit); + return Ok(self.near.compare(ip, match_pos, from)); + } + + let len = from + self.count_ext_dict(ip + from, match_index.wrapping_add(from))?; + let Some(byte) = self.near.read8(ip.wrapping_add(len)) else { + return Ok(Common::up_to_the_end(len)); + }; + let differs = match_index.wrapping_add(len); + let match_byte = if differs < self.dict_limit { + self.dict.buf.at(self.dict.pos_of(differs))? + } else { + let pos = differs.wrapping_sub(self.dict_limit); + self.near.read8(pos).ok_or(BUG)? + }; + Ok(Common::followed_by(len, match_byte, byte)) + } +} + +/// Hints at what the ways down the trees start with for the positions after `ip`, which are likely to be the next: the +/// cell of the hash table for the one 8 bytes ahead, the root that it has for the one `ROOT` bytes ahead, and what is +/// below the root for the one `BELOW` bytes ahead. +#[inline(always)] +pub(crate) fn bt_look_ahead( + hash_table: &[u32], + bt: &Nodes, + input: &TreeInput<'_>, + hasher: Hasher, + ip: u32, +) { + let near = input.near; + if !input.far { + return; + } + let Some(bytes) = near.chunk16(ip) else { + return; + }; + let bytes = u128::from_le_bytes(*bytes); + let hash_of = |ahead: u32| hasher.hash((bytes >> (8 * ahead)) as u64); + prefetch_read(hash_table.as_ptr().wrapping_add(hash_of(8)), Locality::L1); + let root = table_get(hash_table, hash_of(ROOT)); + bt_prefetch(bt, root); + near.prefetch(root.wrapping_sub(input.dict_limit)); + for child in bt_children(bt, table_get(hash_table, hash_of(BELOW))) { + bt_prefetch(bt, child); + near.prefetch(child.wrapping_sub(input.dict_limit)); + } +} + +/// Goes down the tree whose root is `root` for the position `ip` of `input.near`, which `descent` is for and which +/// becomes the root. `visit` is called with each position that is met on the way, which are all at least `low` and at +/// most `nb_compares`, and what it has in common with `ip`, and returns whether to go on. Returns how many more positions +/// could have been compared. +#[inline(always)] +pub(crate) fn bt_walk( + bt: &mut Nodes, + input: &TreeInput<'_>, + ip: u32, + root: u32, + low: u32, + mut nb_compares: u32, + mut descent: Descent, + mut visit: impl FnMut(u32, Common) -> Result, +) -> Result { + if bt.is_empty() { + return Err(BUG); + } + let near = input.near; + let dict_limit = input.dict_limit; + let mut match_index = root; + while nb_compares != 0 && match_index >= low { + let children = bt_children(bt, match_index); + let from = descent.common(); + // Whichever way it goes, the next step does not wait as long for memory. + if input.far { + for child in children { + bt_prefetch(bt, child); + near.prefetch(child.wrapping_sub(dict_limit).wrapping_add(from)); + } + } + let common = match near.compare_chunks(ip, match_index.wrapping_sub(dict_limit), from) { + Ok(common) => common, + Err(from) => input.compare_rest(ip, match_index, from)?, + }; + if !visit(match_index, common)? { + break; + } + let match_is_smaller = common.match_is_smaller.ok_or(BUG)?; + let Some(next) = descent.step(bt, match_index, children, common.len, match_is_smaller) + else { + break; + }; + match_index = next; + nb_compares -= 1; + } + descent.end(bt); + Ok(nb_compares) +} + +/// The tree of a dictionary that a search goes on in. It is only read. +pub(crate) struct DictTree<'a> { + dms: DictMatchState<'a>, + bt: &'a Nodes, + /// The content of the dictionary. A position in it is an index minus `low_limit`. + content: Near<'a>, + low_limit: u32, + high_limit: u32, + bt_low: u32, + /// What to add to an index of the dictionary for an index of the window. + pub(crate) index_delta: u32, +} + +impl<'a> DictTree<'a> { + /// `window_low` is the index of the window that the end of the dictionary is at. + pub(crate) fn new(dms: DictMatchState<'a>, window_low: u32) -> Result { + let (bt, _) = dms.ms.chain_table.as_chunks(); + if bt.len() != 1usize.wrapping_shl(dms.ms.c_params.chain_log.wrapping_sub(1)) { + return Err(BUG); + } + let bt_mask = bt.len().wrapping_sub(1) as u32; + let low_limit = dms.ms.window.low_limit; + let high_limit = dms.end.wrapping_sub(dms.base) as u32; + Ok(Self { + dms, + bt, + content: Near::new(dms.buf.sub(dms.pos_of(low_limit)..dms.end)?)?, + low_limit, + high_limit, + bt_low: if bt_mask < high_limit.wrapping_sub(low_limit) { + high_limit.wrapping_sub(bt_mask) + } else { + low_limit + }, + index_delta: window_low.wrapping_sub(high_limit), + }) + } + + /// [`Near::compare`] for a match in the dictionary, for what [`Near::compare_chunks_with`] leaves. A match that + /// reaches the end of the dictionary goes on at the start of the current segment. + #[cold] + #[inline(never)] + fn compare_rest( + &self, + input: &TreeInput<'_>, + ip: u32, + dict_match_index: u32, + from: u32, + ) -> Result { + let dms = &self.dms; + let match_pos = dms.pos_of(dict_match_index.wrapping_add(from)); + let len = from + input.count_2segments(ip + from, dms.buf, match_pos, dms.end)?; + let Some(byte) = input.near.read8(ip.wrapping_add(len)) else { + return Ok(Common::up_to_the_end(len)); + }; + let differs = dict_match_index.wrapping_add(len); + let match_byte = if differs >= self.high_limit { + let index = differs.wrapping_add(self.index_delta); + let pos = index.wrapping_sub(input.dict_limit); + input.near.read8(pos).ok_or(BUG)? + } else { + dms.buf.at(dms.pos_of(differs))? + }; + Ok(Common::followed_by(len, match_byte, byte)) + } + + /// Goes down the tree for the position `ip` of `input.near`, whose hash for the table of the dictionary is `hash`. + /// `visit` is called with each position that is met on the way, at most `nb_compares` of them, and what it has in + /// common with `ip`, and returns whether to go on. + #[inline(always)] + pub(crate) fn walk( + &self, + input: &TreeInput<'_>, + ip: u32, + hash: usize, + mut nb_compares: u32, + mut visit: impl FnMut(u32, Common) -> Result, + ) -> Result<()> { + let mut dict_match_index = self.dms.ms.hash_table.at(hash)?; + let mut common_smaller = 0u32; + let mut common_larger = 0u32; + while nb_compares != 0 && dict_match_index > self.low_limit { + let [smaller_child, larger_child] = bt_children(self.bt, dict_match_index); + let from = common_smaller.min(common_larger); + let match_pos = dict_match_index.wrapping_sub(self.low_limit); + let chunks = input + .near + .compare_chunks_with(ip, self.content, match_pos, from); + let common = match chunks { + Ok(common) => common, + Err(from) => self.compare_rest(input, ip, dict_match_index, from)?, + }; + if !visit(dict_match_index, common)? || dict_match_index <= self.bt_low { + break; + } + if common.match_is_smaller.ok_or(BUG)? { + common_smaller = common.len; + dict_match_index = larger_child; + } else { + common_larger = common.len; + dict_match_index = smaller_child; + } + nb_compares -= 1; + } + Ok(()) + } +} + /// `table[hash]`. The size of `table` is a power of 2 that `hash` is less than, which the mask tells the compiler. #[inline(always)] pub(crate) fn table_get(table: &[u32], hash: usize) -> u32 { @@ -835,10 +1331,45 @@ pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize } -/// `ZSTD_hash3Ptr`: `h` is from 1 to 32, as for all these. +/// `ZSTD_hash3Ptr` of the position that the 4 bytes `bytes` were read at. `h` is from 1 to 32, as for all these. #[inline(always)] -pub(crate) fn hash3_ptr(buf: &[u8], pos: usize, h: u32) -> Result { - Ok(hash3(read_le32(buf, pos)?, h, 0) as usize) +pub(crate) const fn hash3_of(bytes: u32, h: u32) -> usize { + hash3(bytes, h, 0) as usize +} + +/// [`hash_of`], when `mls` is not a constant. +#[derive(Clone, Copy)] +pub(crate) struct Hasher { + /// Takes the bytes that do not count out, on the left. + factor: u64, + shift: u32, +} + +impl Hasher { + pub(crate) const fn new(h_bits: u32, mls: u32) -> Self { + let factor = match mls { + 5 => PRIME_5_BYTES << (64 - 40), + 6 => PRIME_6_BYTES << (64 - 48), + 7 => PRIME_7_BYTES << (64 - 56), + 8 => PRIME_8_BYTES, + _ => (PRIME_4_BYTES as u64) << 32, + }; + Self { factor, shift: 0 }.with_log(h_bits) + } + + /// The same for a table of another size. + pub(crate) const fn with_log(self, h_bits: u32) -> Self { + Self { + shift: 64u32.wrapping_sub(h_bits) & 63, + ..self + } + } + + /// [`hash_ptr`] of the position that the 8 bytes `bytes` were read at. + #[inline(always)] + pub(crate) const fn hash(self, bytes: u64) -> usize { + (bytes.wrapping_mul(self.factor) >> self.shift) as usize + } } /// [`hash_ptr`] of the position that the 8 bytes `bytes` were read at. @@ -1162,22 +1693,35 @@ pub(crate) struct RawSeqStore { pub(crate) capacity: usize, } -/// `ZSTD_optimal_t` -#[derive(Clone, Copy, Default)] -pub(crate) struct Optimal { +/// `ZSTD_optimal_t`, one array per member: how to get to each of the positions that the optimal parser looks at, by a +/// match followed by literals. +#[derive(Default)] +pub(crate) struct PriceTable { /// The cost of getting to this position. - pub(crate) price: i32, - pub(crate) off: u32, - pub(crate) mlen: u32, - pub(crate) litlen: u32, - pub(crate) rep: [u32; ZSTD_REP_NUM], -} - -/// `ZSTD_OptPrice_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum OptPrice { - Dynamic, - Predef, + pub(crate) price: Vec, + pub(crate) off: Vec, + /// `mlen` and `litlen`. + pub(crate) lens: Vec, + pub(crate) rep: Vec<[u32; ZSTD_REP_NUM]>, + /// The price of each length of match. + pub(crate) ml_price: Vec, +} + +impl PriceTable { + /// Makes room for `len` positions. + pub(crate) fn resize(&mut self, len: usize) -> Result<()> { + if self.price.len() != len { + *self = Self::default(); + *self = Self { + price: try_zeroed(len)?, + off: try_zeroed(len)?, + lens: try_zeroed(len)?, + rep: try_zeroed(len)?, + ml_price: try_zeroed(len)?, + }; + } + Ok(()) + } } /// `optState_t`. `symbolCosts` is in [`BlockSrc`]. @@ -1187,16 +1731,11 @@ pub(crate) struct OptState { pub(crate) match_length_freq: Vec, pub(crate) off_code_freq: Vec, pub(crate) match_table: Vec, - pub(crate) price_table: Vec, + pub(crate) price_table: PriceTable, pub(crate) lit_sum: u32, pub(crate) lit_length_sum: u32, pub(crate) match_length_sum: u32, pub(crate) off_code_sum: u32, - pub(crate) lit_sum_base_price: u32, - pub(crate) lit_length_sum_base_price: u32, - pub(crate) match_length_sum_base_price: u32, - pub(crate) off_code_sum_base_price: u32, - pub(crate) price_type: OptPrice, pub(crate) literal_compression_mode: ParamSwitch, } @@ -1208,16 +1747,17 @@ impl OptState { match_length_freq: Vec::new(), off_code_freq: Vec::new(), match_table: Vec::new(), - price_table: Vec::new(), + price_table: PriceTable { + price: Vec::new(), + off: Vec::new(), + lens: Vec::new(), + rep: Vec::new(), + ml_price: Vec::new(), + }, lit_sum: 0, lit_length_sum: 0, match_length_sum: 0, off_code_sum: 0, - lit_sum_base_price: 0, - lit_length_sum_base_price: 0, - match_length_sum_base_price: 0, - off_code_sum_base_price: 0, - price_type: OptPrice::Dynamic, literal_compression_mode: ParamSwitch::Auto, } } diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 0696bd6dc8ff..20abe131b738 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -3,10 +3,12 @@ use core::hint::{Locality, prefetch_read}; use super::internal::{ - BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Near, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, - count, count_2segments, hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, - offbase_to_offset, offset_to_offbase, + BlockSrc, Descent, DictMatchState, DictMode, DictTree, Hasher, K_SEARCH_STRENGTH, MatchState, + Near, Nodes, Prefix, REPCODE1_TO_OFFBASE, SeqStore, Strategy, TreeInput, + ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, bt_children, bt_look_ahead, bt_low, + bt_nodes, bt_set_children, bt_walk, count, count_2segments, hash_ptr, hash_ptr_salted, + index_overlap_check, offbase_is_offset, offbase_to_offset, offset_to_offbase, table_get, + table_put, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -33,411 +35,298 @@ const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { // ── binary tree search ──────────────────────────────────────────────────── -/// `ZSTD_updateDUBT`: adds the positions from `ms.next_to_update` up to `ip` to the trees, as unsorted chains. -#[inline(always)] -fn update_dubt(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, mls: u32) -> Result<()> { - let hash_log = ms.c_params.hash_log; - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let bt = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let match_index = core::mem::replace(hash_table.at_mut(h)?, idx); - *bt.array_at_mut(2 * (idx & bt_mask) as usize)? = [match_index, ZSTD_DUBT_UNSORTED_MARK]; - } - ms.next_to_update = target; - Ok(()) -} - -/// What [`insert_dubt1`] works out about the position it sorts. -struct Unsorted<'a> { - bt_mask: u32, - /// Where its two slots are in the table. - slot: usize, - /// The next sorted candidate, which is in the first slot. The other one has the previous unsorted candidate. - next_sorted: u32, - dict_limit: u32, - dict_end: usize, - prefix_start: usize, - /// The buffer that the position is in, where it is in it, and where its segment ends. - ip_buf: &'a [u8], - ip: usize, - iend: usize, +/// What a search in the binary trees works on. +struct Dubt<'a> { + hash_table: &'a mut [u32], + bt: &'a mut Nodes, + input: TreeInput<'a>, + hasher: Hasher, + /// `window.lowLimit` + low_limit: u32, + max_distance: u32, + /// `ZSTD_getLowestMatchIndex` of `curr`. window_low: u32, + nb_compares: u32, + next_to_update: u32, + /// The position that is searched for, which is `ip` in `input.near`. + curr: u32, + ip: u32, } -impl<'a> Unsorted<'a> { - fn new(ms: &MatchState, src: &BlockSrc<'a>, curr: u32, input_end: usize) -> Result { - let bt_mask = (1u32 << (ms.c_params.chain_log - 1)) - 1; - let slot = 2 * (curr & bt_mask) as usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let (ip_buf, ip, iend) = if curr >= dict_limit { - (src.buf, src.pos_of(curr), input_end) - } else { - (src.dict_buf, src.dict_pos_of(curr), dict_end) +impl<'a> Dubt<'a> { + fn new( + ms: &'a mut MatchState, + src: &BlockSrc<'a>, + ip: usize, + iend: usize, + mls: u32, + ) -> Result { + let c_params = ms.c_params; + if ms.hash_table.len() != 1usize.wrapping_shl(c_params.hash_log) + || ms.chain_table.len() != 1usize.wrapping_shl(c_params.chain_log) + { + return Err(BUG); + } + let dict = Prefix { + buf: src.dict_buf, + base: src.dict_base, }; - let window_valid = ms.window.low_limit; - let max_distance = 1u32 << ms.c_params.window_log; + let curr = src.index_of(ip); + let dict_limit = ms.window.dict_limit; Ok(Self { - bt_mask, - slot, - next_sorted: ms.chain_table.at(slot)?, - dict_limit, - dict_end, - prefix_start: src.pos_of(dict_limit), - ip_buf, - ip, - iend, - window_low: if curr.wrapping_sub(window_valid) > max_distance { - curr - max_distance - } else { - window_valid - }, + window_low: ms.get_lowest_match_index(curr, c_params.window_log), + hash_table: ms.hash_table.as_mut_slice(), + bt: bt_nodes(&mut ms.chain_table), + input: TreeInput::new(src.prefix(), dict, dict_limit, iend)?, + hasher: Hasher::new(c_params.hash_log, mls), + low_limit: ms.window.low_limit, + max_distance: 1u32.wrapping_shl(c_params.window_log), + nb_compares: 1u32.wrapping_shl(c_params.search_log), + next_to_update: ms.next_to_update, + curr, + ip: curr.wrapping_sub(dict_limit), }) } -} - -/// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. -fn insert_dubt1( - ms: &mut MatchState, - src: &BlockSrc<'_>, - curr: u32, - input_end: usize, - mut nb_compares: u32, - bt_low: u32, -) -> Result<()> { - let at = Unsorted::new(ms, src, curr, input_end)?; - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = at.slot; - let mut larger_ptr = smaller_ptr + 1; - let mut match_index = at.next_sorted; - - while nb_compares != 0 && match_index > at.window_low { - let next_ptr = 2 * (match_index & at.bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let in_prefix = - DICT_MODE != EXT_DICT || match_index as usize + match_length >= at.dict_limit as usize; - // In `src.buf`, `match_pos` may be before the start: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = if in_prefix { - (src.buf, src.pos_of(match_index)) - } else { - (src.dict_buf, src.dict_pos_of(match_index)) - }; - if in_prefix || curr < at.dict_limit { - // Both are in the same segment. - match_length += count( - at.ip_buf, - at.ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - at.iend, - )?; - } else { - match_length += count_2segments( - at.ip_buf, - at.ip + match_length, - match_buf, - match_pos + match_length, - at.iend, - at.dict_end, - at.prefix_start, - )?; - if match_index as usize + match_length >= at.dict_limit as usize { - (match_buf, match_pos) = (src.buf, src.pos_of(match_index)); + /// `ZSTD_updateDUBT`: adds the positions from `next_to_update` up to `curr` to the trees, as unsorted chains. + #[inline(always)] + fn update(&mut self) -> Result<()> { + let near = self.input.near; + for idx in self.next_to_update..self.curr { + let bytes = near.read64(idx.wrapping_sub(self.input.dict_limit)); + let h = self.hasher.hash(bytes.ok_or(BUG)?); + let match_index = table_get(self.hash_table, h); + table_put(self.hash_table, h, idx); + bt_set_children(self.bt, idx, [match_index, ZSTD_DUBT_UNSORTED_MARK]); + let ahead = idx.wrapping_sub(self.input.dict_limit).wrapping_add(8); + if self.input.far + && let Some(bytes) = near.read64(ahead) + { + prefetch_l1(self.hash_table, self.hasher.hash(bytes)); } } + Ok(()) + } - if at.ip + match_length == at.iend { - // Equal: no way to know which is smaller. Dropped, to keep the tree consistent. - break; - } - - if match_buf.at(match_pos.wrapping_add(match_length))? - < at.ip_buf.at(at.ip + match_length)? - { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; + /// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. + #[inline(always)] + fn insert_dubt1(&mut self, curr: u32, nb_compares: u32, bt_low: u32) -> Result<()> { + let window_low = if curr.wrapping_sub(self.low_limit) > self.max_distance { + curr.wrapping_sub(self.max_distance) } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; + self.low_limit + }; + if curr < self.input.dict_limit { + return self.insert_dubt1_in_dict(curr, nb_compares, bt_low, window_low); } - nb_compares -= 1; + sort_dubt1(self.bt, &self.input, curr, nb_compares, bt_low, window_low) } - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0) -} - -/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `best_length`. -fn dubt_find_better_dict_match( - ms: &MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mut best_length: usize, - mut nb_compares: u32, - mls: u32, -) -> Result { - let dms = src.dms.ok_or(BUG)?; - let h = hash_ptr(src.buf, ip, dms.ms.c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(h)?; - - let prefix_start = src.pos_of(ms.window.dict_limit); - let curr = src.index_of(ip); - let dict_high_limit = dms_end_index(&dms); - let dict_low_limit = dms.ms.window.low_limit; - let dict_index_delta = ms.window.low_limit.wrapping_sub(dict_high_limit); - - let dict_bt = dms.ms.chain_table.as_slice(); - let bt_log = dms.ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) { - dict_low_limit - } else { - dict_high_limit - bt_mask - }; - - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - - while nb_compares != 0 && dict_match_index > dict_low_limit { - let next_ptr = 2 * (dict_match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - let match_index = dict_match_index.wrapping_add(dict_index_delta); - match_length += count_2segments( - src.buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - iend, - dms.end, - prefix_start, - )?; + /// [`Self::insert_dubt1`] of a position of the older segment, which all that is before it is in too. + #[cold] + #[inline(never)] + fn insert_dubt1_in_dict( + &mut self, + curr: u32, + nb_compares: u32, + bt_low: u32, + window_low: u32, + ) -> Result<()> { + let no_dict = Prefix { buf: &[], base: 0 }; + let dict = self.input.dict; + let dict_end = dict.pos_of(self.input.dict_limit); + let input = TreeInput::new(dict, no_dict, self.low_limit, dict_end)?; + sort_dubt1(self.bt, &input, curr, nb_compares, bt_low, window_low) + } - if match_length > best_length { - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(offset_ptr.wrapping_add(1)) as i32 - { - best_length = match_length; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // No way to know which is smaller. + /// `ZSTD_DUBT_findBestMatch`, but for the dictionary. Returns the length of the best match, or 0, and how many more + /// positions could have been compared if the dictionary is to be searched. + #[inline(always)] + fn find_best_match(&mut self, off_base_ptr: &mut u32) -> Result { + let curr = self.curr; + let h = self + .hasher + .hash(self.input.near.read64(self.ip).ok_or(BUG)?); + let mut match_index = table_get(self.hash_table, h); + bt_look_ahead::<2, 1>(self.hash_table, self.bt, &self.input, self.hasher, self.ip); + + let unsort_limit = bt_low(self.bt, curr).max(self.window_low); + let mut nb_candidates = self.nb_compares; + let mut previous_candidate = 0u32; + + // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. + let mut slots = bt_children(self.bt, match_index); + while let [next_candidate, ZSTD_DUBT_UNSORTED_MARK] = slots + && match_index > unsort_limit + { + if nb_candidates <= 1 { + // The last candidate is still unsorted: cut the list there. + bt_set_children(self.bt, match_index, [0, 0]); break; } + bt_set_children(self.bt, match_index, [next_candidate, previous_candidate]); + previous_candidate = match_index; + match_index = next_candidate; + slots = bt_children(self.bt, match_index); + nb_candidates -= 1; } - if dict_match_index <= bt_low { - // Beyond the tree. - break; - } - let match_byte = if dict_match_index as usize + match_length >= dict_high_limit as usize { - let match_pos = src.pos_of(match_index); - src.buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? - }; - if match_byte < src.buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dict_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dict_bt.at(next_ptr)?; + // Sort the candidates, the oldest first. + match_index = previous_candidate; + while match_index != 0 { + let [_, next_candidate_idx] = bt_children(self.bt, match_index); + self.insert_dubt1(match_index, nb_candidates, unsort_limit)?; + match_index = next_candidate_idx; + nb_candidates += 1; } - nb_compares -= 1; + + // Find the longest match. + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut best_length = 0u32; + let mut search_dict = true; + match_index = table_get(self.hash_table, h); + table_put(self.hash_table, h, curr); + let nb_compares = bt_walk( + self.bt, + &self.input, + self.ip, + match_index, + self.window_low.saturating_add(1), + self.nb_compares, + Descent::new(self.bt, curr), + |match_index, common| { + if common.len > best_length { + if common.len > match_end_idx.wrapping_sub(match_index) { + match_end_idx = match_index.wrapping_add(common.len); + } + if 4 * (common.len - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(*off_base_ptr) as i32 + { + best_length = common.len; + *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if common.match_is_smaller.is_none() { + // Equal: no way to know which is smaller. Nothing in the dictionary can be longer. + search_dict = false; + return Ok(false); + } + } + Ok(true) + }, + )?; + + // Skip repetitive patterns. + self.next_to_update = match_end_idx.wrapping_sub(8); + Ok(DubtFound { + best_length, + nb_compares: if search_dict { nb_compares } else { 0 }, + }) } +} - Ok(best_length) +impl Dubt<'_> { + /// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `found`. Returns + /// the length of the best match. + #[inline(never)] + fn find_better_dict_match( + &self, + dms: DictMatchState<'_>, + found: &DubtFound, + offset_ptr: &mut u32, + ) -> Result { + let curr = self.curr; + let tree = DictTree::new(dms, self.low_limit)?; + let bytes = self.input.near.read64(self.ip).ok_or(BUG)?; + let hash = self.hasher.with_log(dms.ms.c_params.hash_log).hash(bytes); + let mut best_length = found.best_length; + tree.walk( + &self.input, + self.ip, + hash, + found.nb_compares, + |dict_match_index, common| { + if common.len > best_length { + let match_index = dict_match_index.wrapping_add(tree.index_delta); + if 4 * (common.len - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(offset_ptr.wrapping_add(1)) as i32 + { + best_length = common.len; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + // At the end, there is no way to know which is smaller. + return Ok(common.match_is_smaller.is_some()); + } + Ok(true) + }, + )?; + Ok(best_length) + } } -/// `ZSTD_DUBT_findBestMatch` +/// The body of `ZSTD_insertDUBT1`, for a position of `input.near`. #[inline(always)] -fn dubt_find_best_match( +fn sort_dubt1( + bt: &mut Nodes, + input: &TreeInput<'_>, + curr: u32, + nb_compares: u32, + bt_low: u32, + window_low: u32, +) -> Result<()> { + // The next sorted candidate is in the first slot. The other one has the previous unsorted candidate. + let [next_sorted, _] = bt_children(bt, curr); + bt_walk( + bt, + input, + curr.wrapping_sub(input.dict_limit), + next_sorted, + window_low.saturating_add(1), + nb_compares, + Descent::down_to(curr, bt_low), + // If they are equal, there is no way to know which is smaller. Dropped, to keep the tree consistent. + |_, common| Ok(common.match_is_smaller.is_some()), + )?; + Ok(()) +} + +/// What [`Dubt::find_best_match`] returns. +struct DubtFound { + best_length: u32, + nb_compares: u32, +} + +/// `ZSTD_BtFindBestMatch`, when `mls` and the mode are not constants. +#[inline(never)] +fn bt_search( ms: &mut MatchState, src: &BlockSrc<'_>, ip: usize, - iend: usize, + i_limit: usize, off_base_ptr: &mut u32, mls: u32, + search_dms: bool, ) -> Result { - let h = hash_ptr(src.buf, ip, ms.c_params.hash_log, mls)?; - let mut match_index = ms.hash_table.at(h)?; - - let curr = src.index_of(ip); - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - - let bt_log = ms.c_params.chain_log - 1; - let bt_mask = (1u32 << bt_log) - 1; - let bt_low = curr.saturating_sub(bt_mask); - let unsort_limit = bt_low.max(window_low); - let mut nb_compares = 1u32 << ms.c_params.search_log; - let mut nb_candidates = nb_compares; - let mut previous_candidate = 0u32; - - // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. - let bt = ms.chain_table.as_mut_slice(); - let mut next_candidate = 2 * (match_index & bt_mask) as usize; - while match_index > unsort_limit - && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK - && nb_candidates > 1 - { - bt.put(next_candidate + 1, previous_candidate)?; - previous_candidate = match_index; - match_index = bt.at(next_candidate)?; - next_candidate = 2 * (match_index & bt_mask) as usize; - nb_candidates -= 1; - } - - // If the last candidate is still unsorted, cut the list there. - if match_index > unsort_limit && bt.at(next_candidate + 1)? == ZSTD_DUBT_UNSORTED_MARK { - *bt.array_at_mut(next_candidate)? = [0, 0]; - } - - // Sort the candidates, the oldest first. - match_index = previous_candidate; - while match_index != 0 { - let next_candidate_idx = ms - .chain_table - .at(2 * (match_index & bt_mask) as usize + 1)?; - insert_dubt1::(ms, src, match_index, iend, nb_candidates, unsort_limit)?; - match_index = next_candidate_idx; - nb_candidates += 1; - } - - // Find the longest match. - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt = ms.chain_table.as_mut_slice(); - let mut smaller_ptr = 2 * (curr & bt_mask) as usize; - let mut larger_ptr = smaller_ptr + 1; - let mut match_end_idx = curr + 8 + 1; - let mut best_length = 0usize; - - match_index = core::mem::replace(ms.hash_table.at_mut(h)?, curr); - - while nb_compares != 0 && match_index > window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - // `match_pos` may be before the start of the buffer: only its sum with `match_length` is a position. - let (mut match_buf, mut match_pos) = (src.buf, src.pos_of(match_index)); - - if DICT_MODE != EXT_DICT || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - src.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - let dict_match_pos = src.dict_pos_of(match_index); - match_length += count_2segments( - src.buf, - ip + match_length, - src.dict_buf, - dict_match_pos + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - (match_buf, match_pos) = (src.dict_buf, dict_match_pos); - } - } - - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index + match_length as u32; - } - if 4 * (match_length - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(*off_base_ptr) as i32 - { - best_length = match_length; - *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - if ip + match_length == iend { - // Equal: no way to know which is smaller. - if DICT_MODE == DICT_MATCH_STATE { - // Nothing in the dictionary can be longer. - nb_compares = 0; - } - break; - } - } - - if match_buf.at(match_pos.wrapping_add(match_length))? < src.buf.at(ip + match_length)? { - bt.put(smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. This slot keeps its value: only the other one is cleared. - smaller_ptr = larger_ptr; - break; - } - smaller_ptr = next_ptr + 1; - match_index = bt.at(next_ptr + 1)?; - } else { - bt.put(larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = smaller_ptr; - break; - } - larger_ptr = next_ptr; - match_index = bt.at(next_ptr)?; - } - nb_compares -= 1; - } - - bt.put(smaller_ptr, 0)?; - bt.put(larger_ptr, 0)?; - - if DICT_MODE == DICT_MATCH_STATE && nb_compares != 0 { - best_length = dubt_find_better_dict_match( - ms, - src, - ip, - iend, - off_base_ptr, - best_length, - nb_compares, - mls, - )?; + if ip < src.pos_of(ms.next_to_update) { + // In an area that is skipped. + return Ok(0); } - - // Skip repetitive patterns. - ms.next_to_update = match_end_idx - 8; - Ok(best_length) + let mut dubt = Dubt::new(ms, src, ip, i_limit, mls)?; + dubt.update()?; + let found = dubt.find_best_match(off_base_ptr)?; + let best_length = if search_dms && found.nb_compares != 0 { + dubt.find_better_dict_match(src.dms.ok_or(BUG)?, &found, off_base_ptr)? + } else { + found.best_length + }; + ms.next_to_update = dubt.next_to_update; + Ok(best_length as usize) } /// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. /// `off_base_ptr` is written to if there is one. -#[inline(never)] +#[inline(always)] fn bt_find_best_match( ms: &mut MatchState, src: &BlockSrc<'_>, @@ -445,12 +334,8 @@ fn bt_find_best_match( i_limit: usize, off_base_ptr: &mut u32, ) -> Result { - if ip < src.pos_of(ms.next_to_update) { - // In an area that is skipped. - return Ok(0); - } - update_dubt(ms, src.prefix(), ip, MLS)?; - dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) + let search_dms = DICT_MODE == DICT_MATCH_STATE; + bt_search(ms, src, ip, i_limit, off_base_ptr, MLS, search_dms) } // ── hash chains ─────────────────────────────────────────────────────────── diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index 5f5a5e5a5151..b03d47367eae 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -1,14 +1,16 @@ //! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. +use core::hint::cold_path; use core::mem; use super::hist; use super::huf::HufRepeat; use super::internal::{ - BlockSrc, DictMode, EntropyCTables, Match, MatchState, OptPrice, OptState, Optimal, - ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, ZSTD_OPT_NUM, count, - count_2segments, hash_ptr, hash3_ptr, index_overlap_check, ll_code, ml_code, new_rep, - offset_to_offbase, + BlockSrc, Common, Descent, DictMatchState, DictMode, DictTree, EntropyCTables, Hasher, Match, + MatchState, Near, Nodes, OptState, ParamSwitch, Prefix, PriceTable, RawSeq, RawSeqStore, + SeqStore, Strategy, TreeInput, ZSTD_OPT_NUM, ZSTD_OPT_SIZE, bt_look_ahead, bt_nodes, bt_walk, + hash3_of, index_overlap_check, ll_code, ml_code, new_rep, offset_to_offbase, table_get, + table_put, }; use crate::common::{ LL_BITS, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, @@ -63,17 +65,6 @@ fn compressed_literals(opt_ptr: &OptState) -> bool { opt_ptr.literal_compression_mode != ParamSwitch::Disable } -/// `ZSTD_setBasePrices` -#[inline(always)] -fn set_base_prices(opt_ptr: &mut OptState, opt_level: i32) { - if compressed_literals(opt_ptr) { - opt_ptr.lit_sum_base_price = weight(opt_ptr.lit_sum, opt_level); - } - opt_ptr.lit_length_sum_base_price = weight(opt_ptr.lit_length_sum, opt_level); - opt_ptr.match_length_sum_base_price = weight(opt_ptr.match_length_sum, opt_level); - opt_ptr.off_code_sum_base_price = weight(opt_ptr.off_code_sum, opt_level); -} - /// `sum_u32` fn sum_u32(table: &[u32]) -> u32 { table.iter().fold(0, |total, &elt| total.wrapping_add(elt)) @@ -143,12 +134,7 @@ fn freqs_from_bit_costs( /// `ZSTD_rescaleFreqs`: for the first block, which no statistics tell, starts them from the tables of the dictionary /// if there is one, else from the literals of `src` and from fixed values for the sequences. For the next ones, scales /// down what the blocks before have accumulated. -fn rescale_freqs( - opt_ptr: &mut OptState, - symbol_costs: &EntropyCTables, - src: &[u8], - opt_level: i32, -) -> Result<()> { +fn rescale_freqs(opt_ptr: &mut OptState, symbol_costs: &EntropyCTables, src: &[u8]) -> Result<()> { #[rustfmt::skip] const BASE_LL_FREQS: [u32; MAX_LL as usize + 1] = [ 4, 2, 1, 1, 1, 1, 1, 1, @@ -166,17 +152,12 @@ fn rescale_freqs( ]; let compressed_literals = compressed_literals(opt_ptr); - opt_ptr.price_type = OptPrice::Dynamic; + // `zop_predef`, the fixed prices for a first block of at most `ZSTD_PREDEF_THRESHOLD` bytes, are of no use: the last + // 8 bytes of a block are not searched for matches. if opt_ptr.lit_length_sum == 0 { - if src.len() <= ZSTD_PREDEF_THRESHOLD { - opt_ptr.price_type = OptPrice::Predef; - } - if symbol_costs.huf.repeat_mode == HufRepeat::Valid { // A Huffman table that has all the values is presumed to come from a dictionary. - opt_ptr.price_type = OptPrice::Dynamic; - if compressed_literals { let huf = &symbol_costs.huf.c_table; opt_ptr.lit_sum = @@ -230,866 +211,671 @@ fn rescale_freqs( opt_ptr.off_code_sum = scale_stats(&mut opt_ptr.off_code_freq, MAX_OFF, 11)?; } - set_base_prices(opt_ptr, opt_level); Ok(()) } -/// `ZSTD_rawLiteralsCost`: the price of `literals`, without that of their length. -#[inline(always)] -fn raw_literals_cost(literals: &[u8], opt_ptr: &OptState, opt_level: i32) -> Result { - let lit_length = literals.len() as u32; - if lit_length == 0 { - return Ok(0); - } - - if !compressed_literals(opt_ptr) { - // 8 bits per literal. - return Ok((lit_length << 3).wrapping_mul(BITCOST_MULTIPLIER)); - } - - if opt_ptr.price_type == OptPrice::Predef { - // 6 bits per literal. - return Ok(lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER)); - } - - let mut price = opt_ptr.lit_sum_base_price.wrapping_mul(lit_length); - let lit_price_max = opt_ptr.lit_sum_base_price.wrapping_sub(BITCOST_MULTIPLIER); - for &literal in literals { - let lit_price = weight(opt_ptr.lit_freq.at(usize::from(literal))?, opt_level); - price = price.wrapping_sub(lit_price.min(lit_price_max)); - } - Ok(price) -} - -/// `ZSTD_litLengthPrice`: the price of the symbol for a length of literals. -#[inline(always)] -fn lit_length_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - if opt_ptr.price_type == OptPrice::Predef { - return Ok(weight(lit_length, opt_level)); - } - - // The format has no code for the length of a whole block of the largest size, which is then all literals. It is - // made to cost 1 bit more than the length before it. - let (lit_length, extra) = if lit_length == ZSTD_BLOCKSIZE_MAX as u32 { - (ZSTD_BLOCKSIZE_MAX as u32 - 1, BITCOST_MULTIPLIER) - } else { - (lit_length, 0) - }; - - let ll_code = ll_code(lit_length) as usize; - Ok(extra - .wrapping_add(u32::from(LL_BITS.at(ll_code)?) * BITCOST_MULTIPLIER) - .wrapping_add(opt_ptr.lit_length_sum_base_price) - .wrapping_sub(weight(opt_ptr.lit_length_freq.at(ll_code)?, opt_level))) -} - -/// `ZSTD_getMatchPrice`, the price of the offset and of the match length of a sequence, in two parts to add up: all the -/// lengths of a match are priced, with the same offset. -/// -/// This is the part for the offset. `off_base` is as in [`SeqStore::store_seq`]. Below level 2, it favours small -/// offsets, which are faster to decompress. -#[inline(always)] -fn offset_price(off_base: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - let off_code = highbit32(off_base); - if opt_ptr.price_type == OptPrice::Predef { - return Ok((16 + off_code) * BITCOST_MULTIPLIER); - } - - let off_code_weight = weight(opt_ptr.off_code_freq.at(off_code as usize)?, opt_level); - let mut price = (off_code * BITCOST_MULTIPLIER).wrapping_add( - opt_ptr - .off_code_sum_base_price - .wrapping_sub(off_code_weight), - ); - if opt_level < 2 && off_code >= 20 { - price = price.wrapping_add((off_code - 19) * 2 * BITCOST_MULTIPLIER); - } - // Matches cost a bit more, to have fewer sequences, which is faster to decompress. - Ok(price.wrapping_add(BITCOST_MULTIPLIER / 5)) -} - -/// The part of `ZSTD_getMatchPrice` for the length. It keeps the price of the last code, which many lengths share. -struct MatchLengthPrice<'a> { - opt_ptr: &'a OptState, +/// The least length, minus [`MINMATCH`], that each code of match length stands for, and what follows the last one. +#[rustfmt::skip] +const ML_CODE_START: [u32; MAX_ML as usize + 2] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 34, 36, 38, 40, 44, 48, 56, 64, 80, 96, + 1 << 7, 1 << 8, 1 << 9, 1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16, 1 << 17, +]; + +/// The same for the codes of literal length. +#[rustfmt::skip] +const LL_CODE_START: [u32; MAX_LL as usize + 2] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, 48, + 1 << 6, 1 << 7, 1 << 8, 1 << 9, 1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16, 1 << 17, +]; + +/// The lengths of literals below this have their `LL_INCPRICE` in a table. +const LL_INC_LENGTHS: usize = 64; + +/// The statistics, and the prices that they give to the symbols. A price is only worked out when the frequency of its +/// symbol changes. What depends on the sum of the frequencies, which changes more often, is kept apart. +struct Prices<'a> { opt_level: i32, - last: Option, + compressed_literals: bool, + lit_freq: &'a mut [u32; MAX_LIT as usize + 1], + lit_length_freq: &'a mut [u32; MAX_LL as usize + 1], + match_length_freq: &'a mut [u32; MAX_ML as usize + 1], + off_code_freq: &'a mut [u32; MAX_OFF as usize + 1], + sums: Sums, + /// `litSumBasePrice` + lit_base: u32, + /// `litLengthSumBasePrice` + lit_length_base: u32, + /// `matchLengthSumBasePrice` plus `offCodeSumBasePrice` + match_base: u32, + /// `WEIGHT(litFreq[literal])` + lit_weight: [u32; MAX_LIT as usize + 1], + /// By code. + lit_length: [u32; MAX_LL as usize + 1], + /// `LL_INCPRICE` of the short lengths. + lit_length_inc: [u32; LL_INC_LENGTHS], + /// By code. + offset: [u32; MAX_OFF as usize + 1], + /// By length, up to the longest that is priced. + ml: &'a mut [u32], } +/// `litSum`, `litLengthSum`, `matchLengthSum` and `offCodeSum` #[derive(Clone, Copy)] -struct CodePrice { - ml_code: u32, - price: u32, +struct Sums { + lit: u32, + lit_length: u32, + match_length: u32, + off_code: u32, } -impl<'a> MatchLengthPrice<'a> { - const fn new(opt_ptr: &'a OptState, opt_level: i32) -> Self { - Self { - opt_ptr, +impl<'a> Prices<'a> { + fn new(stats: &'a mut OptState, ml: &'a mut [u32], opt_level: i32) -> Result { + let mut prices = Self { opt_level, - last: None, + compressed_literals: compressed_literals(stats), + sums: Sums { + lit: stats.lit_sum, + lit_length: stats.lit_length_sum, + match_length: stats.match_length_sum, + off_code: stats.off_code_sum, + }, + lit_freq: stats.lit_freq.first_chunk_mut().ok_or(BUG)?, + lit_length_freq: stats.lit_length_freq.first_chunk_mut().ok_or(BUG)?, + match_length_freq: stats.match_length_freq.first_chunk_mut().ok_or(BUG)?, + off_code_freq: stats.off_code_freq.first_chunk_mut().ok_or(BUG)?, + // 8 bits per literal, if they are not compressed. + lit_base: 8 * BITCOST_MULTIPLIER, + lit_length_base: 0, + match_base: 0, + lit_weight: [0; MAX_LIT as usize + 1], + lit_length: [0; MAX_LL as usize + 1], + lit_length_inc: [0; LL_INC_LENGTHS], + offset: [0; MAX_OFF as usize + 1], + ml, + }; + if prices.compressed_literals { + for (weight_of, &freq) in prices.lit_weight.iter_mut().zip(&*prices.lit_freq) { + *weight_of = weight(freq, opt_level); + } + } + for code in 0..=MAX_LL { + prices.price_lit_length(code)?; + } + for code in 0..=MAX_OFF { + prices.price_offset(code)?; } + for code in 0..=MAX_ML { + prices.price_match_length(code)?; + } + prices.set_base_prices(); + Ok(prices) } #[inline(always)] - fn of(&mut self, match_length: u32) -> Result { - let ml_base = match_length.wrapping_sub(MINMATCH); - if self.opt_ptr.price_type == OptPrice::Predef { - return Ok(weight(ml_base, self.opt_level)); + fn price_lit_length(&mut self, code: u32) -> Result<()> { + let code = code as usize; + let bits = u32::from(LL_BITS.at(code)?) * BITCOST_MULTIPLIER; + let price = bits.wrapping_sub(weight(self.lit_length_freq.at(code)?, self.opt_level)); + self.lit_length.put(code, price)?; + // The price changes where the lengths of this code start, and where those of the next one do. + if let Some(inc) = self + .lit_length_inc + .get_mut(LL_CODE_START.at(code)? as usize) + && let Some(&before) = self.lit_length.get(code.wrapping_sub(1)) + { + *inc = price.wrapping_sub(before); } - - let ml_code = ml_code(ml_base); - if let Some(last) = self.last - && last.ml_code == ml_code + if let Some(inc) = self + .lit_length_inc + .get_mut(LL_CODE_START.at(code + 1)? as usize) + && let Some(&after) = self.lit_length.get(code + 1) { - return Ok(last.price); + *inc = after.wrapping_sub(price); } - let freq = self.opt_ptr.match_length_freq.at(ml_code as usize)?; - let price = (u32::from(ML_BITS.at(ml_code as usize)?) * BITCOST_MULTIPLIER) - .wrapping_add(self.opt_ptr.match_length_sum_base_price) - .wrapping_sub(weight(freq, self.opt_level)); - self.last = Some(CodePrice { ml_code, price }); - Ok(price) + Ok(()) } -} -/// `ZSTD_updateStats` -#[inline(always)] -fn update_stats( - opt_ptr: &mut OptState, - literals: &[u8], - off_base: u32, - match_length: u32, -) -> Result<()> { + /// Below level 2, small offsets are favoured, which are faster to decompress. #[inline(always)] - fn increment(freqs: &mut [u32], code: u32, sum: &mut u32) -> Result<()> { - let freq = freqs.at_mut(code as usize)?; - *freq = freq.wrapping_add(1); - *sum = sum.wrapping_add(1); + fn price_offset(&mut self, code: u32) -> Result<()> { + let freq = self.off_code_freq.at(code as usize)?; + let mut price = (code * BITCOST_MULTIPLIER).wrapping_sub(weight(freq, self.opt_level)); + if self.opt_level < 2 && code >= 20 { + price = price.wrapping_add((code - 19) * 2 * BITCOST_MULTIPLIER); + } + // Matches cost a bit more, to have fewer sequences, which is faster to decompress. + self.offset + .put(code as usize, price.wrapping_add(BITCOST_MULTIPLIER / 5)) + } + + #[inline(always)] + fn price_match_length(&mut self, code: u32) -> Result<()> { + let code = code as usize; + let bits = u32::from(ML_BITS.at(code)?) * BITCOST_MULTIPLIER; + let freq = self.match_length_freq.at(code)?; + let price = bits.wrapping_sub(weight(freq, self.opt_level)); + let most = self.ml.len(); + let start = (ML_CODE_START.at(code)? + MINMATCH) as usize; + let end = (ML_CODE_START.at(code + 1)? + MINMATCH) as usize; + self.ml.sub_mut(start.min(most)..end.min(most))?.fill(price); Ok(()) } - let lit_length = literals.len() as u32; - if compressed_literals(opt_ptr) { - for &literal in literals { - let freq = opt_ptr.lit_freq.at_mut(usize::from(literal))?; - *freq = freq.wrapping_add(ZSTD_LITFREQ_ADD); + /// `ZSTD_setBasePrices` + #[inline(always)] + fn set_base_prices(&mut self) { + if self.compressed_literals { + self.lit_base = weight(self.sums.lit, self.opt_level); } - opt_ptr.lit_sum = opt_ptr - .lit_sum - .wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD)); + self.lit_length_base = weight(self.sums.lit_length, self.opt_level); + self.match_base = weight(self.sums.match_length, self.opt_level) + .wrapping_add(weight(self.sums.off_code, self.opt_level)); } - increment( - &mut opt_ptr.lit_length_freq, - ll_code(lit_length), - &mut opt_ptr.lit_length_sum, - )?; - increment( - &mut opt_ptr.off_code_freq, - highbit32(off_base), - &mut opt_ptr.off_code_sum, - )?; - increment( - &mut opt_ptr.match_length_freq, - ml_code(match_length.wrapping_sub(MINMATCH)), - &mut opt_ptr.match_length_sum, - ) -} - -/// `ZSTD_readMINMATCH`: `length` bytes, 3 or 4, to compare with others. Reads 4. -#[inline(always)] -fn read_minmatch(buf: &[u8], pos: usize, length: u32) -> Result { - let value = read_le32(buf, pos)?; - Ok(if length == 3 { value << 8 } else { value }) -} - -/// `ZSTD_insertAndFindFirstIndexHash3`: indexes the 3-byte hashes up to `ip`, which is not included, and looks it up. -/// All of it is in the current segment. -fn insert_and_find_first_index_hash3( - ms: &mut MatchState, - next_to_update3: &mut u32, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - let hash_table3 = ms.hash_table3.as_mut_slice(); - let hash_log3 = ms.hash_log3; - let target = prefix.index_of(ip); - let hash3 = hash3_ptr(prefix.buf, ip, hash_log3)?; + /// `LIT_PRICE`, which is `ZSTD_rawLiteralsCost` of one literal. + #[inline(always)] + fn literal(&self, literal: u8) -> i32 { + let lit_price_max = self.lit_base.wrapping_sub(BITCOST_MULTIPLIER); + let lit_price = self.lit_weight.get(usize::from(literal)).copied(); + self.lit_base + .wrapping_sub(lit_price.unwrap_or(0).min(lit_price_max)) as i32 + } - for idx in *next_to_update3..target { - hash_table3.put(hash3_ptr(prefix.buf, prefix.pos_of(idx), hash_log3)?, idx)?; + /// `ZSTD_litLengthPrice`, but for what does not depend on the length. + #[inline(always)] + fn lit_length_part(&self, lit_length: u32) -> u32 { + // The format has no code for the length of a whole block of the largest size, which is then all literals. It + // is made to cost 1 bit more than the length before it. + let (lit_length, extra) = if lit_length == ZSTD_BLOCKSIZE_MAX as u32 { + (ZSTD_BLOCKSIZE_MAX as u32 - 1, BITCOST_MULTIPLIER) + } else { + (lit_length, 0) + }; + let price = self.lit_length.get(ll_code(lit_length) as usize).copied(); + extra.wrapping_add(price.unwrap_or(0)) } - *next_to_update3 = target; - hash_table3.at(hash3) -} + /// `LL_PRICE` + #[inline(always)] + fn lit_length(&self, lit_length: u32) -> i32 { + self.lit_length_base + .wrapping_add(self.lit_length_part(lit_length)) as i32 + } -// ── binary tree search ──────────────────────────────────────────────────── + /// `LL_INCPRICE`: what one more literal, which makes `lit_length` of them, adds to the price of their length. + #[inline(always)] + fn lit_length_inc(&self, lit_length: u32) -> i32 { + if let Some(&inc) = self.lit_length_inc.get(lit_length as usize) { + return inc as i32; + } + cold_path(); + self.lit_length_part(lit_length) + .wrapping_sub(self.lit_length_part(lit_length.wrapping_sub(1))) as i32 + } -/// `*ptr = value`, where `ptr` is an element of `bt`, or `None` for a place that nothing reads. -#[inline(always)] -fn store(bt: &mut [u32], ptr: Option, value: u32) -> Result<()> { - match ptr { - Some(ptr) => bt.put(ptr, value), - None => Ok(()), + /// `ZSTD_getMatchPrice`, the price of the offset and of the match length of a sequence, but for what depends on the + /// length, which is in `ml`. `off_base` is as in [`SeqStore::store_seq`]. + #[inline(always)] + fn offset(&self, off_base: u32) -> i32 { + let price = self.offset.get(highbit32(off_base) as usize).copied(); + self.match_base.wrapping_add(price.unwrap_or(0)) as i32 } -} -/// `ZSTD_insertBt1`: adds the position `ip`, which is at most `iend - 8`, to the tree. `target` is up to where the -/// caller is filling it, and `dict` the older segment, which is looked at if `ext_dict`. Returns how many positions -/// to move forward. -#[inline(always)] -fn insert_bt1( - ms: &mut MatchState, - prefix: Prefix<'_>, - dict: Prefix<'_>, - ip: usize, - iend: usize, - target: u32, - mls: u32, - ext_dict: bool, -) -> Result { - let c_params = ms.c_params; - let h = hash_ptr(prefix.buf, ip, c_params.hash_log, mls)?; - let bt_log = c_params.chain_log.wrapping_sub(1); - let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = dict.pos_of(dict_limit); - let prefix_start = prefix.pos_of(dict_limit); - let curr = prefix.index_of(ip); - let bt_low = curr.saturating_sub(bt_mask); - let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); - let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); - // Only the positions that are still in the window once the tree is filled are of use. - let window_low = ms.get_lowest_match_index(target, c_params.window_log); - let mut match_end_idx = curr.wrapping_add(8 + 1); - let mut best_length = 8usize; - let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); - - let bt = ms.chain_table.as_mut_slice(); - let hash_entry = ms.hash_table.at_mut(h)?; - let mut match_index = *hash_entry; - *hash_entry = curr; - - while nb_compares != 0 && match_index >= window_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - // What is sure to be common. - let mut match_length = common_length_smaller.min(common_length_larger); - - // A position of the older segment, as one of the current segment, only makes sense with a length that - // brings it there. - let mut match_buf = prefix.buf; - let mut match_pos = prefix.pos_of(match_index); - if !ext_dict || match_index as usize + match_length >= dict_limit as usize { - match_length += count( - prefix.buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - iend, - )?; - } else { - match_length += count_2segments( - prefix.buf, - ip + match_length, - dict.buf, - dict.pos_of(match_index) + match_length, - iend, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - match_buf = dict.buf; - match_pos = dict.pos_of(match_index); - } + /// `ZSTD_updateStats` + #[inline(always)] + fn update_stats(&mut self, literals: &[u8], off_base: u32, match_length: u32) -> Result<()> { + #[inline(always)] + fn increment(freqs: &mut [u32], code: u32, sum: &mut u32) -> Result<()> { + let freq = freqs.at_mut(code as usize)?; + *freq = freq.wrapping_add(1); + *sum = sum.wrapping_add(1); + Ok(()) } - if match_length > best_length { - best_length = match_length; - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); + let lit_length = literals.len() as u32; + if self.compressed_literals { + for &literal in literals { + let literal = usize::from(literal); + let freq = self.lit_freq.at(literal)?.wrapping_add(ZSTD_LITFREQ_ADD); + self.lit_freq.put(literal, freq)?; + self.lit_weight.put(literal, weight(freq, self.opt_level))?; } + self.sums.lit = self + .sums + .lit + .wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD)); } - if ip + match_length == iend { - // They are equal: there is no telling which is smaller. Dropping it keeps the tree consistent. - break; + let ll_code = ll_code(lit_length); + increment(self.lit_length_freq, ll_code, &mut self.sums.lit_length)?; + let off_code = highbit32(off_base); + increment(self.off_code_freq, off_code, &mut self.sums.off_code)?; + let ml_code = ml_code(match_length.wrapping_sub(MINMATCH)); + increment(self.match_length_freq, ml_code, &mut self.sums.match_length)?; + self.price_lit_length(ll_code)?; + self.price_offset(off_code)?; + self.price_match_length(ml_code) + } +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// What the match finders work on for the time of a block: the tables of a [`MatchState`], and what does not change +/// of it. +struct Trees<'a> { + hash_table: &'a mut [u32], + hash_table3: &'a mut [u32], + bt: &'a mut Nodes, + input: TreeInput<'a>, + dms: Option>, + low_limit: u32, + /// How far a match can be, if that is not down to `low_limit`. + max_distance: u32, + hasher: Hasher, + hash_log3: u32, + /// 3 or 4. + min_match: u32, + nb_compares: u32, + sufficient_len: u32, + next_to_update: u32, + next_to_update3: u32, +} + +impl<'a> Trees<'a> { + fn new( + ms: &'a mut MatchState, + prefix: Prefix<'a>, + dict: Prefix<'a>, + dms: Option>, + iend: usize, + mls: u32, + ) -> Result { + let c_params = ms.c_params; + let dict_limit = ms.window.dict_limit; + if ms.hash_table.len() != 1usize.wrapping_shl(c_params.hash_log) + || ms.chain_table.len() != 1usize.wrapping_shl(c_params.chain_log) + { + return Err(BUG); } + Ok(Self { + hash_table: ms.hash_table.as_mut_slice(), + hash_table3: ms.hash_table3.as_mut_slice(), + bt: bt_nodes(&mut ms.chain_table), + input: TreeInput::new(prefix, dict, dict_limit, iend)?, + dms, + low_limit: ms.window.low_limit, + // A dictionary is valid as a whole until it is invalidated as a whole. + max_distance: if ms.loaded_dict_end != 0 { + u32::MAX + } else { + 1u32.wrapping_shl(c_params.window_log) + }, + hasher: Hasher::new(c_params.hash_log, mls), + hash_log3: ms.hash_log3, + min_match: if c_params.min_match == 3 { 3 } else { 4 }, + nb_compares: 1u32.wrapping_shl(c_params.search_log), + sufficient_len: c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1), + next_to_update: ms.next_to_update, + next_to_update3: ms.next_to_update, + }) + } - if match_buf.at(match_pos.wrapping_add(match_length))? < prefix.buf.at(ip + match_length)? { - // All the smaller ones will have at least this much in common. - store(bt, smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. - smaller_ptr = None; - break; - } - smaller_ptr = Some(next_ptr + 1); - match_index = bt.at(next_ptr + 1)?; + /// `ZSTD_getLowestMatchIndex` + #[inline(always)] + const fn lowest_match_index(&self, curr: u32) -> u32 { + if curr.wrapping_sub(self.low_limit) > self.max_distance { + curr.wrapping_sub(self.max_distance) } else { - store(bt, larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = None; - break; - } - larger_ptr = Some(next_ptr); - match_index = bt.at(next_ptr)?; + self.low_limit } - nb_compares -= 1; } - store(bt, smaller_ptr, 0)?; - store(bt, larger_ptr, 0)?; + /// `ZSTD_insertAndFindFirstIndexHash3`: indexes the 3-byte hashes up to `curr`, which is not included and starts + /// with `bytes`, and looks it up. All of it is in the current segment. + #[inline(always)] + fn insert_and_find_first_index_hash3(&mut self, curr: u32, bytes: u32) -> Result { + let input = self.input; + for idx in self.next_to_update3..curr { + let bytes = input.near.read32(idx.wrapping_sub(input.dict_limit)); + let hash3 = hash3_of(bytes.ok_or(BUG)?, self.hash_log3); + table_put(self.hash_table3, hash3, idx); + } + self.next_to_update3 = curr; + Ok(table_get(self.hash_table3, hash3_of(bytes, self.hash_log3))) + } - let positions = if best_length > 384 { - 192.min((best_length - 384) as u32) - } else { - 0 - }; - Ok(positions.max(match_end_idx.wrapping_sub(curr.wrapping_add(8)))) -} + /// Adds the position `ip` of `near`, which starts with `bytes`, to its tree. `visit` is called with each position + /// that is met on the way, which are all at least `low`, and what it has in common with `ip`, and returns whether + /// to go on. Returns how many more positions could have been compared. + #[inline(always)] + fn walk( + &mut self, + ip: u32, + bytes: u64, + low: u32, + visit: impl FnMut(u32, Common) -> Result, + ) -> Result { + let curr = ip.wrapping_add(self.input.dict_limit); + if self.hash_table.is_empty() { + return Err(BUG); + } -/// `ZSTD_updateTree_internal` -#[inline(always)] -fn update_tree_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - dict: Prefix<'_>, - ip: usize, - iend: usize, - mls: u32, - dict_mode: DictMode, -) -> Result<()> { - let target = prefix.index_of(ip); - let mut idx = ms.next_to_update; - - while idx < target { - let forward = insert_bt1( - ms, - prefix, - dict, - prefix.pos_of(idx), - iend, - target, - mls, - dict_mode == DictMode::ExtDict, - )?; - idx = idx.wrapping_add(forward); + let h = self.hasher.hash(bytes); + let match_index = table_get(self.hash_table, h); + table_put(self.hash_table, h, curr); + bt_look_ahead::<4, 2>(self.hash_table, self.bt, &self.input, self.hasher, ip); + + let descent = Descent::new(self.bt, curr); + bt_walk( + self.bt, + &self.input, + ip, + match_index, + low, + self.nb_compares, + descent, + visit, + ) } - ms.next_to_update = target; - Ok(()) -} -/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. -/// Matches are not compared past the position `iend`. -pub(crate) fn update_tree( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - iend: usize, -) -> Result<()> { - let no_dict = Prefix { buf: &[], base: 0 }; - let mls = ms.c_params.min_match; - update_tree_internal(ms, prefix, no_dict, ip, iend, mls, DictMode::NoDict) -} - -/// `ZSTD_insertBtAndGetAllMatches`: adds the position `ip` to the tree, and stores in `matches` the matches found on -/// the way, each longer than the one before and all at least `length_to_beat` long. `ll0` is whether no literals -/// come before `ip`. Returns how many there are. -#[inline(always)] -fn insert_bt_and_get_all_matches( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - dict_mode: DictMode, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, - mls: u32, -) -> Result { - let buf = src.buf; - let c_params = ms.c_params; - let sufficient_len = c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); - let curr = src.index_of(ip); - let min_match = if mls == 3 { 3 } else { 4 }; - let h = hash_ptr(buf, ip, c_params.hash_log, mls)?; - let bt_log = c_params.chain_log.wrapping_sub(1); - let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1); - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let dict_limit = ms.window.dict_limit; - let dict_end = src.dict_pos_of(dict_limit); - let prefix_start = src.pos_of(dict_limit); - let bt_low = curr.saturating_sub(bt_mask); - let window_low = ms.get_lowest_match_index(curr, c_params.window_log); - let match_low = window_low.max(1); - let mut smaller_ptr = Some(2 * (curr & bt_mask) as usize); - let mut larger_ptr = Some(2 * (curr & bt_mask) as usize + 1); - // The farthest position any match refers to, which detects repetitive patterns. - let mut match_end_idx = curr.wrapping_add(8 + 1); - let mut mnum = 0u32; - let mut nb_compares = 1u32.wrapping_shl(c_params.search_log); + /// `ZSTD_insertBt1`: adds `curr`, which is at least 8 bytes before the end, to its tree. `target` is up to where + /// the caller is filling them. Returns how many positions to move forward. + #[inline(always)] + fn insert_bt1(&mut self, curr: u32, target: u32) -> Result { + let ip = curr.wrapping_sub(self.input.dict_limit); + let bytes = self.input.near.read64(ip).ok_or(BUG)?; + // Only the positions that are still in the window once the trees are filled are of use. + let window_low = self.lowest_match_index(target); + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut best_length = 8u32; + + self.walk(ip, bytes, window_low, |match_index, common| { + if common.len > best_length { + cold_path(); + best_length = common.len; + if common.len > match_end_idx.wrapping_sub(match_index) { + match_end_idx = match_index.wrapping_add(common.len); + } + } + // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. + Ok(common.match_is_smaller.is_some()) + })?; - let dms = if dict_mode == DictMode::DictMatchState { - Some(src.dms.ok_or(BUG)?) - } else { - None - }; - let dms_high_limit = dms.map_or(0, |dms| dms.end.wrapping_sub(dms.base) as u32); - let dms_low_limit = dms.map_or(0, |dms| dms.ms.window.low_limit); - let dms_index_delta = dms.map_or(0, |_| window_low.wrapping_sub(dms_high_limit)); - - let mut best_length = length_to_beat.wrapping_sub(1) as usize; - - // The repeat offsets. - let ll0 = u32::from(ll0); - let [rep0, _, _] = *rep; - for rep_code in ll0..ZSTD_REP_NUM as u32 + ll0 { - let rep_offset = match rep.get(rep_code as usize) { - Some(&offset) => offset, - None => rep0.wrapping_sub(1), + let positions = if best_length > 384 { + 192.min(best_length - 384) + } else { + 0 }; + Ok(positions.max(match_end_idx.wrapping_sub(curr.wrapping_add(8)))) + } + + /// `ZSTD_updateTree_internal`: adds the positions from `next_to_update` up to `target` to the trees. + #[inline(never)] + fn update_tree(&mut self, target: u32) -> Result<()> { + let mut idx = self.next_to_update; + while idx < target { + idx = idx.wrapping_add(self.insert_bt1(idx, target)?); + } + self.next_to_update = target; + Ok(()) + } + + /// The length of the match for `ip` at the repeat offset `rep_offset`, which is at least `min_match` if it leads to + /// the current segment. Else it is 0 if it is less. + #[inline(never)] + fn rep_len(&self, ip: u32, rep_offset: u32, window_low: u32) -> Result { + let input = self.input; + let min_match = self.min_match; + if rep_offset.wrapping_sub(1) < ip { + let more = input + .near + .count(ip + min_match, ip - rep_offset + min_match); + return Ok(more + min_match); + } + let curr = ip.wrapping_add(input.dict_limit); let rep_index = curr.wrapping_sub(rep_offset); - let mut rep_len = 0u32; - // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. - if rep_offset.wrapping_sub(1) < curr.wrapping_sub(dict_limit) { - // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. - let rep_match = ip.wrapping_sub(rep_offset as usize); - if rep_index >= window_low - && read_minmatch(buf, ip, min_match)? == read_minmatch(buf, rep_match, min_match)? - { - rep_len = count( - buf, - ip + min_match as usize, - buf, - rep_match + min_match as usize, - i_limit, - )? as u32 - + min_match; + let same_start = + |bytes: u32| Some(minmatch_differs(bytes, input.near.read32(ip)?, min_match) == 0); + if !index_overlap_check(input.dict_limit, rep_index) { + return Ok(0); + } + if let Some(dms) = self.dms { + let dms_high_limit = dms.end.wrapping_sub(dms.base) as u32; + let dms_index_delta = window_low.wrapping_sub(dms_high_limit); + let dms_low = dms.ms.window.low_limit.wrapping_add(dms_index_delta); + // `curr > rep_index >= dms_low` + if rep_offset.wrapping_sub(1) < curr.wrapping_sub(dms_low) { + let rep_match = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); + if same_start(read_le32(dms.buf, rep_match)?).ok_or(BUG)? { + let after = rep_match + min_match as usize; + let more = input.count_2segments(ip + min_match, dms.buf, after, dms.end)?; + return Ok(more + min_match); + } } - } else if dict_mode == DictMode::ExtDict { + } else if self.low_limit < input.dict_limit { // `curr > rep_index >= window_low` if rep_offset.wrapping_sub(1) < curr.wrapping_sub(window_low) - && index_overlap_check(dict_limit, rep_index) - { - let rep_match = src.dict_pos_of(rep_index); - if read_minmatch(buf, ip, min_match)? - == read_minmatch(src.dict_buf, rep_match, min_match)? - { - rep_len = count_2segments( - buf, - ip + min_match as usize, - src.dict_buf, - rep_match + min_match as usize, - i_limit, - dict_end, - prefix_start, - )? as u32 - + min_match; - } - } - } else if let Some(dms) = dms { - // `curr > rep_index >= dms_low_limit` - if rep_offset.wrapping_sub(1) - < curr.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta)) - && index_overlap_check(dict_limit, rep_index) + && same_start(read_le32(input.dict.buf, input.dict.pos_of(rep_index))?) + .ok_or(BUG)? { - let rep_match = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); - if read_minmatch(buf, ip, min_match)? - == read_minmatch(dms.buf, rep_match, min_match)? - { - rep_len = count_2segments( - buf, - ip + min_match as usize, - dms.buf, - rep_match + min_match as usize, - i_limit, - dms.end, - prefix_start, - )? as u32 - + min_match; - } - } - } - if rep_len as usize > best_length { - best_length = rep_len as usize; - matches.put( - mnum as usize, - Match { - off: rep_code - ll0 + 1, - len: rep_len, - }, - )?; - mnum += 1; - if rep_len > sufficient_len || ip + rep_len as usize == i_limit { - // The best there can be. - return Ok(mnum); + let after = rep_index.wrapping_add(min_match); + return Ok(input.count_ext_dict(ip + min_match, after)? + min_match); } } + Ok(0) } - // The table of 3-byte hashes. That of a dictionary is not filled. - if mls == 3 && best_length < mls as usize { - let match_index3 = - insert_and_find_first_index_hash3(ms, next_to_update3, src.prefix(), ip)?; - // Farther than that is likely to cost too much. - if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) { - let mlen = if dict_mode != DictMode::ExtDict || match_index3 >= dict_limit { - count(buf, ip, buf, src.pos_of(match_index3), i_limit)? - } else { - count_2segments( - buf, - ip, - src.dict_buf, - src.dict_pos_of(match_index3), - i_limit, - dict_end, - prefix_start, - )? - }; - - if mlen >= mls as usize { - best_length = mlen; - matches.put( - 0, - Match { - off: offset_to_offbase(curr.wrapping_sub(match_index3)), - len: mlen as u32, - }, - )?; - mnum = 1; - if mlen > sufficient_len as usize || ip + mlen == i_limit { - // The best there can be: `ip` is not inserted. - ms.next_to_update = curr.wrapping_add(1); - return Ok(1); - } - } + /// `ZSTD_btGetAllMatches_internal`: adds the positions up to `ip`, a position of `near`, to the trees, and stores in + /// `matches` the matches for it that are found on the way, each longer than the one before and all at least + /// `min_match` long. `ll0` is whether no literals come before `ip`. Returns how many there are. + #[inline(never)] + fn get_all_matches( + &mut self, + matches: &mut [Match], + ip: u32, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + ) -> Result { + let near = self.input.near; + let dict_limit = self.input.dict_limit; + let curr = ip.wrapping_add(dict_limit); + if curr < self.next_to_update { + // An area that is skipped. + return Ok(0); } - } - - let bt = ms.chain_table.as_mut_slice(); - let hash_entry = ms.hash_table.at_mut(h)?; - let mut match_index = *hash_entry; - *hash_entry = curr; - - while nb_compares != 0 && match_index >= match_low { - let next_ptr = 2 * (match_index & bt_mask) as usize; - // What is sure to be common. - let mut match_length = common_length_smaller.min(common_length_larger); - - // A position of the older segment, as one of the current segment, only makes sense with a length that - // brings it there. - let mut match_buf = buf; - let mut match_pos = src.pos_of(match_index); - if dict_mode != DictMode::ExtDict - || match_index as usize + match_length >= dict_limit as usize - { - match_length += count( - buf, - ip + match_length, - match_buf, - match_pos.wrapping_add(match_length), - i_limit, - )?; - } else { - match_length += count_2segments( - buf, - ip + match_length, - src.dict_buf, - src.dict_pos_of(match_index) + match_length, - i_limit, - dict_end, - prefix_start, - )?; - if (match_index as usize + match_length) < dict_limit as usize { - match_buf = src.dict_buf; - match_pos = src.dict_pos_of(match_index); - } + if self.next_to_update < curr { + self.update_tree(curr)?; } - if match_length > best_length { - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); + let min_match = self.min_match; + let bytes = near.read64(ip).ok_or(BUG)?; + let window_low = self.lowest_match_index(curr); + let mut mnum = 0u32; + let mut best_length = min_match - 1; + + // The repeat offsets. + let [rep0, rep1, rep2] = *rep; + let rep_offsets = if ll0 { + [rep1, rep2, rep0.wrapping_sub(1)] + } else { + [rep0, rep1, rep2] + }; + // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. + let most_in_prefix = ip.min(curr.wrapping_sub(window_low)); + for (off, &rep_offset) in (1..).zip(&rep_offsets) { + // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. + if rep_offset.wrapping_sub(1) < most_in_prefix { + let rep_match = ip.wrapping_sub(rep_offset); + let rep_bytes = near.read32(rep_match).ok_or(BUG)?; + if minmatch_differs(rep_bytes, bytes as u32, min_match) != 0 { + continue; + } + } else if rep_offset.wrapping_sub(1) < ip { + continue; } - best_length = match_length; - matches.put( - mnum as usize, - Match { - off: offset_to_offbase(curr.wrapping_sub(match_index)), - len: match_length as u32, - }, - )?; - mnum += 1; - if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { - // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. - if dict_mode == DictMode::DictMatchState { - // The dictionary is not to be searched either. - nb_compares = 0; + let rep_len = self.rep_len(ip, rep_offset, window_low)?; + if rep_len > best_length { + best_length = rep_len; + matches.put(mnum as usize, Match { off, len: rep_len })?; + mnum += 1; + if rep_len > self.sufficient_len || rep_len == near.len().wrapping_sub(ip) { + // The best there can be. + return Ok(mnum); } - break; } } - if match_buf.at(match_pos.wrapping_add(match_length))? < buf.at(ip + match_length)? { - // All the smaller ones will have at least this much in common. - store(bt, smaller_ptr, match_index)?; - common_length_smaller = match_length; - if match_index <= bt_low { - // Beyond the tree. - smaller_ptr = None; - break; - } - smaller_ptr = Some(next_ptr + 1); - match_index = bt.at(next_ptr + 1)?; - } else { - store(bt, larger_ptr, match_index)?; - common_length_larger = match_length; - if match_index <= bt_low { - larger_ptr = None; - break; + // The table of 3-byte hashes. That of a dictionary is not filled. + let match_low = window_low.max(1); + if best_length < 3 { + let match_index3 = self.insert_and_find_first_index_hash3(curr, bytes as u32)?; + // Farther than that is likely to cost too much. + if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) { + let mlen = if match_index3 >= dict_limit { + count(near, ip, match_index3.wrapping_sub(dict_limit)) + } else { + self.input.count_ext_dict(ip, match_index3)? + }; + + if mlen >= 3 { + best_length = mlen; + matches.put( + 0, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index3)), + len: mlen, + }, + )?; + mnum = 1; + if mlen > self.sufficient_len || mlen == near.len().wrapping_sub(ip) { + // The best there can be: `ip` is not inserted. + self.next_to_update = curr.wrapping_add(1); + return Ok(1); + } + } } - larger_ptr = Some(next_ptr); - match_index = bt.at(next_ptr)?; } - nb_compares -= 1; - } - - store(bt, smaller_ptr, 0)?; - store(bt, larger_ptr, 0)?; - if let Some(dms) = dms - && nb_compares != 0 - { - let dms_c_params = &dms.ms.c_params; - let dms_bt_log = dms_c_params.chain_log.wrapping_sub(1); - let dms_bt_mask = 1u32.wrapping_shl(dms_bt_log).wrapping_sub(1); - let dms_bt_low = if dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit) { - dms_high_limit.wrapping_sub(dms_bt_mask) - } else { - dms_low_limit - }; - let dms_h = hash_ptr(buf, ip, dms_c_params.hash_log, mls)?; - let mut dict_match_index = dms.ms.hash_table.at(dms_h)?; - let dms_bt = dms.ms.chain_table.as_slice(); - common_length_smaller = 0; - common_length_larger = 0; - while nb_compares != 0 && dict_match_index > dms_low_limit { - let next_ptr = 2 * (dict_match_index & dms_bt_mask) as usize; - let mut match_length = common_length_smaller.min(common_length_larger); - match_length += count_2segments( - buf, - ip + match_length, - dms.buf, - dms.pos_of(dict_match_index) + match_length, - i_limit, - dms.end, - prefix_start, - )?; - - if match_length > best_length { - match_index = dict_match_index.wrapping_add(dms_index_delta); - if match_length > match_end_idx.wrapping_sub(match_index) as usize { - match_end_idx = match_index.wrapping_add(match_length as u32); + // The farthest position any match refers to, which detects repetitive patterns. + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut search_dms = self.dms.is_some(); + let nb_compares = self.walk(ip, bytes, match_low, |match_index, common| { + if common.len > best_length { + if common.len > match_end_idx.wrapping_sub(match_index) { + match_end_idx = match_index.wrapping_add(common.len); } - best_length = match_length; + best_length = common.len; matches.put( mnum as usize, Match { off: offset_to_offbase(curr.wrapping_sub(match_index)), - len: match_length as u32, + len: common.len, }, )?; mnum += 1; - if match_length > ZSTD_OPT_NUM || ip + match_length == i_limit { - break; + if common.len > ZSTD_OPT_NUM as u32 || common.match_is_smaller.is_none() { + // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. + // The dictionary is not to be searched either. + search_dms = false; + return Ok(false); } } - - if dict_match_index <= dms_bt_low { - // Beyond the tree. - break; - } - // A match that reaches the end of the dictionary goes on at the start of the current segment. - let match_byte = if dict_match_index as usize + match_length >= dms_high_limit as usize - { - let match_pos = src.pos_of(dict_match_index.wrapping_add(dms_index_delta)); - buf.at(match_pos.wrapping_add(match_length))? - } else { - dms.buf.at(dms.pos_of(dict_match_index) + match_length)? + Ok(true) + })?; + + if search_dms { + let mut found = Found { + mnum, + best_length, + match_end_idx, }; - if match_byte < buf.at(ip + match_length)? { - common_length_smaller = match_length; - dict_match_index = dms_bt.at(next_ptr + 1)?; - } else { - common_length_larger = match_length; - dict_match_index = dms_bt.at(next_ptr)?; - } - nb_compares -= 1; + self.search_dms(matches, ip, bytes, window_low, nb_compares, &mut found)?; + mnum = found.mnum; + match_end_idx = found.match_end_idx; } + + // Repetitive patterns are skipped. + self.next_to_update = match_end_idx.wrapping_sub(8); + Ok(mnum) } - // Repetitive patterns are skipped. - ms.next_to_update = match_end_idx.wrapping_sub(8); - Ok(mnum) + /// The end of `ZSTD_insertBtAndGetAllMatches`: goes on in the tree of the dictionary, for `nb_compares` positions at + /// most. + #[inline(never)] + fn search_dms( + &self, + matches: &mut [Match], + ip: u32, + bytes: u64, + window_low: u32, + nb_compares: u32, + found: &mut Found, + ) -> Result<()> { + let dms = self.dms.ok_or(BUG)?; + let input = self.input; + let curr = ip.wrapping_add(input.dict_limit); + let tree = DictTree::new(dms, window_low)?; + let hash = self.hasher.with_log(dms.ms.c_params.hash_log).hash(bytes); + tree.walk(&input, ip, hash, nb_compares, |dict_match_index, common| { + if common.len > found.best_length { + let match_index = dict_match_index.wrapping_add(tree.index_delta); + if common.len > found.match_end_idx.wrapping_sub(match_index) { + found.match_end_idx = match_index.wrapping_add(common.len); + } + found.best_length = common.len; + matches.put( + found.mnum as usize, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index)), + len: common.len, + }, + )?; + found.mnum += 1; + if common.len > ZSTD_OPT_NUM as u32 || common.match_is_smaller.is_none() { + return Ok(false); + } + } + Ok(true) + }) + } } -/// `ZSTD_getAllMatchesFn`: the arguments are `matches`, `ms`, `next_to_update3`, `src`, `ip`, `i_high_limit`, `rep`, -/// `ll0` and `length_to_beat`. -type GetAllMatchesFn = fn( - &mut [Match], - &mut MatchState, - &mut u32, - &BlockSrc<'_>, - usize, - usize, - &[u32; ZSTD_REP_NUM], - bool, - u32, -) -> Result; - -/// `ZSTD_btGetAllMatches_internal` -#[inline(always)] -fn bt_get_all_matches_internal( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, - dict_mode: DictMode, - mls: u32, -) -> Result { - if src.index_of(ip) < ms.next_to_update { - // An area that is skipped. - return Ok(0); - } - let dict = Prefix { - buf: src.dict_buf, - base: src.dict_base, - }; - update_tree_internal(ms, src.prefix(), dict, ip, i_high_limit, mls, dict_mode)?; - insert_bt_and_get_all_matches( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - dict_mode, - rep, - ll0, - length_to_beat, - mls, - ) +/// [`Near::count`], where few candidates get that far. +#[inline(never)] +fn count(near: Near<'_>, pos: u32, match_pos: u32) -> u32 { + near.count(pos, match_pos) } -/// `ZSTD_btGetAllMatches_noDict_3` to `ZSTD_btGetAllMatches_noDict_6` -fn bt_get_all_matches_no_dict( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::NoDict, - MLS, - ) +/// What a search has found so far. +struct Found { + mnum: u32, + best_length: u32, + match_end_idx: u32, } -/// `ZSTD_btGetAllMatches_extDict_3` to `ZSTD_btGetAllMatches_extDict_6` -fn bt_get_all_matches_ext_dict( - matches: &mut [Match], - ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, - ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::ExtDict, - MLS, - ) +/// `ZSTD_readMINMATCH`: 0 if the first `length` bytes, 3 or 4, of the 4 in `a` and in `b` are the same. +#[inline(always)] +const fn minmatch_differs(a: u32, b: u32, length: u32) -> u32 { + (a ^ b) << ((4 - length) * 8) } -/// `ZSTD_btGetAllMatches_dictMatchState_3` to `ZSTD_btGetAllMatches_dictMatchState_6` -fn bt_get_all_matches_dict_match_state( - matches: &mut [Match], +/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. +/// Matches are not compared past the position `iend`. +pub(crate) fn update_tree( ms: &mut MatchState, - next_to_update3: &mut u32, - src: &BlockSrc<'_>, + prefix: Prefix<'_>, ip: usize, - i_high_limit: usize, - rep: &[u32; ZSTD_REP_NUM], - ll0: bool, - length_to_beat: u32, -) -> Result { - bt_get_all_matches_internal( - matches, - ms, - next_to_update3, - src, - ip, - i_high_limit, - rep, - ll0, - length_to_beat, - DictMode::DictMatchState, - MLS, - ) -} - -/// `ZSTD_selectBtGetAllMatches` -fn select_bt_get_all_matches(ms: &MatchState, dict_mode: DictMode) -> GetAllMatchesFn { - match (dict_mode, ms.c_params.min_match) { - (DictMode::NoDict, ..=3) => bt_get_all_matches_no_dict::<3>, - (DictMode::NoDict, 4) => bt_get_all_matches_no_dict::<4>, - (DictMode::NoDict, 5) => bt_get_all_matches_no_dict::<5>, - (DictMode::NoDict, 6..) => bt_get_all_matches_no_dict::<6>, - (DictMode::ExtDict, ..=3) => bt_get_all_matches_ext_dict::<3>, - (DictMode::ExtDict, 4) => bt_get_all_matches_ext_dict::<4>, - (DictMode::ExtDict, 5) => bt_get_all_matches_ext_dict::<5>, - (DictMode::ExtDict, 6..) => bt_get_all_matches_ext_dict::<6>, - (DictMode::DictMatchState, ..=3) => bt_get_all_matches_dict_match_state::<3>, - (DictMode::DictMatchState, 4) => bt_get_all_matches_dict_match_state::<4>, - (DictMode::DictMatchState, 5) => bt_get_all_matches_dict_match_state::<5>, - (DictMode::DictMatchState, 6..) => bt_get_all_matches_dict_match_state::<6>, - } + iend: usize, +) -> Result<()> { + let no_dict = Prefix { buf: &[], base: 0 }; + let mls = ms.c_params.min_match; + let mut trees = Trees::new(ms, prefix, no_dict, None, iend, mls)?; + trees.update_tree(prefix.index_of(ip))?; + ms.next_to_update = prefix.index_of(ip); + Ok(()) } // ── candidates from long distance matching ──────────────────────────────── @@ -1261,28 +1047,56 @@ impl<'a> OptLdm<'a> { // ── the optimal parser ──────────────────────────────────────────────────── -/// `LIT_PRICE`: the price of the literal at `pos`. -#[inline(always)] -fn lit_price(buf: &[u8], pos: usize, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok(raw_literals_cost(buf.span(pos, 1)?, opt_ptr, opt_level)? as i32) +/// `ZSTD_optimal_t`, as [`PriceTable`] has it: a match followed by literals. +#[derive(Clone, Copy, Default)] +struct Stretch { + off: u32, + mlen: u32, + litlen: u32, + rep: [u32; ZSTD_REP_NUM], } -/// `LL_PRICE` +/// [`PriceTable::lens`] has the length of the match in its low bits, which are enough since the table has +/// [`ZSTD_OPT_NUM`] positions, and the number of literals above. +const LITLEN_SHIFT: u32 = 12; +const ONE_LITERAL: u32 = 1 << LITLEN_SHIFT; + #[inline(always)] -fn ll_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok(lit_length_price(lit_length, opt_ptr, opt_level)? as i32) +const fn litlen_of(lens: u32) -> u32 { + lens >> LITLEN_SHIFT } -/// `LL_INCPRICE`: what one more literal, which makes `lit_length` of them, adds to the price of their length. #[inline(always)] -fn ll_inc_price(lit_length: u32, opt_ptr: &OptState, opt_level: i32) -> Result { - Ok( - ll_price(lit_length, opt_ptr, opt_level)?.wrapping_sub(ll_price( - lit_length.wrapping_sub(1), - opt_ptr, - opt_level, - )?), - ) +const fn mlen_of(lens: u32) -> u32 { + lens & (ONE_LITERAL - 1) +} + +/// With fewer lengths than this, [`try_lengths`] is not worth calling. +const FEW_LENGTHS: u32 = 4; + +/// The positions that the lengths from `first_len` on of a match at the offset `off` lead to take it if that is cheaper +/// than what leads to them so far. `ml_prices` has the price of each of these lengths, and `base_price` is the rest. +#[inline(never)] +fn try_lengths( + prices: &mut [i32], + offs: &mut [u32], + lens: &mut [u32], + ml_prices: &[u32], + first_len: u32, + base_price: i32, + off: u32, +) { + let reached = prices.iter_mut().zip(offs).zip(lens); + for (mlen, (((price, best_off), best_lens), &ml_price)) in + (first_len..).zip(reached.zip(ml_prices)) + { + // Written so that it is done on vectors. + let new_price = base_price.wrapping_add(ml_price as i32); + let cheaper = u32::from(new_price < *price).wrapping_neg(); + *price = new_price.min(*price); + *best_off = (*best_off & !cheaper) | (off & cheaper); + *best_lens = (*best_lens & !cheaper) | (mlen & cheaper); + } } /// `ZSTD_compressBlock_opt_generic` @@ -1295,33 +1109,37 @@ fn compress_block_opt_generic( opt_level: i32, dict_mode: DictMode, ) -> Result { - // The match finders borrow the whole of `ms`. - let mut price_table = mem::take(&mut ms.opt.price_table); - let mut match_table = mem::take(&mut ms.opt.match_table); + // The match finders borrow the rest of `ms`. + let mut opt_state = mem::replace(&mut ms.opt, OptState::new()); + let mut match_table = mem::take(&mut opt_state.match_table); + let mut price_table = mem::take(&mut opt_state.price_table); let result = parse_block( ms, + &mut opt_state, + &mut match_table, + &mut price_table, seq_store, rep, src, - &mut price_table, - &mut match_table, opt_level, dict_mode, ); - ms.opt.price_table = price_table; - ms.opt.match_table = match_table; + opt_state.match_table = match_table; + opt_state.price_table = price_table; + ms.opt = opt_state; result } -/// The body of `ZSTD_compressBlock_opt_generic`. `opt` is `ms.opt.price_table` and `matches` is `ms.opt.match_table`. +/// The body of `ZSTD_compressBlock_opt_generic`. `opt_state` is `ms.opt`, without `matches` and `price_table`. #[inline(always)] fn parse_block( ms: &mut MatchState, + opt_state: &mut OptState, + matches: &mut [Match], + price_table: &mut PriceTable, seq_store: &mut SeqStore, rep: &mut [u32; ZSTD_REP_NUM], src: &BlockSrc<'_>, - opt: &mut [Optimal], - matches: &mut [Match], opt_level: i32, dict_mode: DictMode, ) -> Result { @@ -1333,45 +1151,48 @@ fn parse_block( let ilimit = iend.saturating_sub(8); let prefix_start = src.pos_of(ms.window.dict_limit); - let get_all_matches = select_bt_get_all_matches(ms, dict_mode); - let sufficient_len = ms.c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); let min_match = if ms.c_params.min_match == 3 { 3 } else { 4 }; - let mut next_to_update3 = ms.next_to_update; + let dict = Prefix { + buf: src.dict_buf, + base: src.dict_base, + }; + let dms = if dict_mode == DictMode::DictMatchState { + Some(src.dms.ok_or(BUG)?) + } else { + None + }; + let mls = ms.c_params.min_match.clamp(3, 6); + let mut trees = Trees::new(ms, src.prefix(), dict, dms, iend, mls)?; - let mut last_stretch = Optimal::default(); + let mut last_stretch = Stretch::default(); let mut opt_ldm = OptLdm::new(src.ldm_seq_store); opt_ldm.get_next_match_and_update_seq_store(0, (iend - ip) as u32); - rescale_freqs( - &mut ms.opt, - src.symbol_costs, - buf.sub(istart..iend)?, - opt_level, - )?; + rescale_freqs(opt_state, src.symbol_costs, buf.sub(istart..iend)?)?; + // Of the same size, so that one check is enough for all. + let opt_price: &mut [_; ZSTD_OPT_SIZE] = price_table.price.first_chunk_mut().ok_or(BUG)?; + let opt_off: &mut [_; ZSTD_OPT_SIZE] = price_table.off.first_chunk_mut().ok_or(BUG)?; + let opt_lens: &mut [_; ZSTD_OPT_SIZE] = price_table.lens.first_chunk_mut().ok_or(BUG)?; + let opt_rep: &mut [_; ZSTD_OPT_SIZE] = price_table.rep.first_chunk_mut().ok_or(BUG)?; + let ml_prices = price_table.ml_price.sub_mut(..=sufficient_len as usize)?; + let mut prices = Prices::new(opt_state, ml_prices, opt_level)?; ip += usize::from(ip == prefix_start); 'series: while ip < ilimit { let mut last_pos: u32; + let ll0_price = prices.lit_length(0); + let ll1_inc_price = prices.lit_length_inc(1); - // The forward pass. `opt` holds stretches, each a match followed by literals, unlike a sequence, which is + // The forward pass. The table holds stretches, each a match followed by literals, unlike a sequence, which is // literals followed by a match: that way each position of a run of literals can have another match before it. // It ends with the last stretch in `last_stretch`, which starts at `cur` and ends at `last_pos`. let mut cur = 'forward: { // Find a first match. { let litlen = (ip - anchor) as u32; - let mut nb_matches = get_all_matches( - matches, - ms, - &mut next_to_update3, - src, - ip, - iend, - rep, - litlen == 0, - min_match, - )?; + let mut nb_matches = + trees.get_all_matches(matches, (ip - prefix_start) as u32, rep, litlen == 0)?; opt_ldm.process_match_candidate( matches, &mut nb_matches, @@ -1387,14 +1208,10 @@ fn parse_block( // The price of the literals before the first match is the same whatever comes next, so it is left // out. But how the price of their length changes depends on their length. - let first_price = ll_price(litlen, &ms.opt, opt_level)?; - { - let first = opt.at_mut(0)?; - first.mlen = 0; - first.litlen = litlen; - first.price = first_price; - first.rep = *rep; - } + let first_price = prices.lit_length(litlen); + opt_price.put(0, first_price)?; + opt_lens.put(0, litlen << LITLEN_SHIFT)?; + opt_rep.put(0, *rep)?; // A large match is encoded at once. let longest = matches.at(matches.len() - 1)?; @@ -1407,89 +1224,80 @@ fn parse_block( } // Set the prices of the first matches. - let ll0_price = ll_price(0, &ms.opt, opt_level)?; let mut pos = 1u32; while pos < min_match { - let stretch = opt.at_mut(pos as usize)?; - stretch.price = ZSTD_MAX_PRICE; - stretch.mlen = 0; - stretch.litlen = litlen.wrapping_add(pos); + opt_price.put(pos as usize, ZSTD_MAX_PRICE)?; + opt_lens.put(pos as usize, litlen.wrapping_add(pos) << LITLEN_SHIFT)?; pos += 1; } - let mut ml_price = MatchLengthPrice::new(&ms.opt, opt_level); for candidate in matches { - let off_price = offset_price(candidate.off, &ms.opt, opt_level)?; + let base_price = first_price + .wrapping_add(ll0_price) + .wrapping_add(prices.offset(candidate.off)); while pos <= candidate.len { - let match_price = off_price.wrapping_add(ml_price.of(pos)?) as i32; - let sequence_price = first_price.wrapping_add(match_price); - let stretch = opt.at_mut(pos as usize)?; - stretch.mlen = pos; - stretch.off = candidate.off; - stretch.litlen = 0; - stretch.price = sequence_price.wrapping_add(ll0_price); + let price = base_price.wrapping_add(prices.ml.at(pos as usize)? as i32); + opt_price.put(pos as usize, price)?; + opt_off.put(pos as usize, candidate.off)?; + opt_lens.put(pos as usize, pos)?; pos += 1; } } last_pos = pos - 1; - opt.at_mut(pos as usize)?.price = ZSTD_MAX_PRICE; + opt_price.put(pos as usize, ZSTD_MAX_PRICE)?; } // Check the positions that follow. let mut cur = 0u32; while cur < last_pos { cur += 1; - let inr = ip + cur as usize; + let at = cur as usize; + let inr = ip + at; + if at + 1 >= ZSTD_OPT_SIZE { + return Err(BUG); + } // Get to this position by a literal, if it is cheaper. { - let before = opt.at(cur as usize - 1)?; - let litlen = before.litlen.wrapping_add(1); - let price = before - .price - .wrapping_add(lit_price(buf, inr - 1, &ms.opt, opt_level)?) - .wrapping_add(ll_inc_price(litlen, &ms.opt, opt_level)?); - let prev_match = opt.at(cur as usize)?; - if price <= prev_match.price { - opt.put( - cur as usize, - Optimal { - litlen, - price, - ..before - }, - )?; + let lens = opt_lens.at(at - 1)?.wrapping_add(ONE_LITERAL); + let litlen = litlen_of(lens); + let price = opt_price + .at(at - 1)? + .wrapping_add(prices.literal(buf.at(inr - 1)?)) + .wrapping_add(prices.lit_length_inc(litlen)); + let match_price = opt_price.at(at)?; + if price <= match_price { + let match_lens = opt_lens.at(at)?; + let match_off = opt_off.at(at)?; + opt_price.put(at, price)?; + opt_lens.put(at, lens)?; + let off = opt_off.at(at - 1)?; + opt_off.put(at, off)?; + let rep = opt_rep.at(at - 1)?; + opt_rep.put(at, rep)?; // If this replaces a match, and 1 literal is cheaper than none, the match followed by a // literal may be a cheaper way to the next position. if opt_level >= 1 - && prev_match.litlen == 0 - && ll_inc_price(1, &ms.opt, opt_level)? < 0 + && litlen_of(match_lens) == 0 + && ll1_inc_price < 0 && inr < iend { - let next_lit_price = lit_price(buf, inr, &ms.opt, opt_level)?; - let with_1_literal = prev_match - .price + let next_lit_price = prices.literal(buf.at(inr)?); + let with_1_literal = match_price .wrapping_add(next_lit_price) - .wrapping_add(ll_inc_price(1, &ms.opt, opt_level)?); + .wrapping_add(ll1_inc_price); let with_more_literals = price .wrapping_add(next_lit_price) - .wrapping_add(ll_inc_price( - litlen.wrapping_add(1), - &ms.opt, - opt_level, - )?); + .wrapping_add(prices.lit_length_inc(litlen.wrapping_add(1))); if with_1_literal < with_more_literals - && with_1_literal < opt.at(cur as usize + 1)?.price + && with_1_literal < opt_price.at(at + 1)? { - let prev = opt.at(cur.wrapping_sub(prev_match.mlen) as usize)?; - opt.put( - cur as usize + 1, - Optimal { - rep: new_rep(&prev.rep, prev_match.off, prev.litlen == 0), - litlen: 1, - price: with_1_literal, - ..prev_match - }, - )?; + let prev = cur.wrapping_sub(match_lens) as usize; + let ll0 = litlen_of(opt_lens.at(prev)?) == 0; + opt_price.put(at + 1, with_1_literal)?; + opt_lens.put(at + 1, match_lens + ONE_LITERAL)?; + opt_off.put(at + 1, match_off)?; + let rep = new_rep(&opt_rep.at(prev)?, match_off, ll0); + opt_rep.put(at + 1, rep)?; last_pos = last_pos.max(cur + 1); } } @@ -1497,11 +1305,13 @@ fn parse_block( } // The repeat offsets are not updated while matches are compared, but here, now that one is selected. - let mut here = opt.at(cur as usize)?; - if here.litlen == 0 { - let prev = opt.at(cur.wrapping_sub(here.mlen) as usize)?; - here.rep = new_rep(&prev.rep, here.off, prev.litlen == 0); - opt.at_mut(cur as usize)?.rep = here.rep; + let lens = opt_lens.at(at)?; + let ll0 = litlen_of(lens) == 0; + if ll0 { + let prev = cur.wrapping_sub(lens) as usize; + let prev_ll0 = litlen_of(opt_lens.at(prev)?) == 0; + let rep = new_rep(&opt_rep.at(prev)?, opt_off.at(at)?, prev_ll0); + opt_rep.put(at, rep)?; } // The last match starts at least 8 bytes before the end. @@ -1513,26 +1323,20 @@ fn parse_block( break; } + let price = opt_price.at(at)?; if opt_level == 0 - && opt.at(cur as usize + 1)?.price - <= here.price.wrapping_add(BITCOST_MULTIPLIER as i32 / 2) + && opt_price.at(at + 1)? <= price.wrapping_add(BITCOST_MULTIPLIER as i32 / 2) { // The next position is cheaper to get to: this one is not promising. continue; } { - let base_price = here.price.wrapping_add(ll_price(0, &ms.opt, opt_level)?); - let mut nb_matches = get_all_matches( + let mut nb_matches = trees.get_all_matches( matches, - ms, - &mut next_to_update3, - src, - inr, - iend, - &here.rep, - here.litlen == 0, - min_match, + (inr - prefix_start) as u32, + opt_rep.get(at).ok_or(BUG)?, + ll0, )?; opt_ldm.process_match_candidate( matches, @@ -1558,50 +1362,66 @@ fn parse_block( break 'forward cur; } + // The positions that nothing leads to yet. + let furthest = cur + longest.len; + if furthest > last_pos { + let new = last_pos as usize + 1..=furthest as usize; + opt_price.sub_mut(new.clone())?.fill(ZSTD_MAX_PRICE); + // Not the end of a match. + opt_lens.sub_mut(new)?.fill(ONE_LITERAL); + last_pos = furthest; + } + // Set the prices that the matches found here give. + let base_price = price.wrapping_add(ll0_price); let mut start_ml = min_match; - let mut ml_price = MatchLengthPrice::new(&ms.opt, opt_level); for candidate in matches { if candidate.len < start_ml { start_ml = candidate.len + 1; continue; } - let off_price = offset_price(candidate.off, &ms.opt, opt_level)?; - let furthest = cur + candidate.len; - // The first position that nothing leads to yet is taken at any price. - let mut new = furthest > last_pos; - while last_pos < furthest { - last_pos += 1; - let stretch = opt.at_mut(last_pos as usize)?; - stretch.price = ZSTD_MAX_PRICE; - // Not 0, which would be the end of a match. - stretch.litlen = 1; - } - - // From the longest length to the shortest. - let reached = opt.sub_mut((cur + start_ml) as usize..=furthest as usize)?; - let lengths = (start_ml..=candidate.len).rev(); - for (stretch, mlen) in reached.iter_mut().rev().zip(lengths) { - let match_price = off_price.wrapping_add(ml_price.of(mlen)?) as i32; - let price = base_price.wrapping_add(match_price); - if new || price < stretch.price { - stretch.mlen = mlen; - stretch.off = candidate.off; - stretch.litlen = 0; - stretch.price = price; - } else if opt_level == 0 { - // Shorter lengths are not tried. - break; + let base_price = base_price.wrapping_add(prices.offset(candidate.off)); + if opt_level == 0 || candidate.len - start_ml < FEW_LENGTHS { + // From the longest length to the shortest. + for mlen in (start_ml..=candidate.len).rev() { + let ml_price = prices.ml.at(mlen as usize)?; + let price = base_price.wrapping_add(ml_price as i32); + let pos = (cur + mlen) as usize; + if price < opt_price.at(pos)? { + opt_price.put(pos, price)?; + opt_off.put(pos, candidate.off)?; + opt_lens.put(pos, mlen)?; + } else if opt_level == 0 { + // Shorter lengths are not tried. + break; + } } - new = false; + } else { + let reached = + (cur + start_ml) as usize..=(cur + candidate.len) as usize; + try_lengths( + opt_price.sub_mut(reached.clone())?, + opt_off.sub_mut(reached.clone())?, + opt_lens.sub_mut(reached)?, + prices.ml.sub(start_ml as usize..=candidate.len as usize)?, + start_ml, + base_price, + candidate.off, + ); } start_ml = candidate.len + 1; } } - opt.at_mut(last_pos as usize + 1)?.price = ZSTD_MAX_PRICE; + opt_price.put(last_pos as usize + 1, ZSTD_MAX_PRICE)?; } - last_stretch = opt.at(last_pos as usize)?; + let lens = opt_lens.at(last_pos as usize)?; + last_stretch = Stretch { + off: opt_off.at(last_pos as usize)?, + mlen: mlen_of(lens), + litlen: litlen_of(lens), + rep: opt_rep.at(last_pos as usize)?, + }; last_pos.wrapping_sub(last_stretch.mlen) }; @@ -1613,50 +1433,49 @@ fn parse_block( if last_stretch.litlen == 0 { // It ends with a match. - let prev = opt.at(cur as usize)?; - *rep = new_rep(&prev.rep, last_stretch.off, prev.litlen == 0); + let ll0 = litlen_of(opt_lens.at(cur as usize)?) == 0; + *rep = new_rep(&opt_rep.at(cur as usize)?, last_stretch.off, ll0); } else { *rep = last_stretch.rep; cur = cur.wrapping_sub(last_stretch.litlen); } - // Write the cheapest path in `opt`, backwards from `store_end`, over the stretches that are not on it, and as - // sequences. The literals that end the last stretch are in no sequence: they are parsed again. (The C code - // starts by making them a last sequence without a match, and overwrites it at once.) - let store_end = cur.wrapping_add(2) as usize; - let mut store_start = store_end; + // Follow the cheapest path backwards, and turn its stretches into sequences, which go where the repeat offsets + // were, from the end. The literals that end the last stretch are in no sequence: they are parsed again. + let mut first_seq = opt_rep.len(); let mut stretch_pos = cur; - opt.put(store_end, last_stretch)?; + let mut mlen = last_stretch.mlen; + let mut off = last_stretch.off; loop { - let next_stretch = opt.at(stretch_pos as usize)?; - opt.at_mut(store_start)?.litlen = next_stretch.litlen; - if next_stretch.mlen == 0 { + let lens = opt_lens.at(stretch_pos as usize)?; + first_seq = first_seq.wrapping_sub(1); + opt_rep.put(first_seq, [litlen_of(lens), mlen, off])?; + mlen = mlen_of(lens); + if mlen == 0 { // The start of the series. break; } - store_start = store_start.wrapping_sub(1); - opt.put(store_start, next_stretch)?; - stretch_pos = - stretch_pos.wrapping_sub(next_stretch.litlen.wrapping_add(next_stretch.mlen)); + off = opt_off.at(stretch_pos as usize)?; + stretch_pos = stretch_pos.wrapping_sub(litlen_of(lens).wrapping_add(mlen)); } - for sequence in opt.sub(store_start..=store_end)? { - let llen = sequence.litlen as usize; - let mlen = sequence.mlen as usize; - update_stats( - &mut ms.opt, - buf.span(anchor, llen)?, - sequence.off, - sequence.mlen, - )?; - seq_store.store_seq(llen, buf, anchor, sequence.off, mlen)?; - anchor += llen + mlen; + for &[llen, mlen, off] in opt_rep.sub(first_seq..)? { + let llen = llen as usize; + prices.update_stats(buf.span(anchor, llen)?, off, mlen)?; + seq_store.store_seq(llen, buf, anchor, off, mlen as usize)?; + anchor += llen + mlen as usize; ip = anchor; } - set_base_prices(&mut ms.opt, opt_level); + prices.set_base_prices(); } + let sums = prices.sums; + opt_state.lit_sum = sums.lit; + opt_state.lit_length_sum = sums.lit_length; + opt_state.match_length_sum = sums.match_length; + opt_state.off_code_sum = sums.off_code; + ms.next_to_update = trees.next_to_update; Ok(iend - anchor) } From aae1516593854d28383e4e558376e57937817e0d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:48:21 +0000 Subject: [PATCH 046/199] zstd: a table that gets shorter and longer again is not cleared as long as indices go on --- src/zstd/compress/cctx.rs | 32 +++++++++++----- src/zstd/compress/internal.rs | 72 +++++++++++++++++++++++++++++++---- 2 files changed, 86 insertions(+), 18 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index d0d2d702a346..7987d08de932 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -13,7 +13,7 @@ use super::internal::{ BlockSrc, BufId, Buffers, CompressedBlockState, CompressionParameters, DictMatchState, DictTableLoadMethod, EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, Match, MatchState, Optimal, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqChunk, SeqStore, Strategy, - TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, + Table, TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, min_gain, no_compress_block, offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, }; @@ -160,6 +160,13 @@ pub(crate) fn reset_table( pub(crate) use reset_table as reset_struct_table; +/// `ZSTD_cwksp_mark_tables_dirty` and then `ZSTD_cwksp_mark_tables_clean` +fn forget_unused_indices(ms: &mut MatchState) { + ms.hash_table.forget_rest(); + ms.chain_table.forget_rest(); + ms.hash_table3.forget_rest(); +} + /// `ZSTD_reset_matchState` fn reset_match_state( ms: &mut MatchState, @@ -180,17 +187,20 @@ fn reset_match_state( // As long as indices go on, the ones in the tables are too low to be valid. let clean = force_reset_index == IndexResetPolicy::Reset && crp != CompResetPolicy::LeaveDirty; - reset_table(&mut ms.hash_table, sizes.h_size, clean)?; - reset_table(&mut ms.chain_table, sizes.chain_size, clean)?; - reset_table(&mut ms.hash_table3, sizes.h3_size, clean)?; + ms.hash_table.reset(sizes.h_size, clean)?; + ms.chain_table.reset(sizes.chain_size, clean)?; + ms.hash_table3.reset(sizes.h3_size, clean)?; + if force_reset_index == IndexResetPolicy::Reset { + forget_unused_indices(ms); + } if row_match_finder_used(c_params.strat(), use_row_match_finder) { if for_who == ResetTarget::CCtx { // A new salt instead of erasing the tags. - reset_table(&mut ms.tag_table, sizes.h_size, false)?; + ms.tag_table.reset(sizes.h_size, false)?; ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(u64::from(ms.hash_salt_entropy), 4); } else { - reset_table(&mut ms.tag_table, sizes.h_size, true)?; + ms.tag_table.reset(sizes.h_size, true)?; ms.hash_salt = 0; } let row_log = c_params.search_log.clamp(4, 6); @@ -254,6 +264,7 @@ fn reduce_table(table: &mut [u32], reducer_value: u32, preserve_mark: bool) { /// `ZSTD_reduceIndex` fn reduce_index(ms: &mut MatchState, params: &CCtxParams, reducer_value: u32) { + forget_unused_indices(ms); reduce_table(&mut ms.hash_table, reducer_value, false); if allocate_chain_table(params.c_params.strat(), params.use_row_match_finder) { let preserve_mark = params.c_params.strat() == Strategy::BtLazy2; @@ -909,10 +920,10 @@ impl FrameCompressor { } else { self.oversized_duration = 0; let ms = &mut self.match_state; - ms.hash_table = Vec::new(); - ms.chain_table = Vec::new(); - ms.hash_table3 = Vec::new(); - ms.tag_table = Vec::new(); + ms.hash_table = Table::new(); + ms.chain_table = Table::new(); + ms.hash_table3 = Table::new(); + ms.tag_table = Table::new(); ms.opt.match_table = Vec::new(); ms.opt.price_table = Vec::new(); } @@ -1056,6 +1067,7 @@ impl FrameCompressor { let src = &cdict.match_state; let dst = &mut self.match_state; + forget_unused_indices(dst); let tagged = cdict_indices_are_tagged(&cdict_c_params); copy_table(&mut dst.hash_table, &src.hash_table, tagged)?; copy_table(&mut dst.chain_table, &src.chain_table, tagged)?; diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 081599a8fb94..0922af6806ec 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1,5 +1,6 @@ //! `zstd_compress_internal.h`: what the parts of the compressor share. +use super::cctx::reset_table; use super::fse::{FseCTable, FseRepeat}; use super::huf::{HufCTable, HufRepeat}; use super::sequences::SeqCodes; @@ -1251,6 +1252,61 @@ impl OptState { pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; +/// A table of a match finder. A frame that needs a shorter one than a frame before it had uses the start of that one. +/// The rest stays as it is, for a frame that needs more again, so that nothing is cleared as long as indices go on. +pub(crate) struct Table { + all: Vec, + len: usize, +} + +impl Table { + pub(crate) const fn new() -> Self { + Self { + all: Vec::new(), + len: 0, + } + } + + /// Gives it the length `len`. What it holds is 0, or if `clean` is false possibly what it held at some time. + pub(crate) fn reset(&mut self, len: usize, clean: bool) -> Result<()> { + let kept = if clean { 0 } else { self.all.len() }; + reset_table(&mut self.all, len.max(kept), clean)?; + self.len = len; + Ok(()) + } + + /// For when indices get lower: the ones behind the part in use would seem valid later. + pub(crate) fn forget_rest(&mut self) { + self.all.truncate(self.len); + } + + pub(crate) fn capacity(&self) -> usize { + self.all.capacity() + } + + pub(crate) fn as_slice(&self) -> &[T] { + self.all.get(..self.len).unwrap_or(&[]) + } + + pub(crate) fn as_mut_slice(&mut self) -> &mut [T] { + self.all.get_mut(..self.len).unwrap_or(&mut []) + } +} + +impl core::ops::Deref for Table { + type Target = [T]; + + fn deref(&self) -> &[T] { + self.as_slice() + } +} + +impl core::ops::DerefMut for Table { + fn deref_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + /// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures /// own, are in [`BlockSrc`]. pub(crate) struct MatchState { @@ -1263,14 +1319,14 @@ pub(crate) struct MatchState { pub(crate) hash_log3: u32, pub(crate) row_hash_log: u32, /// For the row-based match finder: a byte of hash for each entry of `hash_table`. - pub(crate) tag_table: Vec, + pub(crate) tag_table: Table, pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], /// Makes what a previous frame left in `tag_table` meaningless. pub(crate) hash_salt: u64, pub(crate) hash_salt_entropy: u32, - pub(crate) hash_table: Vec, - pub(crate) hash_table3: Vec, - pub(crate) chain_table: Vec, + pub(crate) hash_table: Table, + pub(crate) hash_table3: Table, + pub(crate) chain_table: Table, pub(crate) force_non_contiguous: bool, pub(crate) opt: OptState, /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. @@ -1287,13 +1343,13 @@ impl MatchState { next_to_update: 0, hash_log3: 0, row_hash_log: 0, - tag_table: Vec::new(), + tag_table: Table::new(), hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], hash_salt: 0, hash_salt_entropy: 0, - hash_table: Vec::new(), - hash_table3: Vec::new(), - chain_table: Vec::new(), + hash_table: Table::new(), + hash_table3: Table::new(), + chain_table: Table::new(), force_non_contiguous: false, opt: OptState::new(), has_dict_match_state: false, From 75f0c6003ec0899401d866ae5a76f98fc5201a5e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:48:21 +0000 Subject: [PATCH 047/199] zstd: the buffer of literals is cleared only as far as blocks are long --- src/zstd/compress/cctx.rs | 2 +- src/zstd/compress/internal.rs | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 7987d08de932..3025dc07bc48 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -1182,7 +1182,7 @@ impl FrameCompressor { } return Ok(BuildSeqStore::NoCompress); } - self.seq_store.reset(); + self.seq_store.start_block(src_size); ms.opt.literal_compression_mode = ParamSwitch::Auto; { diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 0922af6806ec..ec9de2dde586 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -267,7 +267,7 @@ pub(crate) enum LongLengthType { pub(crate) struct SeqStore { /// Never grows beyond the capacity it is given by [`Self::reserve`]. pub(crate) sequences: Vec, - /// `max_nb_lit` bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. + /// As long as the longest block so far, and [`WILDCOPY_OVERLENGTH`] to copy in chunks, with room for `max_nb_lit`. pub(crate) lit_buffer: Vec, /// How much of `lit_buffer` is literals. pub(crate) lit_len: usize, @@ -366,13 +366,21 @@ impl SeqStore { room(&mut self.sequences, max_nb_seq)?; room(&mut self.codes, max_nb_seq)?; room(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; - self.lit_buffer.resize(max_nb_lit + WILDCOPY_OVERLENGTH, 0); self.max_nb_seq = max_nb_seq; self.max_nb_lit = max_nb_lit; self.reset(); Ok(()) } + /// `ZSTD_resetSeqStore`, for a block of `block_size` bytes. + pub(crate) fn start_block(&mut self, block_size: usize) { + let len = block_size.min(self.max_nb_lit) + WILDCOPY_OVERLENGTH; + if self.lit_buffer.len() < len { + self.lit_buffer.resize(len, 0); + } + self.reset(); + } + /// `ZSTD_resetSeqStore` #[inline] pub(crate) fn reset(&mut self) { From 08619ebfc615d47c8b7358d1b2be3b2998920a12 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:49:47 +0000 Subject: [PATCH 048/199] zstd: fast and dfast know the sizes of the tables of the default parameters, and dfast asks for what it looks at next as soon as a match ends --- src/zstd/compress/double_fast.rs | 82 ++++++++++++++++++++++++-------- src/zstd/compress/fast.rs | 59 ++++++++++++++++------- src/zstd/compress/internal.rs | 28 +++++++++++ 3 files changed, 131 insertions(+), 38 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 74e968e3c907..1f76174d4ab5 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -5,9 +5,10 @@ use super::fast::{ restore_rep, }; use super::internal::{ - BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Prefix, + BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, hash_of, hash_ptr, - hash8, le64_at, offset_to_offbase, table_get, table_put, table_swap, write_tagged_index, + hash8, le64_at, offset_to_offbase, table_get, table_of_size, table_prefetch, table_put, + table_swap, write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -105,6 +106,23 @@ pub(crate) fn fill_double_hash_table( } } +/// Has what a search from `pos` on looks at first in the tables on its way. `pos` is the end of a match that was just +/// found: there is enough to do about that one in the meantime. +#[inline(always)] +fn look_ahead( + near: Near<'_>, + pos: u32, + hash_long: &[u32], + h_bits_l: u32, + hash_small: &[u32], + h_bits_s: u32, +) { + if let Some(bytes) = near.read64(pos) { + table_prefetch(hash_long, hash8(bytes, h_bits_l, 0)); + table_prefetch(hash_small, hash_of::(bytes, h_bits_s)); + } +} + /// What `dfast` looks for matches with, when there is no dictionary and the window is one segment. struct Finder { /// The index of the first position. @@ -122,8 +140,10 @@ impl Finder { /// /// The C code computes the long hash and reads its table one position ahead. Here that is left to the processor: a /// turn of the inner loop only takes the position and the 8 bytes there, `bytes0`, from the one before. + /// + /// `LONG` and `SMALL` are `h_bits_l` and `h_bits_s`, or both 0 for any: see [`table_of_size`]. #[inline(never)] - fn find_matches( + fn find_matches( &mut self, hash_long: &mut [u32], hash_small: &mut [u32], @@ -139,9 +159,14 @@ impl Finder { mut offset_2, } = *self; let near = sink.near; - if hash_long.is_empty() || hash_small.is_empty() { + let h_bits_l = if LONG > 0 { LONG } else { h_bits_l }; + let h_bits_s = if SMALL > 0 { SMALL } else { h_bits_s }; + let Some(hash_long) = table_of_size::(hash_long) else { return; - } + }; + let Some(hash_small) = table_of_size::(hash_small) else { + return; + }; let Some(mut bytes0) = near.read64(ip) else { return; }; @@ -167,6 +192,7 @@ impl Finder { { ip += 1; let end = ip + 4 + sink.count_before(ip + 4, offset_1); + look_ahead::(near, end, hash_long, h_bits_l, hash_small, h_bits_s); sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); break end; } @@ -205,6 +231,7 @@ impl Finder { continue; }; + look_ahead::(near, end, hash_long, h_bits_l, hash_small, h_bits_s); offset_2 = offset_1; offset_1 = ip - match0; // With a larger step, ip1 may be past the end of the match. @@ -255,7 +282,7 @@ impl Finder { let index = base_index.wrapping_add(ip); table_put(hash_small, hash_of::(bytes0, h_bits_s), index); table_put(hash_long, hash8(bytes0, h_bits_l, 0), index); - sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); + sink.store_repeat(end); ip = end; let Some(bytes) = near.read64(ip) else { break 'block; @@ -290,11 +317,14 @@ pub(crate) fn compress_block_double_fast( let mut sink = MatchSink::new(seq_store, &block); let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; - match ms.c_params.min_match { - 5 => finder.find_matches::<5>(hash_long, hash_small, &mut sink), - 6 => finder.find_matches::<6>(hash_long, hash_small, &mut sink), - 7 => finder.find_matches::<7>(hash_long, hash_small, &mut sink), - _ => finder.find_matches::<4>(hash_long, hash_small, &mut sink), + // First, what levels 3 and 4 come with if the input is not known to be small. + match (ms.c_params.min_match, h_bits_l, h_bits_s) { + (5, 17, 16) => finder.find_matches::<5, 17, 16>(hash_long, hash_small, &mut sink), + (5, 18, 18) => finder.find_matches::<5, 18, 18>(hash_long, hash_small, &mut sink), + (5, ..) => finder.find_matches::<5, 0, 0>(hash_long, hash_small, &mut sink), + (6, ..) => finder.find_matches::<6, 0, 0>(hash_long, hash_small, &mut sink), + (7, ..) => finder.find_matches::<7, 0, 0>(hash_long, hash_small, &mut sink), + _ => finder.find_matches::<4, 0, 0>(hash_long, hash_small, &mut sink), } let last_literals = sink.last_literals()?; restore_rep( @@ -494,9 +524,9 @@ pub(crate) fn compress_block_double_fast_dict_match_state( } impl Finder { - /// `ZSTD_compressBlock_doubleFast_extDict_generic` + /// `ZSTD_compressBlock_doubleFast_extDict_generic`. `LONG` and `SMALL` are as for [`Self::find_matches`]. #[inline(never)] - fn find_matches_ext_dict( + fn find_matches_ext_dict( &mut self, hash_long: &mut [u32], hash_small: &mut [u32], @@ -515,9 +545,14 @@ impl Finder { let older_len = window.older.len(); // The index of the first place. let origin_index = base_index.wrapping_sub(older_len); - if hash_long.is_empty() || hash_small.is_empty() { + let h_bits_l = if LONG > 0 { LONG } else { h_bits_l }; + let h_bits_s = if SMALL > 0 { SMALL } else { h_bits_s }; + let Some(hash_long) = table_of_size::(hash_long) else { return; - } + }; + let Some(hash_small) = table_of_size::(hash_small) else { + return; + }; // `ip` is before the last position that can be looked at. while let Some(here) = near.window::<{ HASH_READ_SIZE + 1 }>(ip) { @@ -534,6 +569,7 @@ impl Finder { ip += 1; let rep_place = window.place_of(ip) - offset_1; let end = ip + 4 + window.count(ip + 4, rep_place + 4); + look_ahead::(near, end, hash_long, h_bits_l, hash_small, h_bits_s); sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); ip = end; } else { @@ -556,6 +592,7 @@ impl Finder { continue; }; let end = ip + known + window.count(ip + known, place + known); + look_ahead::(near, end, hash_long, h_bits_l, hash_small, h_bits_s); let back = window.count_backwards(ip, sink.anchor, place); offset_2 = offset_1; offset_1 = window.place_of(ip) - place; @@ -599,7 +636,7 @@ impl Finder { let index = base_index.wrapping_add(ip); table_put(hash_small, hash_of::(bytes, h_bits_s), index); table_put(hash_long, hash8(bytes, h_bits_l, 0), index); - sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); + sink.store_repeat(end); ip = end; } } @@ -634,11 +671,14 @@ pub(crate) fn compress_block_double_fast_ext_dict( let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; - match ms.c_params.min_match { - 5 => finder.find_matches_ext_dict::<5>(hash_long, hash_small, &mut sink, block.window), - 6 => finder.find_matches_ext_dict::<6>(hash_long, hash_small, &mut sink, block.window), - 7 => finder.find_matches_ext_dict::<7>(hash_long, hash_small, &mut sink, block.window), - _ => finder.find_matches_ext_dict::<4>(hash_long, hash_small, &mut sink, block.window), + let (long, small, sink) = (hash_long, hash_small, &mut sink); + match (ms.c_params.min_match, h_bits_l, h_bits_s) { + (5, 17, 16) => finder.find_matches_ext_dict::<5, 17, 16>(long, small, sink, block.window), + (5, 18, 18) => finder.find_matches_ext_dict::<5, 18, 18>(long, small, sink, block.window), + (5, ..) => finder.find_matches_ext_dict::<5, 0, 0>(long, small, sink, block.window), + (6, ..) => finder.find_matches_ext_dict::<6, 0, 0>(long, small, sink, block.window), + (7, ..) => finder.find_matches_ext_dict::<7, 0, 0>(long, small, sink, block.window), + _ => finder.find_matches_ext_dict::<4, 0, 0>(long, small, sink, block.window), } let last_literals = sink.last_literals()?; let [rep0, rep1, _] = rep; diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 10b4c217d0ad..a978517a481e 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -4,7 +4,7 @@ use super::internal::{ BlockSrc, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, hash_of, hash_ptr, le32_at, le64_at, - offset_to_offbase, table_get, table_put, table_swap, write_tagged_index, + offset_to_offbase, table_get, table_of_size, table_put, table_swap, write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -312,6 +312,20 @@ impl<'a, 'b> MatchSink<'a, 'b> { } } + /// Stores the match with the last offset that goes on from the one before up to `end`. + #[inline(always)] + pub(super) fn store_repeat(&mut self, end: u32) { + let ip = self.anchor; + self.anchor = end; + if (self.seq_store) + .store_seq_only(0, REPCODE1_TO_OFFBASE, end - ip) + .is_err() + { + core::hint::cold_path(); + self.failed = true; + } + } + /// Whether the bytes before `ip` and before what is `offset` before it are the same. #[inline(always)] fn same_byte_before(&self, ip: u32, offset: u32) -> bool { @@ -436,8 +450,10 @@ impl Finder { /// Each turn of the inner loop looks at two positions, that of index `current0` and the next one, for a match, and /// at `ip2` for a repeat. The C code computes hashes and reads the table one position ahead. Here that is left to /// the processor: a turn only takes the position and the 8 bytes there, `bytes0`, from the one before. + /// + /// `HLOG` is `hlog`, or 0 for any: see [`table_of_size`]. #[inline(never)] - fn find_matches( + fn find_matches( &mut self, hash_table: &mut [u32], sink: &mut MatchSink<'_, '_>, @@ -452,9 +468,10 @@ impl Finder { mut rep_offset2, } = *self; let near = sink.near; - if hash_table.is_empty() { + let hlog = if HLOG > 0 { HLOG } else { hlog }; + let Some(hash_table) = table_of_size::(hash_table) else { return; - } + }; let Some(mut bytes0) = near.read64(ip0) else { return; }; @@ -562,7 +579,7 @@ impl Finder { impl Finder { /// `ZSTD_compressBlock_fast_extDict_generic`, which goes about it as [`Self::find_matches`] does. #[inline(never)] - fn find_matches_ext_dict( + fn find_matches_ext_dict( &mut self, hash_table: &mut [u32], sink: &mut MatchSink<'_, '_>, @@ -581,9 +598,10 @@ impl Finder { let older_len = window.older.len(); // The index of the first place. let origin_index = base_index.wrapping_sub(older_len); - if hash_table.is_empty() { + let hlog = if HLOG > 0 { HLOG } else { hlog }; + let Some(hash_table) = table_of_size::(hash_table) else { return; - } + }; let Some(mut bytes0) = near.read64(ip0) else { return; }; @@ -713,11 +731,15 @@ pub(crate) fn compress_block_fast( }; let mut sink = MatchSink::new(seq_store, &block); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; - match ms.c_params.min_match { - 5 => finder.find_matches::<5>(hash_table, &mut sink), - 6 => finder.find_matches::<6>(hash_table, &mut sink), - 7 => finder.find_matches::<7>(hash_table, &mut sink), - _ => finder.find_matches::<4>(hash_table, &mut sink), + // First, what levels 1, 2 and those below 0 come with if the input is not known to be small. + match (ms.c_params.min_match, hlog) { + (7, 14) => finder.find_matches::<7, 14>(hash_table, &mut sink), + (6, 16) => finder.find_matches::<6, 16>(hash_table, &mut sink), + (6, 13) => finder.find_matches::<6, 13>(hash_table, &mut sink), + (5, _) => finder.find_matches::<5, 0>(hash_table, &mut sink), + (6, _) => finder.find_matches::<6, 0>(hash_table, &mut sink), + (7, _) => finder.find_matches::<7, 0>(hash_table, &mut sink), + _ => finder.find_matches::<4, 0>(hash_table, &mut sink), } let last_literals = sink.last_literals()?; restore_rep( @@ -976,11 +998,14 @@ pub(crate) fn compress_block_fast_ext_dict( }; let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; - match ms.c_params.min_match { - 5 => finder.find_matches_ext_dict::<5>(hash_table, &mut sink, block.window), - 6 => finder.find_matches_ext_dict::<6>(hash_table, &mut sink, block.window), - 7 => finder.find_matches_ext_dict::<7>(hash_table, &mut sink, block.window), - _ => finder.find_matches_ext_dict::<4>(hash_table, &mut sink, block.window), + match (ms.c_params.min_match, hlog) { + (7, 14) => finder.find_matches_ext_dict::<7, 14>(hash_table, &mut sink, block.window), + (6, 16) => finder.find_matches_ext_dict::<6, 16>(hash_table, &mut sink, block.window), + (6, 13) => finder.find_matches_ext_dict::<6, 13>(hash_table, &mut sink, block.window), + (5, _) => finder.find_matches_ext_dict::<5, 0>(hash_table, &mut sink, block.window), + (6, _) => finder.find_matches_ext_dict::<6, 0>(hash_table, &mut sink, block.window), + (7, _) => finder.find_matches_ext_dict::<7, 0>(hash_table, &mut sink, block.window), + _ => finder.find_matches_ext_dict::<4, 0>(hash_table, &mut sink, block.window), } let last_literals = sink.last_literals()?; restore_rep( diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 6fbab725601c..bc19a4dc62ad 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -8,6 +8,7 @@ use crate::common::{ }; use crate::error::{ErrorCode, Result}; use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64, write_le24}; +use core::hint::{Locality, prefetch_read}; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; pub(crate) const HASH_READ_SIZE: usize = 8; @@ -459,7 +460,17 @@ impl<'a> SeqSink<'a> { )?, } self.lit_len += lit_length; + self.store_seq_only(lit_length, off_base, match_length) + } + /// `ZSTD_storeSeqOnly` + #[inline(always)] + pub(crate) fn store_seq_only( + &mut self, + lit_length: u32, + off_base: u32, + match_length: u32, + ) -> Result<()> { let ml_base = match_length.wrapping_sub(MINMATCH); if lit_length > 0xFFFF || ml_base > 0xFFFF { self.store.long_length_type = if ml_base > 0xFFFF { @@ -806,6 +817,23 @@ impl<'a> Near<'a> { } } +/// `table`, of `1 << BITS` cells, with that known to the compiler, which then neither has to keep the size around nor +/// to check a hash of `BITS` bits against it. With 0, `table` as it is. +#[inline(always)] +pub(crate) fn table_of_size(table: &mut [u32]) -> Option<&mut [u32]> { + if BITS > 0 { + table.get_mut(..1 << BITS) + } else { + Some(table) + } +} + +/// Has `table[hash]` on its way. +#[inline(always)] +pub(crate) fn table_prefetch(table: &[u32], hash: usize) { + prefetch_read(table.as_ptr().wrapping_add(hash), Locality::L1); +} + /// `table[hash]`. `hash` is less than the size of `table`: to check that delays nothing, which a mask would. #[inline(always)] pub(crate) fn table_get(table: &[u32], hash: usize) -> u32 { From 7bf8f5b1c24f2eb58046263a2fa96140f59ea066 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:51:02 +0000 Subject: [PATCH 049/199] zstd: dfast only asks whether an index was too low once the bytes are the same --- src/zstd/compress/double_fast.rs | 4 ++-- src/zstd/compress/internal.rs | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 1f76174d4ab5..2deb5db37343 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -201,9 +201,9 @@ impl Finder { // read instead is unlikely to be the same. let matchl0 = idxl0.wrapping_sub(base_index); let matchs0 = idxs0.wrapping_sub(base_index); - let (match0, end) = if near.diff64(matchl0, bytes0) == 0 { + let (match0, end) = if near.has::<{ u64::MAX }>(matchl0, bytes0) { (matchl0, ip + 8 + near.count(ip + 8, matchl0 + 8)) - } else if near.diff64(matchs0, bytes0) as u32 == 0 { + } else if near.has::<0xFFFF_FFFF>(matchs0, bytes0) { // A short match: a longer one at the next position is preferred. let end = ip + 4 + near.count(ip + 4, matchs0 + 4); let idxl1 = table_get(hash_long, hash8(bytes1, h_bits_l, 0)); diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index bc19a4dc62ad..14355365f251 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -9,6 +9,7 @@ use crate::common::{ use crate::error::{ErrorCode, Result}; use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64, write_le24}; use core::hint::{Locality, prefetch_read}; +use core::sync::atomic::{Ordering, compiler_fence}; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; pub(crate) const HASH_READ_SIZE: usize = 8; @@ -709,6 +710,26 @@ impl<'a> Near<'a> { } } + /// Whether the bits of `MASK` are the same in `bytes` and in the 8 bytes at `pos`. If `pos` is too far to read them, + /// that is only found out once other bytes turned out to be the same. + #[inline(always)] + pub(crate) fn has(self, pos: u32, bytes: u64) -> bool { + let Some(last) = self.0.len().checked_sub(8) else { + return false; + }; + let pos = pos as usize; + let at = pos.min(last); + let Some(there) = self.0.get(at..at + 8).and_then(<[u8]>::first_chunk) else { + return false; + }; + if (u64::from_le_bytes(*there) ^ bytes) & MASK != 0 { + return false; + } + // Costs nothing, and keeps the compiler from asking the second question first. + compiler_fence(Ordering::SeqCst); + pos <= last + } + #[inline(always)] fn chunk16(self, pos: u32) -> Option<&'a [u8; 16]> { self.window(pos) From f5e9ea1c74d93d0f35e14c198519bd8e1c4e61ae Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:52:46 +0000 Subject: [PATCH 050/199] zstd: likewise with two segments and with a dictionary's match state --- src/zstd/compress/double_fast.rs | 9 ++++----- src/zstd/compress/fast.rs | 10 +++++----- src/zstd/compress/internal.rs | 15 --------------- 3 files changed, 9 insertions(+), 25 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 2deb5db37343..0d5312f7c7f4 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -197,8 +197,7 @@ impl Finder { break end; } - // An index that is too low leads past the end. Whether one is takes no branch to find out: what is - // read instead is unlikely to be the same. + // An index that is too low leads past the end. let matchl0 = idxl0.wrapping_sub(base_index); let matchs0 = idxs0.wrapping_sub(base_index); let (match0, end) = if near.has::<{ u64::MAX }>(matchl0, bytes0) { @@ -378,7 +377,7 @@ impl Finder { } else { // The place of the match, and how much of it is known. `ip` is moved to where it is for. let (place, known) = 'found: { - if near.diff64(match_long, bytes0) == 0 { + if near.has::<{ u64::MAX }>(match_long, bytes0) { break 'found (window.place_of(match_long), 8); } if let Some(place) = dict_tables.long.get::<8>(bytes0) @@ -574,9 +573,9 @@ impl Finder { ip = end; } else { // The match, and how much of it is known. `ip` is moved to where it is for. - let (place, known) = if window.diff64_after_first(place_long, bytes0) == 0 { + let (place, known) = if window.has_after_first::<{ u64::MAX }>(place_long, bytes0) { (place_long, 8) - } else if window.diff64_after_first(place_short, bytes0) as u32 == 0 { + } else if window.has_after_first::<0xFFFF_FFFF>(place_short, bytes0) { // A short match: a long one at the next position is preferred. let bytes1 = le64_at::<1, _>(here); let place3 = table_swap(hash_long, hash8(bytes1, h_bits_l, 0), curr + 1) diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index a978517a481e..337e81bec93b 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -394,12 +394,12 @@ impl<'a> Segments<'a> { spot.segment.read64(spot.pos) } - /// What differs between `bytes` and the 8 bytes at `place`: 0 if nothing does. Not 0 in the lower half if they are - /// not in one segment, or if `place` is the first one, which takes no branch to find out. + /// Whether the bits of `MASK` are the same in `bytes` and in the 8 bytes at `place`, which are in one segment, and + /// `place` is not the first one. #[inline(always)] - pub(super) fn diff64_after_first(self, place: u32, bytes: u64) -> u64 { + pub(super) fn has_after_first(self, place: u32, bytes: u64) -> bool { let spot = self.spot(place); - spot.segment.diff64(spot.pos, bytes) | u64::from(place == 0) + spot.segment.has::(spot.pos, bytes) && place > 0 } /// The 4 bytes that are `offset` before the position `pos`, if they are in one segment. Which one that is does not @@ -874,7 +874,7 @@ impl Finder { break Some(place); } let match_pos = match_index.wrapping_sub(base_index); - if near.diff64(match_pos, bytes0) as u32 == 0 { + if near.has::<0xFFFF_FFFF>(match_pos, bytes0) { break Some(window.place_of(match_pos)); } diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 14355365f251..c04d1de81464 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -695,21 +695,6 @@ impl<'a> Near<'a> { Some(u64::from_le_bytes(*bytes)) } - /// What differs between `bytes` and the 8 bytes at `pos`: 0 if nothing does. Not 0 in the lower half if `pos` is too - /// far to read them, which takes no branch to find out. - #[inline(always)] - pub(crate) fn diff64(self, pos: u32, bytes: u64) -> u64 { - let Some(last) = self.0.len().checked_sub(8) else { - return 1; - }; - let pos = pos as usize; - let at = pos.min(last); - match self.0.get(at..at + 8).and_then(<[u8]>::first_chunk) { - Some(there) => (u64::from_le_bytes(*there) ^ bytes) | u64::from(pos > last), - None => 1, - } - } - /// Whether the bits of `MASK` are the same in `bytes` and in the 8 bytes at `pos`. If `pos` is too far to read them, /// that is only found out once other bytes turned out to be the same. #[inline(always)] From ef0f7338a458ddbef7893ae5dbaf537b7abe14e3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:52:51 +0000 Subject: [PATCH 051/199] zstd: room for sequences and literals is made for the blocks that come, not for the largest there could be --- src/zstd/compress/block_entropy.rs | 4 +- src/zstd/compress/cctx.rs | 6 +-- src/zstd/compress/internal.rs | 59 +++++++++++++++++++++--------- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/src/zstd/compress/block_entropy.rs b/src/zstd/compress/block_entropy.rs index 73847b74a829..30f0cbb2f908 100644 --- a/src/zstd/compress/block_entropy.rs +++ b/src/zstd/compress/block_entropy.rs @@ -29,9 +29,7 @@ pub(crate) fn literals_compression_is_disabled(cctx_params: &CCtxParams) -> bool /// `ZSTD_seqToCodes`, and the distributions of the codes. fn seq_to_codes(store: &mut SeqStore, chunk: &SeqChunk) -> Result { let range = chunk.seq_start..chunk.seq_end; - if store.codes.len() < store.sequences.len() { - (store.codes).resize(store.sequences.len(), SeqCodes::default()); - } + store.make_room_for_codes()?; Ok(sequences::seq_to_codes( store.codes.sub_mut(range.clone())?, store.sequences.sub(range)?, diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 3025dc07bc48..e75fe38ac7cd 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -971,9 +971,7 @@ impl FrameCompressor { self.ldm_sequences = empty_raw_seq_store(); } - if self.seq_store.max_nb_seq != max_nb_seq || self.seq_store.max_nb_lit != block_size { - self.seq_store.reserve(max_nb_seq, block_size)?; - } + self.seq_store.set_limits(max_nb_seq, block_size); self.reference_external_sequences(Vec::new()); self.initialized = true; @@ -1182,7 +1180,7 @@ impl FrameCompressor { } return Ok(BuildSeqStore::NoCompress); } - self.seq_store.start_block(src_size); + self.seq_store.start_block(src_size)?; ms.opt.literal_compression_mode = ParamSwitch::Auto; { diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index ec9de2dde586..4db50f27c4f8 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -265,13 +265,13 @@ pub(crate) enum LongLengthType { /// `SeqStore_t`: the sequences and the literals of a block. pub(crate) struct SeqStore { - /// Never grows beyond the capacity it is given by [`Self::reserve`]. + /// Never grows beyond the capacity it is given by [`Self::start_block`]. pub(crate) sequences: Vec, - /// As long as the longest block so far, and [`WILDCOPY_OVERLENGTH`] to copy in chunks, with room for `max_nb_lit`. + /// As long as the longest block so far, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. pub(crate) lit_buffer: Vec, /// How much of `lit_buffer` is literals. pub(crate) lit_len: usize, - /// As many as the most sequences that a block had so far, with room for `max_nb_seq`. + /// As many as the most sequences that a block had so far. pub(crate) codes: Vec, pub(crate) max_nb_seq: usize, pub(crate) max_nb_lit: usize, @@ -356,29 +356,52 @@ impl SeqStore { } } - /// Room for `max_nb_seq` sequences and `max_nb_lit` bytes of literals. - pub(crate) fn reserve(&mut self, max_nb_seq: usize, max_nb_lit: usize) -> Result<()> { - fn room(vec: &mut Vec, len: usize) -> Result<()> { - vec.clear(); + /// A block has at most `max_nb_seq` sequences and `max_nb_lit` bytes. + pub(crate) fn set_limits(&mut self, max_nb_seq: usize, max_nb_lit: usize) { + self.max_nb_seq = max_nb_seq; + self.max_nb_lit = max_nb_lit; + } + + /// `ZSTD_resetSeqStore`, with room for the sequences and the literals of a block of `block_size` bytes. + pub(crate) fn start_block(&mut self, block_size: usize) -> Result<()> { + fn room(vec: &mut Vec, len: usize, max: usize) -> Result<()> { + if len <= vec.capacity() { + return Ok(()); + } + let len = len.max(2 * vec.capacity()).min(max); + // What it holds is not worth a copy. + *vec = Vec::new(); vec.try_reserve_exact(len) .map_err(|_| ErrorCode::MemoryAllocation) } - room(&mut self.sequences, max_nb_seq)?; - room(&mut self.codes, max_nb_seq)?; - room(&mut self.lit_buffer, max_nb_lit + WILDCOPY_OVERLENGTH)?; - self.max_nb_seq = max_nb_seq; - self.max_nb_lit = max_nb_lit; self.reset(); + let block_size = block_size.min(self.max_nb_lit); + let nb_seq = (block_size / MINMATCH as usize).min(self.max_nb_seq); + room(&mut self.sequences, nb_seq, self.max_nb_seq)?; + let len = block_size + WILDCOPY_OVERLENGTH; + room( + &mut self.lit_buffer, + len, + self.max_nb_lit + WILDCOPY_OVERLENGTH, + )?; + if self.lit_buffer.len() < len { + self.lit_buffer.resize(len, 0); + } Ok(()) } - /// `ZSTD_resetSeqStore`, for a block of `block_size` bytes. - pub(crate) fn start_block(&mut self, block_size: usize) { - let len = block_size.min(self.max_nb_lit) + WILDCOPY_OVERLENGTH; - if self.lit_buffer.len() < len { - self.lit_buffer.resize(len, 0); + /// Makes `codes` at least as long as `sequences`. + pub(crate) fn make_room_for_codes(&mut self) -> Result<()> { + let len = self.sequences.len(); + if self.codes.capacity() < len { + self.codes = Vec::new(); + (self.codes.try_reserve_exact(self.sequences.capacity())) + .map_err(|_| ErrorCode::MemoryAllocation)?; } - self.reset(); + if self.codes.len() < len { + self.codes.resize(len, SeqCodes::default()); + } + Ok(()) } /// `ZSTD_resetSeqStore` From 5a0d69b807cd2972e7d6508a7985bc617f3300c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:57:50 +0000 Subject: [PATCH 052/199] zstd: the table sizes of levels 1 and 3 for small inputs are known as well --- src/zstd/compress/double_fast.rs | 5 ++++- src/zstd/compress/fast.rs | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 0d5312f7c7f4..b20e02c5225d 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -316,10 +316,13 @@ pub(crate) fn compress_block_double_fast( let mut sink = MatchSink::new(seq_store, &block); let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; - // First, what levels 3 and 4 come with if the input is not known to be small. + // First, what levels 3 and 4 come with if the input is not known to be small, and level 3 if it is. match (ms.c_params.min_match, h_bits_l, h_bits_s) { (5, 17, 16) => finder.find_matches::<5, 17, 16>(hash_long, hash_small, &mut sink), (5, 18, 18) => finder.find_matches::<5, 18, 18>(hash_long, hash_small, &mut sink), + (4, 16, 16) => finder.find_matches::<4, 16, 16>(hash_long, hash_small, &mut sink), + (5, 16, 15) => finder.find_matches::<5, 16, 15>(hash_long, hash_small, &mut sink), + (4, 15, 14) => finder.find_matches::<4, 15, 14>(hash_long, hash_small, &mut sink), (5, ..) => finder.find_matches::<5, 0, 0>(hash_long, hash_small, &mut sink), (6, ..) => finder.find_matches::<6, 0, 0>(hash_long, hash_small, &mut sink), (7, ..) => finder.find_matches::<7, 0, 0>(hash_long, hash_small, &mut sink), diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 337e81bec93b..b31204521a5c 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -731,11 +731,13 @@ pub(crate) fn compress_block_fast( }; let mut sink = MatchSink::new(seq_store, &block); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; - // First, what levels 1, 2 and those below 0 come with if the input is not known to be small. + // First, what levels 1, 2 and those below 0 come with if the input is not known to be small, and level 1 if it is. match (ms.c_params.min_match, hlog) { (7, 14) => finder.find_matches::<7, 14>(hash_table, &mut sink), (6, 16) => finder.find_matches::<6, 16>(hash_table, &mut sink), (6, 13) => finder.find_matches::<6, 13>(hash_table, &mut sink), + (6, 14) => finder.find_matches::<6, 14>(hash_table, &mut sink), + (5, 15) => finder.find_matches::<5, 15>(hash_table, &mut sink), (5, _) => finder.find_matches::<5, 0>(hash_table, &mut sink), (6, _) => finder.find_matches::<6, 0>(hash_table, &mut sink), (7, _) => finder.find_matches::<7, 0>(hash_table, &mut sink), From 08907965dbec3996f0a677f55af20739faa12a04 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:49:55 +0000 Subject: [PATCH 053/199] zstd: faster greedy, lazy and lazy2 The parser is one loop around one inlined search, on positions that are u32. A search at the second or third position of a lazy parse only looks for matches that are long enough to be taken, given how close the closest candidate is, so most candidates are turned down by the 4 bytes where such a match would end. Candidates in a dictionary are turned down the same way. Rows: the tags are compared as vectors, hashes of 4 to 6 bytes are one multiplication of 8 bytes, the tables start where a cache line does, a new tag table is written once rather than read and then written, and for long inputs and rows of 64 a second tag tells which candidates are shorter than 8 bytes. --- src/zstd/compress/cctx.rs | 26 +- src/zstd/compress/internal.rs | 196 ++- src/zstd/compress/lazy.rs | 2281 +++++++++++++++++++++------------ 3 files changed, 1656 insertions(+), 847 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index e4c1da17d745..507c29106579 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -12,8 +12,8 @@ use super::huf::{self, HufRepeat}; use super::internal::{ BlockSrc, BufId, Buffers, CompressedBlockState, CompressionParameters, DictMatchState, DictTableLoadMethod, EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, - MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqChunk, SeqStore, Strategy, - TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, + MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, RowTags, SeqChunk, SeqStore, Strategy, + Table, TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, min_gain, no_compress_block, offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, }; @@ -181,6 +181,7 @@ fn reset_match_state( crp: CompResetPolicy, force_reset_index: IndexResetPolicy, for_who: ResetTarget, + src_size: Option, ) -> Result<()> { let sizes = MatchStateSizes::new(c_params, use_row_match_finder, for_who == ResetTarget::CCtx); @@ -188,28 +189,34 @@ fn reset_match_state( ms.window = Window::new(); } ms.hash_log3 = sizes.hash_log3; - ms.lazy_skipping = false; invalidate_match_state(ms); // As long as indices go on, the ones in the tables are too low to be valid. let clean = force_reset_index == IndexResetPolicy::Reset && crp != CompResetPolicy::LeaveDirty; - reset_table(&mut ms.hash_table, sizes.h_size, clean)?; + let use_rows = row_match_finder_used(c_params.strat(), use_row_match_finder); + // An entry of a row is written before it is read. + ms.hash_table.reset(sizes.h_size, clean, !use_rows)?; reset_table(&mut ms.chain_table, sizes.chain_size, clean)?; reset_table(&mut ms.hash_table3, sizes.h3_size, clean)?; - if row_match_finder_used(c_params.strat(), use_row_match_finder) { + if use_rows { + ms.row_tags = match for_who { + ResetTarget::CCtx => lazy::row_tags_at_start(c_params, src_size), + ResetTarget::CDict => RowTags::Flat, + }; + let tag_table_size = lazy::row_tag_table_size(ms.row_tags, sizes.h_size); if for_who == ResetTarget::CCtx { // A new salt instead of erasing the tags. - reset_table(&mut ms.tag_table, sizes.h_size, false)?; + ms.tag_table.reset(tag_table_size, false, true)?; ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(u64::from(ms.hash_salt_entropy), 4); } else { - reset_table(&mut ms.tag_table, sizes.h_size, true)?; + ms.tag_table.reset(tag_table_size, true, true)?; ms.hash_salt = 0; } let row_log = c_params.search_log.clamp(4, 6); ms.row_hash_log = c_params.hash_log - row_log; } else { - ms.tag_table = Vec::new(); + ms.tag_table = Table::new(); } if for_who == ResetTarget::CCtx && c_params.strat() >= Strategy::BtOpt { @@ -536,6 +543,7 @@ impl CDict { CompResetPolicy::MakeClean, IndexResetPolicy::Reset, ResetTarget::CDict, + None, )?; let bufs = Buffers { in_buff: &[], @@ -928,6 +936,7 @@ impl FrameCompressor { crp, needs_index_reset, ResetTarget::CCtx, + (pledged_src_size != ZSTD_CONTENTSIZE_UNKNOWN).then_some(pledged_src_size), )?; if params.ldm_params.enable_ldm == ParamSwitch::Enable { @@ -1047,6 +1056,7 @@ impl FrameCompressor { copy_table(&mut dst.chain_table, &src.chain_table, tagged)?; if row_match_finder_used(cdict_c_params.strat(), cdict.use_row_match_finder) { dst.tag_table.write_at(0, &src.tag_table)?; + dst.row_tags = RowTags::Flat; dst.hash_salt = src.hash_salt; } dst.hash_table3.fill(0); diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index ffd4868daab7..680e5322fcd2 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -7,7 +7,7 @@ use crate::common::{ ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; use crate::error::{ErrorCode, Result}; -use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64, write_le24}; +use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64, try_zeroed, write_le24}; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; pub(crate) const HASH_READ_SIZE: usize = 8; @@ -675,6 +675,19 @@ impl<'a> Near<'a> { Some(u32::from_le_bytes(*bytes)) } + /// [`Self::read32`] at a position that may not fit in 32 bits, but is far from the most a `usize` can be. + #[inline(always)] + pub(crate) fn read32_at(self, pos: usize) -> Option { + let bytes = self.0.get(pos..pos + 4)?.first_chunk()?; + Some(u32::from_le_bytes(*bytes)) + } + + /// What there is from `start` on. + #[inline(always)] + pub(crate) fn tail(self, start: u32) -> Option { + self.0.get(start as usize..).map(Self) + } + #[inline(always)] pub(crate) fn read64(self, pos: u32) -> Option { let pos = pos as usize; @@ -715,6 +728,35 @@ impl<'a> Near<'a> { len } + /// How many of the 16 bytes at `match_pos` are the same as `bytes`, from the first on. It takes no branch. + #[inline(always)] + fn count16(self, bytes: &[u8; 16], match_pos: u32) -> Option { + let mut same = 0u16; + for (i, (byte, match_byte)) in bytes.iter().zip(self.chunk16(match_pos)?).enumerate() { + same |= u16::from(byte == match_byte) << i; + } + Some((!u32::from(same)).trailing_zeros()) + } + + /// [`Self::count`], of which only the comparison of the first 16 bytes is inlined. It takes few registers of a loop + /// that has other uses for them. + #[inline(always)] + pub(crate) fn count_short(self, pos: u32, match_pos: u32) -> u32 { + match self + .chunk16(pos) + .and_then(|bytes| self.count16(bytes, match_pos)) + { + None => self.count_outlined(pos, match_pos), + Some(16) => 16 + self.count_outlined(pos + 16, match_pos + 16), + Some(len) => len, + } + } + + #[inline(never)] + fn count_outlined(self, pos: u32, match_pos: u32) -> u32 { + self.count(pos, match_pos) + } + /// How many bytes before `pos`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. #[inline(always)] pub(crate) fn count_backwards( @@ -862,13 +904,7 @@ pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result< /// `ZSTD_hashPtrSalted` #[inline(always)] -pub(crate) fn hash_ptr_salted( - buf: &[u8], - pos: usize, - h_bits: u32, - mls: u32, - hash_salt: u64, -) -> Result { +fn hash_ptr_salted(buf: &[u8], pos: usize, h_bits: u32, mls: u32, hash_salt: u64) -> Result { Ok(match mls { 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), @@ -878,6 +914,44 @@ pub(crate) fn hash_ptr_salted( }) } +/// [`hash_ptr_salted`] of 4 to 6 bytes, out of the 8 bytes at the position: the multiplication also does the shift that +/// leaves the other bytes out. +#[derive(Clone, Copy)] +pub(crate) struct Hash8 { + mul: u64, + salt: u64, + shift: u32, +} + +impl Hash8 { + /// `mls` is 4 if it is not 5 or 6, and `h_bits` is from 1 to 32. + pub(crate) const fn new(mls: u32, h_bits: u32, hash_salt: u64) -> Self { + let shift = 64 - h_bits; + match mls { + 5 => Self { + mul: PRIME_5_BYTES << (64 - 40), + salt: hash_salt, + shift, + }, + 6 => Self { + mul: PRIME_6_BYTES << (64 - 48), + salt: hash_salt, + shift, + }, + _ => Self { + mul: (PRIME_4_BYTES as u64) << 32, + salt: hash_salt << 32, + shift, + }, + } + } + + #[inline(always)] + pub(crate) const fn of(self, bytes: u64) -> u32 { + (bytes.wrapping_mul(self.mul) ^ self.salt).wrapping_shr(self.shift) as u32 + } +} + // ── the window ──────────────────────────────────────────────────────────── /// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are @@ -1225,6 +1299,98 @@ impl OptState { pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; +/// A table that starts where a cache line does, as those of the C code do: a row of the row-based match finder is then +/// in as few cache lines as can be. +pub(crate) struct Table { + /// Longer than the table, by what it takes to have such a start. + storage: Vec, + start: usize, + len: usize, +} + +impl Table { + const CACHE_LINE: usize = 64; + + pub(crate) const fn new() -> Self { + Self { + storage: Vec::new(), + start: 0, + len: 0, + } + } + + #[inline] + pub(crate) fn as_slice(&self) -> &[T] { + let table = self.storage.get(self.start..self.start + self.len); + table.unwrap_or_default() + } + + #[inline] + pub(crate) fn as_mut_slice(&mut self) -> &mut [T] { + let table = self.storage.get_mut(self.start..self.start + self.len); + table.unwrap_or_default() + } +} + +impl Table { + /// Gives it the length `len`. What it holds is 0, or if `clean` is false possibly what it held before. + /// + /// `read_first` tells that most of it is to be read before it is written. New memory is then written to right away: + /// else the system would provide each page twice, once to be read and once to be written. + pub(crate) fn reset(&mut self, len: usize, clean: bool, read_first: bool) -> Result<()> { + let is_new = self.len != len; + if is_new { + // First, so that both are not allocated at once. + *self = Self::new(); + let slack = Self::CACHE_LINE / size_of::(); + let storage = try_zeroed::(len + slack)?; + let start = storage.as_ptr().align_offset(Self::CACHE_LINE); + *self = Self { + storage, + start: if start <= slack { start } else { 0 }, + len, + }; + } + if if is_new { read_first } else { clean } { + self.fill(T::default()); + } + Ok(()) + } +} + +impl core::ops::Deref for Table { + type Target = [T]; + + #[inline] + fn deref(&self) -> &[T] { + self.as_slice() + } +} + +impl core::ops::DerefMut for Table { + #[inline] + fn deref_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + +/// What the second tags of the row-based match finder have been good for, in a frame. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct SecondTagStats { + pub(crate) nb_positions: u64, + /// How many candidates they ruled out. + pub(crate) nb_ruled_out: u64, +} + +/// How `MatchState::tag_table` is laid out. See `lazy.rs`. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum RowTags { + /// As in the C code. + Flat, + /// The tags of each row are followed by its second tags. + Second(SecondTagStats), +} + /// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures /// own, are in [`BlockSrc`]. pub(crate) struct MatchState { @@ -1237,12 +1403,12 @@ pub(crate) struct MatchState { pub(crate) hash_log3: u32, pub(crate) row_hash_log: u32, /// For the row-based match finder: a byte of hash for each entry of `hash_table`. - pub(crate) tag_table: Vec, - pub(crate) hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], + pub(crate) tag_table: Table, + pub(crate) row_tags: RowTags, /// Makes what a previous frame left in `tag_table` meaningless. pub(crate) hash_salt: u64, pub(crate) hash_salt_entropy: u32, - pub(crate) hash_table: Vec, + pub(crate) hash_table: Table, pub(crate) hash_table3: Vec, pub(crate) chain_table: Vec, pub(crate) force_non_contiguous: bool, @@ -1250,7 +1416,6 @@ pub(crate) struct MatchState { /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. pub(crate) has_dict_match_state: bool, pub(crate) c_params: CompressionParameters, - pub(crate) lazy_skipping: bool, } impl MatchState { @@ -1261,11 +1426,11 @@ impl MatchState { next_to_update: 0, hash_log3: 0, row_hash_log: 0, - tag_table: Vec::new(), - hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + tag_table: Table::new(), + row_tags: RowTags::Flat, hash_salt: 0, hash_salt_entropy: 0, - hash_table: Vec::new(), + hash_table: Table::new(), hash_table3: Vec::new(), chain_table: Vec::new(), force_non_contiguous: false, @@ -1280,7 +1445,6 @@ impl MatchState { target_length: 0, strategy: None, }, - lazy_skipping: false, } } diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 0696bd6dc8ff..7975d8982c3a 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -1,18 +1,20 @@ //! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. use core::hint::{Locality, prefetch_read}; +use core::num::NonZeroU32; +use core::ops::Range; use super::internal::{ - BlockSrc, DictMatchState, DictMode, K_SEARCH_STRENGTH, MatchState, Near, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, - count, count_2segments, hash_ptr, hash_ptr_salted, index_overlap_check, offbase_is_offset, - offbase_to_offset, offset_to_offbase, + BlockSrc, CompressionParameters, DictMatchState, DictMode, Hash8, K_SEARCH_STRENGTH, + MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, RowTags, SecondTagStats, SeqSink, SeqStore, + Strategy, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, count, count_2segments, hash_ptr, + index_overlap_check, offbase_is_offset, offbase_to_offset, offset_to_offbase, table_get, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32}; +use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64}; -const K_LAZY_SKIPPING_STEP: usize = 8; +const K_LAZY_SKIPPING_STEP: u32 = 8; // `ZSTD_dictMode_e`, as an argument of a const generic. const NO_DICT: u8 = DictMode::NoDict as u8; @@ -453,18 +455,382 @@ fn bt_find_best_match( dubt_find_best_match::(ms, src, ip, i_limit, off_base_ptr, MLS) } +// ── what the match finders share ────────────────────────────────────────── + +/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match +/// state is searched. +#[derive(Clone, Copy)] +struct DictSegment<'a> { + /// All of it, and nothing else. + bytes: &'a [u8], + /// The index of its first byte. + start_index: u32, +} + +/// `ZSTD_getLowestMatchIndex`, with what it reads from the match state. +#[derive(Clone, Copy)] +struct Reach { + lowest_valid: u32, + /// More than any distance if there is a dictionary, which is valid as a whole until it is invalidated as a whole. + max_distance: u32, +} + +impl Reach { + const fn new(ms: &MatchState) -> Self { + Self { + lowest_valid: ms.window.low_limit, + max_distance: if ms.loaded_dict_end != 0 { + u32::MAX + } else { + 1 << ms.c_params.window_log + }, + } + } + + /// The lowest index a match for the position `curr` can be at. + #[inline(always)] + const fn low_limit(self, curr: u32) -> u32 { + if curr.wrapping_sub(self.lowest_valid) > self.max_distance { + curr - self.max_distance + } else { + self.lowest_valid + } + } +} + +/// The block that a parser works on. Positions are counted from the start of the current segment of the window. +#[derive(Clone, Copy)] +struct LazyBlock<'a> { + /// The current segment, up to the end of the block. + near: Near<'a>, + /// The index of the position 0. + base_index: u32, + /// Where the block starts. + istart: u32, + dict: DictSegment<'a>, + reach: Reach, +} + +impl<'a> LazyBlock<'a> { + fn new(ms: &MatchState, src: &BlockSrc<'a>, dict_mode: DictMode) -> Result { + let base_index = ms.window.dict_limit; + let prefix_start = src.pos_of(base_index); + let dict = match dict_mode { + DictMode::NoDict => DictSegment { + bytes: &[], + start_index: base_index, + }, + DictMode::ExtDict => { + let start_index = ms.window.low_limit; + let segment = src.dict_pos_of(start_index)..src.dict_pos_of(base_index); + DictSegment { + bytes: src.dict_buf.sub(segment)?, + start_index, + } + } + DictMode::DictMatchState => { + let dms = src.dms.ok_or(BUG)?; + let bytes = dms.buf.sub(dms.pos_of(dms.ms.window.dict_limit)..dms.end)?; + DictSegment { + bytes, + start_index: base_index.wrapping_sub(bytes.len() as u32), + } + } + }; + Ok(Self { + near: Near::new(src.buf.sub(prefix_start..src.end)?)?, + base_index, + istart: (src.start - prefix_start) as u32, + dict, + reach: Reach::new(ms), + }) + } + + /// `ZSTD_count_2segments`: how long a match is that starts at `match_pos` in the dictionary. + #[inline(never)] + fn count_from_dict(&self, ip: u32, match_pos: usize) -> Result { + let bytes = self.near.bytes(); + let dict = self.dict.bytes; + let len = count_2segments( + bytes, + ip as usize, + dict, + match_pos, + bytes.len(), + dict.len(), + 0, + )?; + Ok(len as u32) + } +} + +/// A match. +#[derive(Clone, Copy)] +struct Found { + len: u32, + off_base: NonZeroU32, +} + +impl Found { + /// With the first repeat offset. + #[inline(always)] + const fn repeat(len: u32) -> Self { + Self { + len, + off_base: NonZeroU32::MIN, + } + } + + /// What the offset costs, in what the parsers compare matches by. + #[inline(always)] + const fn cost(self) -> i32 { + highbit32(self.off_base.get()) as i32 + } + + /// What the parsers compare matches by. + #[inline(always)] + const fn gain(self) -> i32 { + (self.len * 4) as i32 - self.cost() + } +} + +/// What a match has to be like for the parser to take it. +#[derive(Clone, Copy)] +struct Wanted { + /// It is longer than this, which is at least 3. + longer_than: u32, + /// Its [`Found::gain`] is more than this. + gain: i32, +} + +impl Wanted { + /// Less than the gain of any match. + const ANY_GAIN: i32 = i32::MIN / 2; + + /// What a match has to be longer than, given that no candidate is at a higher index than `closest`. `BY_GAIN` is + /// whether `gain` may be something else than [`Self::ANY_GAIN`]. + #[inline(always)] + fn floor(self, curr: u32, closest: u32) -> u32 { + if !BY_GAIN { + return self.longer_than; + } + let cost = highbit32(offset_to_offbase(curr.wrapping_sub(closest))) as i32; + ((self.gain + cost) >> 2).max(self.longer_than as i32) as u32 + } +} + +/// What became of a candidate. +#[derive(PartialEq, Eq)] +enum Offered { + /// It is not in the segment. + Outside, + Considered, + /// It goes up to the end of the block: there is nothing longer. + Longest, +} + +/// What a match has to be longer than, which is at least 3: the best one so far, if there is one. +#[derive(Clone, Copy)] +struct Best { + len: u32, + /// 0 if there is no match. + off_base: u32, +} + +impl Best { + #[inline(always)] + fn found(self) -> Option { + Some(Found { + len: self.len, + off_base: NonZeroU32::new(self.off_base)?, + }) + } +} + +/// The longest match so far for a position, while candidates are compared with it. +struct Longest<'a> { + /// The current segment, from the lowest index that a match can be at or from its start. + segment: Near<'a>, + /// The index of its first byte. + start_index: u32, + /// The position that matches are for, in `segment`. + ip: u32, + /// A match has to be more than 3 bytes longer than this. + tail_start: u32, + /// What such a match has in common with `ip`: the 4 bytes from `tail_start` on. + tail: u32, + /// 0 as long as there is no match. + off_base: u32, +} + +impl<'a> Longest<'a> { + /// `None` if there is nothing after `best.len` bytes, up to the end of the block. + #[inline(always)] + fn new(segment: Near<'a>, start_index: u32, curr: u32, best: Best) -> Option { + let ip = curr.wrapping_sub(start_index); + let tail_start = best.len - 3; + Some(Self { + segment, + start_index, + ip, + tail_start, + tail: segment.read32(ip + tail_start)?, + off_base: best.off_base, + }) + } + + /// There is a match of `len` bytes. + #[inline(always)] + fn offer_match(&mut self, len: u32, off_base: u32) -> Offered { + if len > self.tail_start + 3 { + self.tail_start = len - 3; + self.off_base = off_base; + let Some(tail) = self.segment.read32(self.ip + self.tail_start) else { + return Offered::Longest; + }; + self.tail = tail; + } + Offered::Considered + } + + /// There may be a match for the position `ip` of `block` at `match_pos` in its dictionary. + #[inline(always)] + fn offer_in_dict( + &mut self, + block: &LazyBlock<'_>, + ip: u32, + match_pos: usize, + ) -> Result { + // Where the match would go on in the current segment, it takes more to tell. + let tail_pos = match_pos + self.tail_start as usize; + let tail = block.dict.bytes.get(tail_pos..tail_pos + 4); + if let Some(tail) = tail.and_then(<[u8]>::first_chunk) + && u32::from_le_bytes(*tail) != self.tail + { + return Ok(Offered::Considered); + } + let len = block.count_from_dict(ip, match_pos)?; + let dict_len = block.base_index.wrapping_sub(block.dict.start_index); + let offset = (ip + dict_len).wrapping_sub(match_pos as u32); + Ok(self.offer_match(len, offset_to_offbase(offset))) + } + + /// There may be a match at `match_index`. + #[inline(always)] + fn offer(&mut self, match_index: u32) -> Offered { + let match_pos = match_index.wrapping_sub(self.start_index); + let tail_pos = match_pos as usize + self.tail_start as usize; + let Some(tail) = self.segment.read32_at(tail_pos) else { + return Offered::Outside; + }; + if tail != self.tail { + return Offered::Considered; + } + let len = self.segment.count_short(self.ip, match_pos); + self.offer_match(len, offset_to_offbase(self.ip.wrapping_sub(match_pos))) + } + + #[inline(always)] + const fn best(&self) -> Best { + Best { + len: self.tail_start + 3, + off_base: self.off_base, + } + } +} + +/// `ZSTD_searchMax`, with the state of the search in local variables for as long as a block lasts. +trait Finder<'a>: Sized { + /// How far from the end of the block the last search is. + const MARGIN: u32; + + /// `block` is at least `MARGIN` bytes. + fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result; + + /// The longest match for `ip`, the most recent one of that length, unless it is not as `wanted`. Then it may be + /// `None` just as well. Positions are searched in increasing order. While `skipping`, those in between are not + /// indexed. + /// `BY_GAIN` is whether `wanted.gain` may be something else than [`Wanted::ANY_GAIN`]. + fn find( + &mut self, + ip: u32, + wanted: Wanted, + skipping: bool, + ) -> Result>; + + /// Every position is indexed again from now on. + fn stop_skipping(&mut self) -> Result<()>; + + /// The end of the block. + fn finish(self) -> Result<()>; +} + +// ── binary trees, for the parser ────────────────────────────────────────── + +struct TreeFinder<'a, const DICT_MODE: u8> { + ms: &'a mut MatchState, + src: &'a BlockSrc<'a>, + /// Where the position 0 is in `src.buf`. + prefix_start: usize, + mls: u32, +} + +impl<'a, const DICT_MODE: u8> Finder<'a> for TreeFinder<'a, DICT_MODE> { + const MARGIN: u32 = 8; + + #[inline(always)] + fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result { + Ok(Self { + mls: ms.c_params.min_match.clamp(4, 6), + prefix_start: src.pos_of(block.base_index), + ms, + src, + }) + } + + #[inline(always)] + fn find( + &mut self, + ip: u32, + wanted: Wanted, + _skipping: bool, + ) -> Result> { + let ip = self.prefix_start + ip as usize; + let iend = self.src.end; + let mut off_base = 999_999_999u32; + let len = match self.mls { + 4 => bt_find_best_match::<4, DICT_MODE>(self.ms, self.src, ip, iend, &mut off_base), + 5 => bt_find_best_match::<5, DICT_MODE>(self.ms, self.src, ip, iend, &mut off_base), + _ => bt_find_best_match::<6, DICT_MODE>(self.ms, self.src, ip, iend, &mut off_base), + }? as u32; + if len <= wanted.longer_than { + return Ok(None); + } + Ok(NonZeroU32::new(off_base).map(|off_base| Found { len, off_base })) + } + + #[inline(always)] + fn stop_skipping(&mut self) -> Result<()> { + Ok(()) + } + + #[inline(always)] + fn finish(self) -> Result<()> { + Ok(()) + } +} + // ── hash chains ─────────────────────────────────────────────────────────── -/// `ZSTD_insertAndFindFirstIndex_internal` -#[inline(always)] -fn insert_and_find_first_index_internal( +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. +pub(crate) fn insert_and_find_first_index( ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, - mls: u32, - lazy_skipping: bool, -) -> Result { +) -> Result<()> { let hash_log = ms.c_params.hash_log; + let mls = ms.c_params.min_match; let chain_mask = (1u32 << ms.c_params.chain_log) - 1; let target = prefix.index_of(ip); let hash_table = ms.hash_table.as_mut_slice(); @@ -474,161 +840,207 @@ fn insert_and_find_first_index_internal( let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); chain_table.put((idx & chain_mask) as usize, next_in_chain)?; - // While skipping, not every position is inserted. - if lazy_skipping { - break; - } } ms.next_to_update = target; - hash_table.at(hash_ptr(prefix.buf, ip, hash_log, mls)?) -} - -/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. Returns the index of the first candidate for `ip`. -pub(crate) fn insert_and_find_first_index( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, -) -> Result { - insert_and_find_first_index_internal(ms, prefix, ip, ms.c_params.min_match, false) + Ok(()) } -/// What [`hc_find_best_match`] works out about the position it searches for. -struct ChainSearch { - chain_mask: u32, - dict_limit: u32, - prefix_start: usize, - dict_end: usize, - curr: u32, - low_limit: u32, +/// The hash chains of a dictionary. +#[derive(Clone, Copy)] +struct ChainDict<'a> { + hash_table: &'a [u32], + chain_table: &'a [u32], + hasher: Hash8, + /// The index, among its own, of its first byte. + lowest_index: u32, + /// Below this, an index is no longer in the chain table. min_chain: u32, } -impl ChainSearch { - #[inline(always)] - fn new(ms: &MatchState, src: &BlockSrc<'_>, ip: usize) -> Self { - let chain_size = 1u32 << ms.c_params.chain_log; - let dict_limit = ms.window.dict_limit; - let curr = src.index_of(ip); +impl<'a> ChainDict<'a> { + fn new(dms: &DictMatchState<'a>, mls: u32) -> Self { Self { - chain_mask: chain_size - 1, - dict_limit, - prefix_start: src.pos_of(dict_limit), - dict_end: src.dict_pos_of(dict_limit), - curr, - low_limit: ms.get_lowest_match_index(curr, ms.c_params.window_log), - min_chain: curr.saturating_sub(chain_size), + hash_table: &dms.ms.hash_table, + chain_table: &dms.ms.chain_table, + hasher: Hash8::new(mls, dms.ms.c_params.hash_log, 0), + lowest_index: dms.ms.window.dict_limit, + min_chain: dms_end_index(dms).saturating_sub(1 << dms.ms.c_params.chain_log), } } } -/// `ZSTD_HcFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn hc_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let search = ChainSearch::new(ms, src, ip); - let mut nb_attempts = 1u32 << ms.c_params.search_log; - let mut ml = 4 - 1; - - let mut match_index = - insert_and_find_first_index_internal(ms, src.prefix(), ip, MLS, ms.lazy_skipping)?; - let chain_table = ms.chain_table.as_slice(); - - while match_index >= search.low_limit && nb_attempts > 0 { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= search.dict_limit { - let match_pos = src.pos_of(match_index); - // Whether it can be longer. - if read_le32(buf, match_pos + ml - 3)? == read_le32(buf, ip + ml - 3)? { - current_ml = count(buf, ip, buf, match_pos, i_limit)?; - } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - search.dict_end, - search.prefix_start, - )? + 4; - } +/// The hash table and the chain table. +struct Chains<'a> { + hash_table: &'a mut [u32], + chain_table: &'a mut [u32], +} + +impl Chains<'_> { + /// Adds the position that `bytes` is the 8 bytes of. + #[inline(always)] + fn insert(&mut self, hasher: Hash8, bytes: u64, index: u32) { + let hash = hasher.of(bytes) as usize & self.hash_table.len().wrapping_sub(1); + let in_chain = index as usize & self.chain_table.len().wrapping_sub(1); + if let Some(first) = self.hash_table.get_mut(hash) + && let Some(next) = self.chain_table.get_mut(in_chain) + { + *next = core::mem::replace(first, index); } + } - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(search.curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } + /// Adds every position that has 8 bytes in `positions`. The first one has the index `index`. + #[inline(never)] + fn insert_all(mut self, hasher: Hash8, positions: &[u8], mut index: u32) { + for bytes in positions.array_windows() { + self.insert(hasher, u64::from_le_bytes(*bytes), index); + index = index.wrapping_add(1); } + } +} - if match_index <= search.min_chain { - break; +/// `ZSTD_HcFindBestMatch` +struct ChainFinder<'a, const DICT_MODE: u8> { + block: LazyBlock<'a>, + chains: Chains<'a>, + hasher: Hash8, + nb_attempts: u32, + /// `ms.next_to_update`, as a position. + next_to_update: u32, + ms_next_to_update: &'a mut u32, + dms: Option>, +} + +impl ChainFinder<'_, DICT_MODE> { + /// `ZSTD_insertAndFindFirstIndex_internal`: returns the index of the first candidate for `ip`. + #[inline(always)] + fn insert_and_find_first_index(&mut self, ip: u32, skipping: bool) -> Result { + let near = self.block.near; + let first = self.next_to_update; + let index = self.block.base_index.wrapping_add(first); + // While skipping, not every position is inserted. + if first.wrapping_add(1) == ip || (skipping && first < ip) { + let bytes = near.read64(first).ok_or(BUG)?; + self.chains.insert(self.hasher, bytes, index); + } else if first < ip { + let positions = near.bytes().sub(first as usize..ip as usize + 7)?; + let chains = Chains { + hash_table: self.chains.hash_table, + chain_table: self.chains.chain_table, + }; + chains.insert_all(self.hasher, positions, index); } - match_index = chain_table.at((match_index & search.chain_mask) as usize)?; - nb_attempts -= 1; - } - - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_chain_table = dms.ms.chain_table.as_slice(); - let dms_chain_size = 1u32 << dms.ms.c_params.chain_log; - let dms_chain_mask = dms_chain_size - 1; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_size = dms_end_index(&dms); - let dms_index_delta = search.dict_limit.wrapping_sub(dms_size); - let dms_min_chain = dms_size.saturating_sub(dms_chain_size); - - let h = hash_ptr(buf, ip, dms.ms.c_params.hash_log, MLS)?; - match_index = dms.ms.hash_table.at(h)?; - - while match_index >= dms_lowest_index && nb_attempts > 0 { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - search.prefix_start, - )? + 4; - } + self.next_to_update = ip; + let hash = self.hasher.of(near.read64(ip).ok_or(BUG)?) as usize; + Ok(table_get(self.chains.hash_table, hash)) + } +} + +impl<'a, const DICT_MODE: u8> Finder<'a> for ChainFinder<'a, DICT_MODE> { + const MARGIN: u32 = 8; + + #[inline(always)] + fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result { + let mls = ms.c_params.min_match.clamp(4, 6); + Ok(Self { + block, + chains: Chains { + hash_table: &mut ms.hash_table, + chain_table: &mut ms.chain_table, + }, + hasher: Hash8::new(mls, ms.c_params.hash_log, 0), + nb_attempts: 1 << ms.c_params.search_log, + next_to_update: ms.next_to_update.wrapping_sub(block.base_index), + ms_next_to_update: &mut ms.next_to_update, + dms: src.dms.as_ref().map(|dms| ChainDict::new(dms, mls)), + }) + } - if current_ml > ml { - ml = current_ml; - let offset = search - .curr - .wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { + #[inline(always)] + fn find( + &mut self, + ip: u32, + wanted: Wanted, + skipping: bool, + ) -> Result> { + let block = self.block; + let curr = block.base_index.wrapping_add(ip); + let low_limit = block.reach.low_limit(curr); + let min_chain = curr.saturating_sub(self.chains.chain_table.len() as u32); + let mut nb_attempts = self.nb_attempts; + let mut match_index = self.insert_and_find_first_index(ip, skipping)?; + if DICT_MODE == NO_DICT && match_index < low_limit { + return Ok(None); + } + + let closest = if DICT_MODE == DICT_MATCH_STATE { + match_index.max(low_limit) + } else { + match_index + }; + let start_index = if DICT_MODE == EXT_DICT { + low_limit.max(block.base_index) + } else { + low_limit + }; + let segment = block + .near + .tail(start_index.wrapping_sub(block.base_index)) + .ok_or(BUG)?; + let best = Best { + len: wanted.floor::(curr, closest), + off_base: 0, + }; + let Some(mut longest) = Longest::new(segment, start_index, curr, best) else { + return Ok(None); + }; + while nb_attempts > 0 { + let mut offered = longest.offer(match_index); + if offered == Offered::Outside { + if DICT_MODE != EXT_DICT || match_index < low_limit { break; } + let match_pos = (match_index - block.dict.start_index) as usize; + offered = longest.offer_in_dict(&block, ip, match_pos)?; } - - if match_index <= dms_min_chain { + if offered == Offered::Longest || match_index <= min_chain { break; } - match_index = dms_chain_table.at((match_index & dms_chain_mask) as usize)?; + match_index = table_get(self.chains.chain_table, match_index as usize); nb_attempts -= 1; } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = self.dms.ok_or(BUG)?; + let hash = dms.hasher.of(block.near.read64(ip).ok_or(BUG)?) as usize; + match_index = dms.hash_table.at(hash)?; + + while match_index >= dms.lowest_index && nb_attempts > 0 { + let match_pos = (match_index - dms.lowest_index) as usize; + if longest.offer_in_dict(&block, ip, match_pos)? == Offered::Longest { + break; + } + if match_index <= dms.min_chain { + break; + } + match_index = table_get(dms.chain_table, match_index as usize); + nb_attempts -= 1; + } + } + + Ok(longest.best().found()) } - Ok(ml) + #[inline(always)] + fn stop_skipping(&mut self) -> Result<()> { + Ok(()) + } + + #[inline(always)] + fn finish(self) -> Result<()> { + *self.ms_next_to_update = self.block.base_index.wrapping_add(self.next_to_update); + Ok(()) + } } // ── row-based match finder ──────────────────────────────────────────────── @@ -636,199 +1048,283 @@ fn hc_find_best_match( // The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits // of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag // table is the "head": which entry was written last. +// +// Unlike that of the C code, the tag table may have a second byte for each entry, after the tags of the row: a hash of +// the bytes 4 to 7 of the position. An entry that has another one than the position that is searched for is a match of +// less than 8 bytes, which is often enough to know without reading what it leads to. pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; -const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1; const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; -/// `ZSTD_row_nextIndex`: moves the head of a row, and returns it: the entry to write to. -#[inline(always)] -fn row_next_index(head: &mut u8, row_mask: u32) -> u32 { - let mut next = u32::from(*head).wrapping_sub(1) & row_mask; - if next == 0 { - next = row_mask; +/// How long a match is at least, if it has the same second tag for a reason. +const SECOND_TAG_LENGTH: u32 = 8; + +/// How the tag table is laid out at the start of a frame of `src_size` bytes, if that is known. +pub(crate) const fn row_tags_at_start( + c_params: &CompressionParameters, + src_size: Option, +) -> RowTags { + // In shorter rows the candidates are few, and the second tags cost more than they save. As does the memory for + // them, for a short input. + match src_size { + Some(src_size) if c_params.search_log >= 6 && src_size >> c_params.hash_log != 0 => { + RowTags::Second(SecondTagStats { + nb_positions: 0, + nb_ruled_out: 0, + }) + } + _ => RowTags::Flat, } - *head = next as u8; - next } -/// `ZSTD_row_prefetch`: of the row that starts at `rel_row`. -#[inline(always)] -fn row_prefetch(hash_table: &[u32], tag_table: &[u8], rel_row: usize, row_log: u32) { - prefetch_l1(hash_table, rel_row); - if row_log >= 5 { - prefetch_l1(hash_table, rel_row + 16); - } - prefetch_l1(tag_table, rel_row); - if row_log == 6 { - prefetch_l1(tag_table, rel_row + 32); +/// The size of the tag table that goes with a hash table of `h_size` entries. +pub(crate) const fn row_tag_table_size(tags: RowTags, h_size: usize) -> usize { + match tags { + RowTags::Flat => h_size, + RowTags::Second(_) => 2 * h_size, } } -/// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `idx` on, or fewer so as not to go past `i_limit`. -#[inline(always)] -fn row_fill_hash_cache( - ms: &mut MatchState, - prefix: Prefix<'_>, - row_log: u32, - mls: u32, - idx: u32, - i_limit: usize, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let max_elems = (i_limit + 1).saturating_sub(prefix.pos_of(idx)); - let lim = idx + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE) as u32; - - for idx in idx..lim { - let hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - ms.hash_salt, - )? as u32; - let row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - row_prefetch(&ms.hash_table, &ms.tag_table, row, row_log); - let cached = (idx & ZSTD_ROW_HASH_CACHE_MASK) as usize; - ms.hash_cache.put(cached, hash)?; +/// Gives up the second tags, if there are: the tags of each row move to where they are when there are none. +fn row_flatten_tags(layout: &mut RowTags, tag_table: &mut [u8]) -> Result<()> { + const ROW_ENTRIES: usize = 64; + if core::mem::replace(layout, RowTags::Flat) == RowTags::Flat { + return Ok(()); + } + for row in 1..tag_table.len() / (2 * ROW_ENTRIES) { + tag_table.move_within(2 * ROW_ENTRIES * row, ROW_ENTRIES * row, ROW_ENTRIES)?; } Ok(()) } -/// `ZSTD_row_nextCachedHash`: the hash of the position `idx`, from the cache, where the hash of the position 8 bytes -/// further takes its place. +/// `ZSTD_row_nextIndex`, for every byte that a head can be. +const fn row_next_heads(row_entries: usize) -> [u8; 256] { + let row_mask = row_entries - 1; + let mut next_heads = [0; 256]; + let mut rest: &mut [u8] = &mut next_heads; + let mut head = 0usize; + while let Some((next_head, others)) = rest.split_first_mut() { + let next = head.wrapping_sub(1) & row_mask; + *next_head = if next == 0 { row_mask } else { next } as u8; + rest = others; + head += 1; + } + next_heads +} + +/// What tells where a position goes in the tables. +#[derive(Clone, Copy)] +struct RowHash { + /// The number of the row, and below it the tag. + hash: u32, + /// 0 if there are none. + second_tag: u8, +} + +/// What it takes to hash a position. +#[derive(Clone, Copy)] +struct RowHasher(Hash8); + +impl RowHasher { + /// The hash of the position that `bytes` is the 8 bytes of. + #[inline(always)] + const fn of(self, bytes: u64) -> RowHash { + RowHash { + hash: self.0.of(bytes), + // The tag depends on the first bytes already. + second_tag: if SECOND_TAGS { + (((bytes >> 32) as u32).wrapping_mul(2_654_435_761) >> 24) as u8 + } else { + 0 + }, + } + } +} + +/// A row of the tables. +struct Row<'a, const ROW_ENTRIES: usize> { + indices: &'a mut [u32; ROW_ENTRIES], + tags: &'a mut [u8; ROW_ENTRIES], + /// If they are in use. + second_tags: Option<&'a mut [u8; ROW_ENTRIES]>, +} + +impl Row<'_, ROW_ENTRIES> { + const NEXT_HEADS: [u8; 256] = row_next_heads(ROW_ENTRIES); + + /// Moves the head, and writes the entry that it is at. + #[inline(always)] + fn insert(&mut self, hash: RowHash, index: u32) { + let Some(head) = self.tags.first_mut() else { + return; + }; + let next = Self::NEXT_HEADS.get(usize::from(*head)).copied(); + let next = next.unwrap_or(1); + *head = next; + let next = usize::from(next) & (ROW_ENTRIES - 1); + if let Some(tag) = self.tags.get_mut(next) { + *tag = hash.hash as u8; + } + if let Some(second_tags) = &mut self.second_tags + && let Some(second_tag) = second_tags.get_mut(next) + { + *second_tag = hash.second_tag; + } + if let Some(entry) = self.indices.get_mut(next) { + *entry = index; + } + } +} + +/// Where the row for `hash` starts in the hash table. #[inline(always)] -fn row_next_cached_hash( - cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], - hash_table: &[u32], - tag_table: &[u8], - prefix: Prefix<'_>, - idx: u32, - hash_log: u32, - row_log: u32, - mls: u32, - hash_salt: u64, -) -> Result { - let new_hash = hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx) + ZSTD_ROW_HASH_CACHE_SIZE, - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32; - let row = ((new_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - row_prefetch(hash_table, tag_table, row, row_log); - let cached = cache.at_mut((idx & ZSTD_ROW_HASH_CACHE_MASK) as usize)?; - Ok(core::mem::replace(cached, new_hash)) -} - -/// `ZSTD_row_update_internalImpl`: inserts the positions from `update_start_idx` up to `update_end_idx`. +const fn rel_row(hash: u32) -> usize { + ((hash >> ZSTD_ROW_HASH_TAG_BITS) as usize) << ROW_ENTRIES.trailing_zeros() +} + +/// Where it starts in the tag table. #[inline(always)] -fn row_update_internal_impl( - ms: &mut MatchState, - prefix: Prefix<'_>, - update_start_idx: u32, - update_end_idx: u32, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - let hash_log = ms.row_hash_log; - let hash_salt = ms.hash_salt; - let hash_table = ms.hash_table.as_mut_slice(); - let tag_table = ms.tag_table.as_mut_slice(); - - for idx in update_start_idx..update_end_idx { - let hash = if use_cache { - row_next_cached_hash( - &mut ms.hash_cache, - hash_table, - tag_table, - prefix, - idx, - hash_log, - row_log, - mls, - hash_salt, - )? +const fn rel_tag_row(hash: u32) -> usize { + if SECOND_TAGS { + 2 * rel_row::(hash) + } else { + rel_row::(hash) + } +} + +/// The hash table and the tag table. +struct Rows<'a> { + indices: &'a mut [u32], + tags: &'a mut [u8], +} + +impl Rows<'_> { + /// The row for `hash`. + #[inline(always)] + fn row( + &mut self, + hash: u32, + ) -> Option> { + let start = rel_row::(hash); + let indices = self.indices.get_mut(start..start + ROW_ENTRIES)?; + let start = rel_tag_row::(hash); + let (tags, second_tags) = if SECOND_TAGS { + let tags = self.tags.get_mut(start..start + 2 * ROW_ENTRIES)?; + let (tags, second_tags) = tags.split_at_mut_checked(ROW_ENTRIES)?; + (tags, Some(second_tags.first_chunk_mut()?)) } else { - hash_ptr_salted( - prefix.buf, - prefix.pos_of(idx), - hash_log + ZSTD_ROW_HASH_TAG_BITS, - mls, - hash_salt, - )? as u32 + (self.tags.get_mut(start..start + ROW_ENTRIES)?, None) }; - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let pos = row_next_index(tag_table.at_mut(rel_row)?, row_mask) as usize; - tag_table.put(rel_row + pos, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8)?; - hash_table.put(rel_row + pos, idx)?; + Some(Row { + indices: indices.first_chunk_mut()?, + tags: tags.first_chunk_mut()?, + second_tags, + }) + } + + /// `ZSTD_row_prefetch`: of the row for `hash`. + #[inline(always)] + fn prefetch(&self, hash: u32) { + row_prefetch::(self.indices, self.tags, hash); } - Ok(()) } -/// `ZSTD_row_update_internal`: inserts the positions from `ms.next_to_update` up to `ip`. With the cache, which is -/// when compressing, not all of them if they are many. +/// `ZSTD_row_prefetch`: of the row for `hash`. #[inline(always)] -fn row_update_internal( - ms: &mut MatchState, - prefix: Prefix<'_>, - ip: usize, - mls: u32, - row_log: u32, - row_mask: u32, - use_cache: bool, -) -> Result<()> { - const K_SKIP_THRESHOLD: u32 = 384; - const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; - const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; +fn row_prefetch( + indices: &[u32], + tags: &[u8], + hash: u32, +) { + if ROW_ENTRIES == 16 { + // Somewhere in the row, which is no more than a cache line. + let entry = (hash >> (ZSTD_ROW_HASH_TAG_BITS - 4)) as usize; + prefetch_l1(indices, entry); + prefetch_l1(tags, entry); + return; + } + let start = rel_row::(hash); + prefetch_l1(indices, start); + prefetch_l1(indices, start + 16); + let start = rel_tag_row::(hash); + prefetch_l1(tags, start); + if SECOND_TAGS { + prefetch_l1(tags, start + ROW_ENTRIES); + } +} + +/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the +/// row-based match finder. +pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { + #[inline(always)] + fn update( + mut rows: Rows<'_>, + hasher: RowHasher, + prefix: Prefix<'_>, + indices: Range, + ) -> Result<()> { + for idx in indices { + let hash = hasher.of::(read_le64(prefix.buf, prefix.pos_of(idx))?); + rows.row::(hash.hash) + .ok_or(BUG)? + .insert(hash, idx); + } + Ok(()) + } - let mut idx = ms.next_to_update; + row_flatten_tags(&mut ms.row_tags, &mut ms.tag_table)?; + let mls = ms.c_params.min_match.min(6); + let hash_bits = ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + let hasher = RowHasher(Hash8::new(mls, hash_bits, ms.hash_salt)); + let rows = Rows { + indices: &mut ms.hash_table, + tags: &mut ms.tag_table, + }; let target = prefix.index_of(ip); + let indices = ms.next_to_update..target; + ms.next_to_update = target; + match ms.c_params.search_log.clamp(4, 6) { + 4 => update::<16>(rows, hasher, prefix, indices), + 5 => update::<32>(rows, hasher, prefix, indices), + _ => update::<64>(rows, hasher, prefix, indices), + } +} - if use_cache && target.wrapping_sub(idx) > K_SKIP_THRESHOLD { - let bound = idx + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; - row_update_internal_impl(ms, prefix, idx, bound, mls, row_log, row_mask, use_cache)?; - idx = target - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; - row_fill_hash_cache(ms, prefix, row_log, mls, idx, ip + 1)?; +/// Bit `n` is set if the entry `n` has the tag `tag`. This becomes a comparison of vectors. +#[inline(always)] +fn tags_match_mask(tags: &[u8; 16], tag: u8) -> u32 { + let mut matches = 0u16; + for (n, &entry) in tags.iter().enumerate() { + matches |= u16::from(entry == tag) << n; } - row_update_internal_impl(ms, prefix, idx, target, mls, row_log, row_mask, use_cache)?; - ms.next_to_update = target; - Ok(()) + u32::from(matches) } -/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the -/// row-based match finder. -pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { - let row_log = ms.c_params.search_log.clamp(4, 6); - let row_mask = (1u32 << row_log) - 1; - let mls = ms.c_params.min_match.min(6); - row_update_internal(ms, prefix, ip, mls, row_log, row_mask, false) +/// Bit `n` is set if the entry `n` has the tag `tag`. +#[inline(always)] +fn row_tags_match_mask(tags: &[u8; ROW_ENTRIES], tag: u8) -> u64 { + match *tags.as_chunks::<16>().0 { + [ref tags] => u64::from(tags_match_mask(tags, tag)), + [ref low, ref high] => { + u64::from(tags_match_mask(low, tag) | tags_match_mask(high, tag) << 16) + } + [ref tags_0, ref tags_1, ref tags_2, ref tags_3] => { + let low = tags_match_mask(tags_0, tag) | tags_match_mask(tags_1, tag) << 16; + let high = tags_match_mask(tags_2, tag) | tags_match_mask(tags_3, tag) << 16; + u64::from(low) | u64::from(high) << 32 + } + _ => 0, + } } -/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` of the row has the tag `tag`. +/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` has the tag `tag`. #[inline(always)] fn row_get_match_mask( - tag_row: &[u8; ROW_ENTRIES], + tags: &[u8; ROW_ENTRIES], tag: u8, head: u32, ) -> u64 { - const LOW_7_BITS: u64 = 0x7F7F_7F7F_7F7F_7F7F; - let tags = u64::from(tag) * 0x0101_0101_0101_0101; - let mut matches = 0u64; - let (chunks, _) = tag_row.as_chunks::<8>(); - for (n, chunk) in chunks.iter().enumerate() { - // A byte is 0 where the tags are the same. - let diff = u64::from_le_bytes(*chunk) ^ tags; - // The high bit of a byte is set if any of its bits is. No byte carries into the next. - let differs = ((diff & LOW_7_BITS) + LOW_7_BITS) | diff; - let same = (!differs & !LOW_7_BITS) >> 7; - // Bit `8 * i` goes to bit `56 + i`, and no two bits go to the same place. - let chunk_matches = same.wrapping_mul(0x0102_0408_1020_4080) >> 56; - matches |= chunk_matches << (8 * n); - } + let matches = row_tags_match_mask(tags, tag); match ROW_ENTRIES { 16 => u64::from((matches as u16).rotate_right(head)), 32 => u64::from((matches as u32).rotate_right(head)), @@ -836,543 +1332,665 @@ fn row_get_match_mask( } } -/// The first loop of `ZSTD_RowFindBestMatch`: copies the entries of a row that have the tag `tag` to `match_buffer`, -/// the most recent first. It stops after `nb_attempts` of them, which it decreases, or at the first that is less than -/// `low_limit`. `prefetch` is called with each. Returns how many there are. -#[inline(always)] -fn row_collect_matches( - row: &[u32; ROW_ENTRIES], - tag_row: &[u8; ROW_ENTRIES], +/// [`row_get_match_mask`]. On its own, it stays a comparison of vectors. +#[inline(never)] +fn row_get_second_match_mask( + tags: &[u8; ROW_ENTRIES], tag: u8, - low_limit: u32, - nb_attempts: &mut u32, - match_buffer: &mut [u32; ROW_ENTRIES], - prefetch: impl Fn(u32), -) -> Result { - let row_mask = ROW_ENTRIES as u32 - 1; - let head = u32::from(tag_row.at(0)?) & row_mask; - let mut num_matches = 0usize; - let mut matches = row_get_match_mask(tag_row, tag, head); - - while matches > 0 && *nb_attempts > 0 { - let match_pos = (head + matches.trailing_zeros()) & row_mask; - matches &= matches - 1; - if match_pos == 0 { - // The head, which happens to be the same as the tag. - continue; - } - let match_index = row.at(match_pos as usize)?; - if match_index < low_limit { - break; + head: u32, +) -> u64 { + row_get_match_mask(tags, tag, head) +} + +/// The bits of [`row_get_match_mask`], but for the one of the entry 0. +#[inline(always)] +const fn row_entries_mask(head: u32) -> u64 { + !(1 << (head.wrapping_neg() & (ROW_ENTRIES as u32 - 1))) +} + +/// The rows of a dictionary. +#[derive(Clone, Copy)] +struct RowDict<'a> { + indices: &'a [u32], + tags: &'a [u8], + hasher: Hash8, + /// The index, among its own, of its first byte. + lowest_index: u32, +} + +impl<'a> RowDict<'a> { + fn new(dms: &DictMatchState<'a>, mls: u32) -> Self { + Self { + indices: &dms.ms.hash_table, + tags: &dms.ms.tag_table, + hasher: Hash8::new(mls, dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS, 0), + lowest_index: dms.ms.window.dict_limit, } - prefetch(match_index); - match_buffer.put(num_matches, match_index)?; - num_matches += 1; - *nb_attempts -= 1; } - Ok(num_matches) } -/// `ZSTD_RowFindBestMatch`: returns the length of the best match for `ip`, or 3. `offset_ptr` is written to if there -/// is one. -#[inline(never)] -fn row_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - offset_ptr: &mut u32, -) -> Result { - let buf = src.buf; - let hash_log = ms.row_hash_log; - let dict_limit = ms.window.dict_limit; - let prefix_start = src.pos_of(dict_limit); - let dict_end = src.dict_pos_of(dict_limit); - let curr = src.index_of(ip); - let low_limit = ms.get_lowest_match_index(curr, ms.c_params.window_log); - let row_log = ROW_ENTRIES.trailing_zeros(); - let row_mask = ROW_ENTRIES as u32 - 1; - // There are no more searches than entries in a row. - let capped_search_log = ms.c_params.search_log.min(row_log); - let hash_salt = ms.hash_salt; - let mut nb_attempts = 1u32 << capped_search_log; - let mut ml = 4 - 1; - let mut match_buffer = [0u32; ROW_ENTRIES]; - - let mut dms_hash = 0u32; - let mut dms_rel_row = 0usize; - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_hash_log = dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; - dms_hash = hash_ptr(buf, ip, dms_hash_log, MLS)? as u32; - dms_rel_row = ((dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - row_prefetch(&dms.ms.hash_table, &dms.ms.tag_table, dms_rel_row, row_log); - } - - // Update the tables up to, and not including, `ip`. - let hash = if ms.lazy_skipping { - // While skipping, not every position is inserted. - ms.next_to_update = curr; - hash_ptr_salted(buf, ip, hash_log + ZSTD_ROW_HASH_TAG_BITS, MLS, hash_salt)? as u32 - } else { - let prefix = src.prefix(); - row_update_internal(ms, prefix, ip, MLS, row_log, row_mask, true)?; - row_next_cached_hash( - &mut ms.hash_cache, - &ms.hash_table, - &ms.tag_table, - prefix, - curr, - hash_log, - row_log, - MLS, - hash_salt, - )? - }; - ms.hash_salt_entropy = ms.hash_salt_entropy.wrapping_add(hash); +/// The tables of the row-based match finder, and what it takes to add to them. +struct RowIndex<'a> { + rows: Rows<'a>, + /// The hashes of the 8 positions from `next_to_update` on, each at its index modulo 8. + hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], + second_tag_cache: [u8; ZSTD_ROW_HASH_CACHE_SIZE], + hasher: RowHasher, + near: Near<'a>, + base_index: u32, + /// `ms.next_to_update`, as a position. + next_to_update: u32, +} - { - let rel_row = ((hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log) as usize; - let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8; - let row = ms.hash_table.array_at_mut::(rel_row)?; - let tag_row = ms.tag_table.array_at_mut::(rel_row)?; - let num_matches = row_collect_matches( - row, - tag_row, - tag, - low_limit, - &mut nb_attempts, - &mut match_buffer, - |match_index| { - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - prefetch_l1(buf, src.pos_of(match_index)); - } else { - prefetch_l1(src.dict_buf, src.dict_pos_of(match_index)); - } +impl RowIndex<'_> { + /// `ZSTD_row_nextCachedHash`: the hash of `pos`, from the cache, where `new_hash` takes its place: the hash of the + /// position 8 bytes further. + #[inline(always)] + fn next_cached_hash( + &mut self, + pos: u32, + new_hash: RowHash, + ) -> RowHash { + self.rows + .prefetch::(new_hash.hash); + let cached = (self.base_index.wrapping_add(pos) & ZSTD_ROW_HASH_CACHE_MASK) as usize; + let (Some(hash), Some(second_tag)) = ( + self.hash_cache.get_mut(cached), + self.second_tag_cache.get_mut(cached), + ) else { + return new_hash; + }; + RowHash { + hash: core::mem::replace(hash, new_hash.hash), + second_tag: if SECOND_TAGS { + core::mem::replace(second_tag, new_hash.second_tag) + } else { + 0 }, - )?; + } + } + + /// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `next_to_update` on, or fewer so as not to go past + /// `limit`. + #[inline(never)] + fn fill_hash_cache( + &mut self, + limit: u32, + ) -> Option<()> { + let from = self.next_to_update; + let max_elems = (limit + 1).saturating_sub(from); + for pos in from..from + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE as u32) { + let hash = self.hasher.of::(self.near.read64(pos)?); + self.next_cached_hash::(pos, hash); + } + Some(()) + } + + /// `ZSTD_row_update_internalImpl`: inserts the positions from `next_to_update` up to `end`. + #[inline(always)] + fn insert_up_to( + &mut self, + end: u32, + ) -> Option<()> { + let bytes = self.near.bytes(); + if end as usize + 2 * ZSTD_ROW_HASH_CACHE_SIZE > bytes.len() { + return None; + } + for pos in self.next_to_update..end { + let ahead = pos as usize + ZSTD_ROW_HASH_CACHE_SIZE; + let ahead = bytes.get(ahead..ahead + 8)?.first_chunk()?; + let new_hash = self.hasher.of::(u64::from_le_bytes(*ahead)); + let hash = self.next_cached_hash::(pos, new_hash); + let index = self.base_index.wrapping_add(pos); + self.rows + .row::(hash.hash)? + .insert(hash, index); + } + self.next_to_update = end; + Some(()) + } + + /// `ZSTD_row_update_internal`: inserts the positions from `next_to_update` up to `ip`, but not all of them if they + /// are many. + #[inline(always)] + fn update(&mut self, ip: u32) -> Option<()> { + const K_SKIP_THRESHOLD: u32 = 384; + + if ip.wrapping_sub(self.next_to_update) > K_SKIP_THRESHOLD { + self.skip::(ip)?; + } + self.insert_up_to::(ip) + } + + /// The part of `ZSTD_row_update_internal` for a long way up to `ip`. + #[cold] + #[inline(never)] + fn skip(&mut self, ip: u32) -> Option<()> { + const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; + const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; + + let end = self.next_to_update + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; + self.insert_up_to::(end)?; + self.next_to_update = ip - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; + self.fill_hash_cache::(ip + 1) + } +} + +/// The entries of a row that may be matches for a position. +#[derive(Clone, Copy)] +struct RowCandidates<'a, const ROW_ENTRIES: usize> { + indices: &'a [u32; ROW_ENTRIES], + head: u32, + /// See [`row_get_match_mask`]. + matches: u64, +} + +impl RowCandidates<'_, ROW_ENTRIES> { + /// The most recent one. There is one. + #[inline(always)] + fn first(&self) -> u32 { + let entry = (self.head + self.matches.trailing_zeros()) as usize & (ROW_ENTRIES - 1); + self.indices.get(entry).copied().unwrap_or(0) + } + + #[inline(always)] + const fn remove_first(&mut self) { + self.matches &= self.matches.wrapping_sub(1); + } +} - // Insert `ip`. - let pos = row_next_index(tag_row.at_mut(0)?, row_mask) as usize; - tag_row.put(pos, tag)?; - row.put(pos, ms.next_to_update)?; - ms.next_to_update += 1; - - // The current segment, up to where matches end. - let near = Near::new(buf.sub(prefix_start..i_limit)?)?; - let near_ip = curr - dict_limit; - // What a longer match has where the best one so far ends. - let mut tail = near.read32(near_ip + ml as u32 - 3).ok_or(BUG)?; - - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - if DICT_MODE != EXT_DICT || match_index >= dict_limit { - let match_pos = match_index.wrapping_sub(dict_limit); - if near.read32(match_pos.wrapping_add(ml as u32 - 3)) == Some(tail) { - current_ml = near.count(near_ip, match_pos) as usize; +/// What the row-based match finder looks for matches in. +struct RowSearch<'a> { + block: LazyBlock<'a>, + nb_attempts: u32, + dms: Option>, + /// How many candidates the second tags ruled out. + nb_ruled_out: u32, +} + +impl RowSearch<'_> { + /// The part of `ZSTD_RowFindBestMatch` that follows the update of the tables. `candidates` is the entries of the + /// row that have the tag of `ip`, and `long_matches` those that have its second tag as well, if there are second + /// tags. `dict_hash` is the hash of `ip` for the rows of the dictionary, if there are. See [`Finder::find`]. + #[inline(always)] + fn longest_match( + &mut self, + mut candidates: RowCandidates<'_, ROW_ENTRIES>, + long_matches: Option, + dict_hash: u32, + ip: u32, + wanted: Wanted, + ) -> Option { + let block = &self.block; + let matches = candidates.matches; + let curr = block.base_index.wrapping_add(ip); + let low_limit = block.reach.low_limit(curr); + let closest = if DICT_MODE == DICT_MATCH_STATE { + candidates.first().max(low_limit) + } else { + candidates.first() + }; + let start_index = if DICT_MODE == EXT_DICT { + low_limit.max(block.base_index) + } else { + low_limit + }; + let segment = block + .near + .tail(start_index.wrapping_sub(block.base_index))?; + let floor = Best { + len: wanted.floor::(curr, closest), + off_base: 0, + }; + + // If there is a match that long, the others are of no interest. + if let Some(long_matches) = long_matches + && (long_matches != 0 || floor.len + 1 >= SECOND_TAG_LENGTH) + { + candidates.matches = long_matches; + self.nb_ruled_out += (matches ^ long_matches).count_ones(); + } + + loop { + let mut longest = Longest::new(segment, start_index, curr, floor)?; + let mut nb_attempts = self.nb_attempts; + let all = candidates.matches == matches; + + while candidates.matches != 0 && nb_attempts > 0 { + let match_index = candidates.first(); + candidates.remove_first(); + let mut offered = longest.offer(match_index); + if offered == Offered::Outside { + if DICT_MODE != EXT_DICT || match_index < low_limit { + break; + } + let match_pos = (match_index - block.dict.start_index) as usize; + offered = longest.offer_in_dict(block, ip, match_pos).ok()?; } - } else { - let match_pos = src.dict_pos_of(match_index); - if read_le32(src.dict_buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - src.dict_buf, - match_pos + 4, - i_limit, - dict_end, - prefix_start, - )? + 4; + if offered == Offered::Longest { + return longest.best().found(); } + nb_attempts -= 1; } - if current_ml > ml { - ml = current_ml; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - if ip + current_ml == i_limit { - // The best there can be. The next attempt would read too far. - break; - } - tail = near.read32(near_ip + ml as u32 - 3).ok_or(BUG)?; + if DICT_MODE == DICT_MATCH_STATE { + self.longest_dict_match::(dict_hash, ip, nb_attempts, &mut longest)?; + } + let best = longest.best(); + if all || best.len >= SECOND_TAG_LENGTH || floor.len + 1 >= SECOND_TAG_LENGTH { + return best.found(); } + // The second tags were the same for no reason. + candidates.matches = matches; } } - if DICT_MODE == DICT_MATCH_STATE { - let dms = src.dms.ok_or(BUG)?; - let dms_lowest_index = dms.ms.window.dict_limit; - let dms_index_delta = dict_limit.wrapping_sub(dms_end_index(&dms)); - - let num_matches = row_collect_matches( - dms.ms.hash_table.array_at::(dms_rel_row)?, - dms.ms.tag_table.array_at::(dms_rel_row)?, - (dms_hash & ZSTD_ROW_HASH_TAG_MASK) as u8, - dms_lowest_index, - &mut nb_attempts, - &mut match_buffer, - |match_index| prefetch_l1(dms.buf, dms.pos_of(match_index)), - )?; + /// The part of `ZSTD_RowFindBestMatch` that searches the dictionary. + #[inline(always)] + fn longest_dict_match( + &self, + hash: u32, + ip: u32, + mut nb_attempts: u32, + longest: &mut Longest<'_>, + ) -> Option<()> { + let block = &self.block; + let dms = self.dms?; + let start = rel_row::(hash); + let tags = dms.tags.get(start..start + ROW_ENTRIES)?; + let tags = tags.first_chunk::()?; + let indices = dms.indices.get(start..start + ROW_ENTRIES)?; + let head = u32::from(*tags.first()?) & (ROW_ENTRIES as u32 - 1); + let matches = row_get_match_mask(tags, hash as u8, head); + let mut candidates = RowCandidates { + indices: indices.first_chunk::()?, + head, + matches: matches & row_entries_mask::(head), + }; - for &match_index in match_buffer.sub(..num_matches)? { - let mut current_ml = 0; - let match_pos = dms.pos_of(match_index); - if read_le32(dms.buf, match_pos)? == read_le32(buf, ip)? { - current_ml = count_2segments( - buf, - ip + 4, - dms.buf, - match_pos + 4, - i_limit, - dms.end, - prefix_start, - )? + 4; + while candidates.matches != 0 && nb_attempts > 0 { + let match_index = candidates.first(); + candidates.remove_first(); + if match_index < dms.lowest_index { + break; } - - if current_ml > ml { - ml = current_ml; - let offset = curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)); - *offset_ptr = offset_to_offbase(offset); - if ip + current_ml == i_limit { - break; - } + nb_attempts -= 1; + let match_pos = (match_index - dms.lowest_index) as usize; + if longest.offer_in_dict(block, ip, match_pos).ok()? == Offered::Longest { + break; } } + Some(()) } +} - Ok(ml) +/// What the row-based match finder leaves in the match state at the end of a block. +struct RowsSaved<'a> { + next_to_update: &'a mut u32, + hash_salt_entropy: &'a mut u32, + tags: &'a mut RowTags, } -// ── the parsers ─────────────────────────────────────────────────────────── +/// The state of [`RowFinder`], which does not depend on the size of the rows. +struct RowState<'a> { + index: RowIndex<'a>, + search: RowSearch<'a>, + hash_salt_entropy: u32, + saved: RowsSaved<'a>, +} -// `searchMethod_e`, as an argument of a const generic. -const SEARCH_HASH_CHAIN: u8 = 0; -const SEARCH_BINARY_TREE: u8 = 1; -const SEARCH_ROW_HASH: u8 = 2; +impl<'a> RowState<'a> { + fn new(ms: &'a mut MatchState, src: &BlockSrc<'a>, block: LazyBlock<'a>) -> Self { + let mls = ms.c_params.min_match.clamp(4, 6); + let hash_bits = ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + Self { + index: RowIndex { + rows: Rows { + indices: &mut ms.hash_table, + tags: &mut ms.tag_table, + }, + hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + second_tag_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + hasher: RowHasher(Hash8::new(mls, hash_bits, ms.hash_salt)), + near: block.near, + base_index: block.base_index, + next_to_update: ms.next_to_update.wrapping_sub(block.base_index), + }, + search: RowSearch { + block, + // There are no more searches than entries in a row. + nb_attempts: 1 << ms.c_params.search_log.min(6), + dms: src.dms.as_ref().map(|dms| RowDict::new(dms, mls)), + nb_ruled_out: 0, + }, + hash_salt_entropy: ms.hash_salt_entropy, + saved: RowsSaved { + next_to_update: &mut ms.next_to_update, + hash_salt_entropy: &mut ms.hash_salt_entropy, + tags: &mut ms.row_tags, + }, + } + } -/// `ZSTD_searchMax`: the best match for `ip`. `mls` and `row_log` are from 4 to 6. -#[inline(always)] -fn search_max( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - iend: usize, - offset_ptr: &mut u32, - mls: u32, - row_log: u32, -) -> Result { - match SEARCH_METHOD { - SEARCH_HASH_CHAIN => match mls { - 4 => hc_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => hc_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => hc_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - SEARCH_BINARY_TREE => match mls { - 4 => bt_find_best_match::<4, DICT_MODE>(ms, src, ip, iend, offset_ptr), - 5 => bt_find_best_match::<5, DICT_MODE>(ms, src, ip, iend, offset_ptr), - _ => bt_find_best_match::<6, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, - _ => match (mls, row_log) { - (4, 4) => row_find_best_match::<4, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, 5) => row_find_best_match::<4, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (4, _) => row_find_best_match::<4, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 4) => row_find_best_match::<5, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, 5) => row_find_best_match::<5, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (5, _) => row_find_best_match::<5, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 4) => row_find_best_match::<6, 16, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, 5) => row_find_best_match::<6, 32, DICT_MODE>(ms, src, ip, iend, offset_ptr), - (_, _) => row_find_best_match::<6, 64, DICT_MODE>(ms, src, ip, iend, offset_ptr), - }, + /// The end of a block. + fn finish(self) -> Result<()> { + let Self { + index, + search, + hash_salt_entropy, + saved, + } = self; + *saved.next_to_update = index.base_index.wrapping_add(index.next_to_update); + *saved.hash_salt_entropy = hash_salt_entropy; + if let RowTags::Second(stats) = saved.tags { + stats.nb_positions += u64::from(search.block.near.len() - search.block.istart); + stats.nb_ruled_out += u64::from(search.nb_ruled_out); + // Below that, the second tags cost more than they save. + if 2 * stats.nb_ruled_out < 5 * stats.nb_positions { + row_flatten_tags(saved.tags, index.rows.tags)?; + } + } + Ok(()) } } -/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match -/// state is searched. -#[derive(Clone, Copy)] -struct DictSegment<'a> { - buf: &'a [u8], - /// Its sum with an index of the window is a position in `buf`. It wraps around. - base: usize, - /// Where the segment starts in `buf`. - start: usize, - /// Where it ends. - end: usize, +/// `ZSTD_RowFindBestMatch` +struct RowFinder<'a, const ROW_ENTRIES: usize, const SECOND_TAGS: bool, const DICT_MODE: u8>( + RowState<'a>, +); + +impl<'a, const ROW_ENTRIES: usize, const SECOND_TAGS: bool, const DICT_MODE: u8> Finder<'a> + for RowFinder<'a, ROW_ENTRIES, SECOND_TAGS, DICT_MODE> +{ + const MARGIN: u32 = 8 + ZSTD_ROW_HASH_CACHE_SIZE as u32; + + #[inline(always)] + fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result { + let mut finder = Self(RowState::new(ms, src, block)); + finder.stop_skipping()?; + Ok(finder) + } + + #[inline(always)] + fn find( + &mut self, + ip: u32, + wanted: Wanted, + skipping: bool, + ) -> Result> { + let RowState { + index, + search, + hash_salt_entropy, + .. + } = &mut self.0; + + let mut dict_hash = 0; + if DICT_MODE == DICT_MATCH_STATE { + let dms = search.dms.ok_or(BUG)?; + dict_hash = dms.hasher.of(index.near.read64(ip).ok_or(BUG)?); + row_prefetch::(dms.indices, dms.tags, dict_hash); + } + + // Update the tables up to, and not including, `ip`. + let hash = if skipping { + let bytes = index.near.read64(ip).ok_or(BUG)?; + index.hasher.of::(bytes) + } else { + if index.next_to_update != ip { + index.update::(ip).ok_or(BUG)?; + } + let ahead = index.near.read64(ip + ZSTD_ROW_HASH_CACHE_SIZE as u32); + let new_hash = index.hasher.of::(ahead.ok_or(BUG)?); + index.next_cached_hash::(ip, new_hash) + }; + *hash_salt_entropy = hash_salt_entropy.wrapping_add(hash.hash); + index.next_to_update = ip + 1; + + let mut row = index + .rows + .row::(hash.hash) + .ok_or(BUG)?; + let head = u32::from(*row.tags.first().ok_or(BUG)?) & (ROW_ENTRIES as u32 - 1); + let matches = row_get_match_mask(row.tags, hash.hash as u8, head) + & row_entries_mask::(head); + let curr = index.base_index.wrapping_add(ip); + if DICT_MODE != DICT_MATCH_STATE && matches == 0 { + row.insert(hash, curr); + return Ok(None); + } + let candidates = RowCandidates { + indices: row.indices, + head, + matches, + }; + let second_tags = row.second_tags.as_ref(); + let long_matches = second_tags + .map(|tags| matches & row_get_second_match_mask(tags, hash.second_tag, head)); + let found = search.longest_match::( + candidates, + long_matches, + dict_hash, + ip, + wanted, + ); + row.insert(hash, curr); + Ok(found) + } + + #[inline(always)] + fn stop_skipping(&mut self) -> Result<()> { + let ilimit = self.0.index.near.len() - Self::MARGIN; + self.0 + .index + .fill_hash_cache::(ilimit) + .ok_or(BUG) + } + + #[inline(always)] + fn finish(self) -> Result<()> { + self.0.finish() + } } +// ── the parsers ─────────────────────────────────────────────────────────── + /// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. #[inline(always)] -fn rep_length( - ms: &MatchState, - src: &BlockSrc<'_>, - dict: &DictSegment<'_>, - ip: usize, - offset: u32, -) -> Result { - let buf = src.buf; - let iend = src.end; +fn rep_length(block: &LazyBlock<'_>, ip: u32, offset: u32) -> Result { + let near = block.near; + let ip_bytes = near.read32(ip).ok_or(BUG)?; if DICT_MODE == NO_DICT { - let rep_match = ip.wrapping_sub(offset as usize); - if offset > 0 && read_le32(buf, rep_match)? == read_le32(buf, ip)? { - return Ok(count(buf, ip + 4, buf, rep_match + 4, iend)? + 4); + let rep_match = ip.wrapping_sub(offset); + if offset > 0 && near.read32(rep_match) == Some(ip_bytes) { + return Ok(near.count(ip + 4, rep_match + 4) + 4); } return Ok(0); } - let dict_limit = ms.window.dict_limit; - let curr = src.index_of(ip); + let curr = block.base_index.wrapping_add(ip); let rep_index = curr.wrapping_sub(offset); - if !index_overlap_check(dict_limit, rep_index) { + if !index_overlap_check(block.base_index, rep_index) { return Ok(0); } - if DICT_MODE == EXT_DICT { - // The offset may lead to what is no longer in the window. - let window_low = ms.get_lowest_match_index(curr, ms.c_params.window_log); - if offset > curr.wrapping_sub(window_low) { + // The offset may lead to what is no longer in the window. + if DICT_MODE == EXT_DICT && offset > curr.wrapping_sub(block.reach.low_limit(curr)) { + return Ok(0); + } + if rep_index < block.base_index { + let rep_match = rep_index.wrapping_sub(block.dict.start_index) as usize; + if read_le32(block.dict.bytes, rep_match)? != ip_bytes { return Ok(0); } + return Ok(block.count_from_dict(ip + 4, rep_match + 4)? + 4); } - let (rep_buf, rep_match, rep_end) = if rep_index < dict_limit { - let rep_match = dict.base.wrapping_add(rep_index as usize); - (dict.buf, rep_match, dict.end) - } else { - (buf, src.pos_of(rep_index), iend) - }; - if read_le32(rep_buf, rep_match)? != read_le32(buf, ip)? { + let rep_match = rep_index - block.base_index; + if near.read32(rep_match) != Some(ip_bytes) { return Ok(0); } - let prefix_start = src.pos_of(dict_limit); - Ok(count_2segments( - buf, - ip + 4, - rep_buf, - rep_match + 4, - iend, - rep_end, - prefix_start, - )? + 4) + Ok(near.count(ip + 4, rep_match + 4) + 4) } -/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions -/// further a better match is looked for, once there is one. -#[inline(never)] -fn compress_block_lazy_generic( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, -) -> Result { - let buf = src.buf; - let istart = src.start; - let mut ip = istart; - let mut anchor = istart; - let iend = src.end; - let prefix_lowest_index = ms.window.dict_limit; - let prefix_lowest = src.pos_of(prefix_lowest_index); - let mls = ms.c_params.min_match.clamp(4, 6); - let row_log = ms.c_params.search_log.clamp(4, 6); - - let [rep_0, rep_1, _] = rep; - let mut offset_1 = *rep_0; - let mut offset_2 = *rep_1; - let mut offset_saved_1 = 0u32; - let mut offset_saved_2 = 0u32; - - ms.lazy_skipping = false; +/// The repeat offsets at the start of a block. +struct Offsets { + offset_1: u32, + offset_2: u32, + /// What they were, for those that are too far to be of use and are 0. + saved_1: u32, + saved_2: u32, +} - let margin = if SEARCH_METHOD == SEARCH_ROW_HASH { - 8 + ZSTD_ROW_HASH_CACHE_SIZE - } else { - 8 - }; - let Some(ilimit) = iend.checked_sub(margin) else { - // There is no position to search, nor to hash. - return Ok(iend - istart); - }; +/// Where the parser starts in a block, and with what. +struct Start { + ip: u32, + offsets: Offsets, +} - let dict = match DICT_MODE { - NO_DICT => DictSegment { - buf: &[], - base: 0, - start: 0, - end: 0, - }, - EXT_DICT => DictSegment { - buf: src.dict_buf, - base: src.dict_base, - start: src.dict_pos_of(ms.window.low_limit), - end: src.dict_pos_of(prefix_lowest_index), - }, - _ => { - let dms = src.dms.ok_or(BUG)?; - let dict_index_delta = prefix_lowest_index.wrapping_sub(dms_end_index(&dms)); - DictSegment { - buf: dms.buf, - base: dms.base.wrapping_sub(dict_index_delta as usize), - start: dms.pos_of(dms.ms.window.dict_limit), - end: dms.end, +impl Start { + fn new( + ms: &MatchState, + block: &LazyBlock<'_>, + rep: &[u32; ZSTD_REP_NUM], + dict_mode: DictMode, + ) -> Self { + let [offset_1, offset_2, _] = *rep; + let mut offsets = Offsets { + offset_1, + offset_2, + saved_1: 0, + saved_2: 0, + }; + // As in the C code, what counts as nothing to refer to is not the same in all modes. + let nothing_before = + block.istart == 0 && (dict_mode == DictMode::ExtDict || block.dict.bytes.is_empty()); + let ip = block.istart + u32::from(nothing_before); + if dict_mode == DictMode::NoDict { + let curr = block.base_index.wrapping_add(ip); + let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); + let max_rep = curr - window_low; + if offset_2 > max_rep { + offsets.saved_2 = offset_2; + offsets.offset_2 = 0; + } + if offset_1 > max_rep { + offsets.saved_1 = offset_1; + offsets.offset_1 = 0; } } - }; - - if DICT_MODE == EXT_DICT { - ip += usize::from(ip == prefix_lowest); - } else { - let dict_and_prefix_length = (ip - prefix_lowest) + (dict.end - dict.start); - ip += usize::from(dict_and_prefix_length == 0); - } - if DICT_MODE == NO_DICT { - let curr = src.index_of(ip); - let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); - let max_rep = curr - window_low; - if offset_2 > max_rep { - offset_saved_2 = offset_2; - offset_2 = 0; - } - if offset_1 > max_rep { - offset_saved_1 = offset_1; - offset_1 = 0; - } + Self { ip, offsets } } +} - if SEARCH_METHOD == SEARCH_ROW_HASH { - row_fill_hash_cache(ms, src.prefix(), row_log, mls, ms.next_to_update, ilimit)?; - } +/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions +/// further a better match is looked for, once there is one. +#[inline(never)] +fn compress_block_lazy_generic<'a, F: Finder<'a>, const DEPTH: u32, const DICT_MODE: u8>( + args: LazyArgs<'a>, +) -> Result { + let LazyArgs { + ms, + seq_store, + rep, + src, + block, + start, + } = args; + let block = █ + let near = block.near; + let Some(ilimit) = near.len().checked_sub(F::MARGIN) else { + // There is no position to search, nor to hash. + return Ok((near.len() - block.istart) as usize); + }; + let mut finder = F::new(ms, src, *block)?; + let mut seq_store = SeqSink::new(seq_store); + let mut ip = start.ip; + let mut anchor = block.istart; + let Offsets { + mut offset_1, + mut offset_2, + saved_1, + mut saved_2, + } = start.offsets; + let mut skipping = false; 'parse: while ip < ilimit { - let mut match_length; - let mut off_base = REPCODE1_TO_OFFBASE; + // Check the repeat offset. + let mut best = Found::repeat(rep_length::(block, ip + 1, offset_1)?); let mut start = ip + 1; - 'found: { - // Check the repeat offset. - match_length = rep_length::(ms, src, &dict, ip + 1, offset_1)?; - if DEPTH == 0 && match_length != 0 { - break 'found; - } - - // First search. - { - let mut off_base_found = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut off_base_found, - mls, - row_log, - )?; - if ml2 > match_length { - match_length = ml2; - start = ip; - off_base = off_base_found; - } - } - - if match_length < 4 { - // Jump faster over incompressible sections. - let step = (ip - anchor) >> K_SEARCH_STRENGTH; - ip += step + 1; - // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are of - // no use out of the tables. As in the C code, it does not start at the same step in all modes. - ms.lazy_skipping = if DICT_MODE == EXT_DICT { - step > K_LAZY_SKIPPING_STEP + if DEPTH > 0 || best.len == 0 { + // How many positions were searched since the last match that was taken, that one included. + let mut since_best = 0; + let mut wanted = Wanted { + longer_than: best.len.max(3), + gain: Wanted::ANY_GAIN, + }; + + loop { + since_best += 1; + let found = if DEPTH > 0 { + finder.find::(ip, wanted, skipping)? } else { - step + 1 > K_LAZY_SKIPPING_STEP + finder.find::(ip, wanted, skipping)? }; - continue 'parse; - } + if let Some(found) = found + && (DEPTH == 0 || found.gain() > wanted.gain) + { + best = found; + start = ip; + since_best = 1; + } - // Look for something better at the next positions. - if DEPTH >= 1 { - while ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 3) as i32; - let gain1 = (match_length * 3) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 4; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } + if best.len < 4 { + // Jump faster over incompressible sections. + let step = (ip - anchor) >> K_SEARCH_STRENGTH; + ip += step + 1; + // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are + // of no use out of the tables. As in the C code, it does not start at the same step in all modes. + skipping = if DICT_MODE == EXT_DICT { + step > K_LAZY_SKIPPING_STEP + } else { + step + 1 > K_LAZY_SKIPPING_STEP + }; + continue 'parse; + } - // Nothing better: try one position further. - if DEPTH == 2 && ip < ilimit { - ip += 1; - if off_base != 0 { - let ml_rep = rep_length::(ms, src, &dict, ip, offset_1)?; - let gain2 = (ml_rep * 4) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 1; - if ml_rep >= 4 && gain2 > gain1 { - match_length = ml_rep; - off_base = REPCODE1_TO_OFFBASE; - start = ip; - } - } - { - let mut ofb_candidate = 999_999_999u32; - let ml2 = search_max::( - ms, - src, - ip, - iend, - &mut ofb_candidate, - mls, - row_log, - )?; - let gain2 = (ml2 * 4) as i32 - highbit32(ofb_candidate) as i32; - let gain1 = (match_length * 4) as i32 - highbit32(off_base) as i32 + 7; - if ml2 >= 4 && gain2 > gain1 { - match_length = ml2; - off_base = ofb_candidate; - start = ip; - continue; - } - } - } + // Look for something better at the next positions. + if since_best > DEPTH || ip >= ilimit { break; } + ip += 1; + let rep_len = rep_length::(block, ip, offset_1)?; + let weight = 2 + since_best; + let gain2 = (rep_len * weight) as i32; + let gain1 = (best.len * weight) as i32 - best.cost() + 1; + if rep_len >= 4 && gain2 > gain1 { + best = Found::repeat(rep_len); + start = ip; + } + wanted = Wanted { + longer_than: 3, + gain: best.gain() + 1 + 3 * since_best as i32, + }; } // Catch up: the match may start earlier. - if offbase_is_offset(off_base) { - let offset = offbase_to_offset(off_base); - let match_index = src.index_of(start).wrapping_sub(offset); - let (match_buf, mut match_pos, m_start) = - if DICT_MODE != NO_DICT && match_index < prefix_lowest_index { - let match_pos = dict.base.wrapping_add(match_index as usize); - (dict.buf, match_pos, dict.start) + if offbase_is_offset(best.off_base.get()) { + let offset = offbase_to_offset(best.off_base.get()); + let match_index = block.base_index.wrapping_add(start).wrapping_sub(offset); + let (segment, mut match_pos) = + if DICT_MODE != NO_DICT && match_index < block.base_index { + (block.dict.bytes, match_index - block.dict.start_index) } else { - (buf, src.pos_of(match_index), prefix_lowest) + (near.bytes(), start - offset) }; while start > anchor - && match_pos > m_start - && buf.at(start - 1)? == match_buf.at(match_pos - 1)? + && match_pos > 0 + && near.read8(start - 1) == segment.get(match_pos as usize - 1).copied() { start -= 1; match_pos -= 1; - match_length += 1; + best.len += 1; } offset_2 = offset_1; offset_1 = offset; @@ -1380,73 +1998,98 @@ fn compress_block_lazy_generic(ms, src, &dict, ip, offset_2)?; - if match_length == 0 { + let rep_len = rep_length::(block, ip, offset_2)?; + if rep_len == 0 { break; } (offset_1, offset_2) = (offset_2, offset_1); - seq_store.store_seq(0, buf, anchor, REPCODE1_TO_OFFBASE, match_length)?; - ip += match_length; + seq_store.store_seq(0, near, anchor, REPCODE1_TO_OFFBASE, rep_len)?; + ip += rep_len; anchor = ip; } } + finder.finish()?; // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. - if offset_saved_1 != 0 && offset_1 != 0 { - offset_saved_2 = offset_saved_1; + if saved_1 != 0 && offset_1 != 0 { + saved_2 = saved_1; } // Save the repeat offsets for the next block. - *rep_0 = if offset_1 != 0 { - offset_1 - } else { - offset_saved_1 - }; - *rep_1 = if offset_2 != 0 { - offset_2 - } else { - offset_saved_2 - }; + let [rep_0, rep_1, _] = rep; + *rep_0 = if offset_1 != 0 { offset_1 } else { saved_1 }; + *rep_1 = if offset_2 != 0 { offset_2 } else { saved_2 }; // The size of the last literals. - Ok(iend - anchor) + Ok((near.len() - anchor) as usize) +} + +/// What [`compress_block_lazy_generic`] works on. +struct LazyArgs<'a> { + ms: &'a mut MatchState, + seq_store: &'a mut SeqStore, + rep: &'a mut [u32; ZSTD_REP_NUM], + src: &'a BlockSrc<'a>, + block: LazyBlock<'a>, + start: Start, } -/// [`compress_block_lazy_generic`], for a `dict_mode` that is not a constant. +/// [`compress_block_lazy_generic`], with the hash chains or the rows. #[inline(always)] -fn compress_block_lazy_any_dict( - ms: &mut MatchState, - seq_store: &mut SeqStore, - rep: &mut [u32; ZSTD_REP_NUM], - src: &BlockSrc<'_>, - dict_mode: DictMode, +fn compress_block_lazy_hashed( + args: LazyArgs<'_>, + use_row: bool, ) -> Result { - match dict_mode { - DictMode::NoDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) + if !use_row { + return compress_block_lazy_generic::, DEPTH, DICT_MODE>(args); + } + // With a dictionary, the second tags are given up: how many candidates there are matters if it is searched next. + if DICT_MODE != NO_DICT { + row_flatten_tags(&mut args.ms.row_tags, &mut args.ms.tag_table)?; + } + match (args.ms.c_params.search_log.clamp(4, 6), args.ms.row_tags) { + (4, _) => { + compress_block_lazy_generic::, DEPTH, DICT_MODE>(args) } - DictMode::ExtDict => { - compress_block_lazy_generic::(ms, seq_store, rep, src) + (5, _) => { + compress_block_lazy_generic::, DEPTH, DICT_MODE>(args) } - DictMode::DictMatchState => { - compress_block_lazy_generic::( - ms, seq_store, rep, src, - ) + (_, RowTags::Flat) => { + compress_block_lazy_generic::, DEPTH, DICT_MODE>(args) + } + (_, RowTags::Second(_)) => { + compress_block_lazy_generic::, DEPTH, NO_DICT>(args) + } + } +} + +/// [`compress_block_lazy`], for a mode that is a constant. +#[inline(always)] +fn compress_block_lazy_any_strategy( + args: LazyArgs<'_>, + strategy: Strategy, + use_row: bool, +) -> Result { + match strategy { + Strategy::Greedy => compress_block_lazy_hashed::<0, DICT_MODE>(args, use_row), + Strategy::Lazy => compress_block_lazy_hashed::<1, DICT_MODE>(args, use_row), + Strategy::Lazy2 => compress_block_lazy_hashed::<2, DICT_MODE>(args, use_row), + Strategy::BtLazy2 if !use_row => { + compress_block_lazy_generic::, 2, DICT_MODE>(args) } + _ => Err(BUG), } } @@ -1461,28 +2104,20 @@ pub(crate) fn compress_block_lazy( use_row: bool, dict_mode: DictMode, ) -> Result { - match (strategy, use_row) { - (Strategy::Greedy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Greedy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, false) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) - } - (Strategy::Lazy2, true) => { - compress_block_lazy_any_dict::(ms, seq_store, rep, src, dict_mode) + let block = LazyBlock::new(ms, src, dict_mode)?; + let args = LazyArgs { + start: Start::new(ms, &block, rep, dict_mode), + ms, + seq_store, + rep, + src, + block, + }; + match dict_mode { + DictMode::NoDict => compress_block_lazy_any_strategy::(args, strategy, use_row), + DictMode::ExtDict => compress_block_lazy_any_strategy::(args, strategy, use_row), + DictMode::DictMatchState => { + compress_block_lazy_any_strategy::(args, strategy, use_row) } - (Strategy::BtLazy2, false) => compress_block_lazy_any_dict::( - ms, seq_store, rep, src, dict_mode, - ), - _ => Err(BUG), } } From 9f3963650e23c007bf43d058c66c7f3e11098d63 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:03:53 +0000 Subject: [PATCH 054/199] zstd: fast only asks whether an index was too low once the bytes are the same --- src/zstd/compress/double_fast.rs | 33 ++++++++++++++++---------------- src/zstd/compress/fast.rs | 16 ++++++++-------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index b20e02c5225d..7ab44b702c47 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -576,23 +576,24 @@ impl Finder { ip = end; } else { // The match, and how much of it is known. `ip` is moved to where it is for. - let (place, known) = if window.has_after_first::<{ u64::MAX }>(place_long, bytes0) { - (place_long, 8) - } else if window.has_after_first::<0xFFFF_FFFF>(place_short, bytes0) { - // A short match: a long one at the next position is preferred. - let bytes1 = le64_at::<1, _>(here); - let place3 = table_swap(hash_long, hash8(bytes1, h_bits_l, 0), curr + 1) - .wrapping_sub(origin_index); - if window.read64(place3) == Some(bytes1) && place3 > 0 { - ip += 1; - (place3, 8) + let (place, known) = + if window.has::<{ u64::MAX }>(place_long, bytes0) && place_long > 0 { + (place_long, 8) + } else if window.has::<0xFFFF_FFFF>(place_short, bytes0) && place_short > 0 { + // A short match: a long one at the next position is preferred. + let bytes1 = le64_at::<1, _>(here); + let place3 = table_swap(hash_long, hash8(bytes1, h_bits_l, 0), curr + 1) + .wrapping_sub(origin_index); + if window.read64(place3) == Some(bytes1) && place3 > 0 { + ip += 1; + (place3, 8) + } else { + (place_short, 4) + } } else { - (place_short, 4) - } - } else { - ip += ((ip - sink.anchor) >> K_SEARCH_STRENGTH) + 1; - continue; - }; + ip += ((ip - sink.anchor) >> K_SEARCH_STRENGTH) + 1; + continue; + }; let end = ip + known + window.count(ip + known, place + known); look_ahead::(near, end, hash_long, h_bits_l, hash_small, h_bits_s); let back = window.count_backwards(ip, sink.anchor, place); diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index b31204521a5c..767646046c63 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -394,12 +394,12 @@ impl<'a> Segments<'a> { spot.segment.read64(spot.pos) } - /// Whether the bits of `MASK` are the same in `bytes` and in the 8 bytes at `place`, which are in one segment, and - /// `place` is not the first one. + /// Whether the bits of `MASK` are the same in `bytes` and in the 8 bytes at `place`, which are in one segment, as they + /// are wherever a table leads to. #[inline(always)] - pub(super) fn has_after_first(self, place: u32, bytes: u64) -> bool { + pub(super) fn has(self, place: u32, bytes: u64) -> bool { let spot = self.spot(place); - spot.segment.has::(spot.pos, bytes) && place > 0 + spot.segment.has::(spot.pos, bytes) } /// The 4 bytes that are `offset` before the position `pos`, if they are in one segment. Which one that is does not @@ -499,7 +499,7 @@ impl Finder { break (ip2 - back, end, REPCODE1_TO_OFFBASE); } let mut match0 = idx0.wrapping_sub(base_index); - if near.read32(match0) == Some(bytes0 as u32) { + if near.has::<0xFFFF_FFFF>(match0, bytes0) { table_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); } else { // As far as they are hashed or compared. @@ -507,7 +507,7 @@ impl Finder { current0 += 1; let idx1 = table_swap(hash_table, hash_of::(bytes1, hlog), current0); match0 = idx1.wrapping_sub(base_index); - if near.read32(match0) != Some(bytes1 as u32) { + if !near.has::<0xFFFF_FFFF>(match0, bytes1) { current0 = base_index.wrapping_add(ip2); bytes0 = le64_at::<0, _>(ahead); ip2 += step; @@ -632,14 +632,14 @@ impl Finder { } let mut place = idx0.wrapping_sub(origin_index); let mut next = u32::MAX; - if window.read32(place) == Some(bytes0 as u32) { + if window.has::<0xFFFF_FFFF>(place, bytes0) { table_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); } else { let bytes1 = bytes0 >> 8; current0 += 1; let idx1 = table_swap(hash_table, hash_of::(bytes1, hlog), current0); place = idx1.wrapping_sub(origin_index); - if window.read32(place) != Some(bytes1 as u32) { + if !window.has::<0xFFFF_FFFF>(place, bytes1) { current0 = base_index.wrapping_add(ip2); bytes0 = le64_at::<0, _>(ahead); ip2 += step; From 045f5752d2e1320dd80d109c6add303ed65595b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:04:58 +0000 Subject: [PATCH 055/199] zstd: nor does dfast for the long match after a short one --- src/zstd/compress/double_fast.rs | 6 +++--- src/zstd/compress/fast.rs | 7 ------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 7ab44b702c47..aa676f5ee5b6 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -207,7 +207,7 @@ impl Finder { let end = ip + 4 + near.count(ip + 4, matchs0 + 4); let idxl1 = table_get(hash_long, hash8(bytes1, h_bits_l, 0)); let matchl1 = idxl1.wrapping_sub(base_index); - if matchl1 > 0 && near.read64(matchl1) == Some(bytes1) { + if near.has::<{ u64::MAX }>(matchl1, bytes1) && matchl1 > 0 { let end1 = ip1 + 8 + near.count(ip1 + 8, matchl1 + 8); if end1 - ip1 > end - ip { ip = ip1; @@ -406,7 +406,7 @@ impl Finder { let bytes1 = le64_at::<1, _>(here); let match_long1 = table_swap(hash_long, hash8(bytes1, h_bits_l, 0), curr + 1) .wrapping_sub(base_index); - if near.read64(match_long1) == Some(bytes1) { + if near.has::<{ u64::MAX }>(match_long1, bytes1) { ip += 1; break 'found (window.place_of(match_long1), 8); } @@ -584,7 +584,7 @@ impl Finder { let bytes1 = le64_at::<1, _>(here); let place3 = table_swap(hash_long, hash8(bytes1, h_bits_l, 0), curr + 1) .wrapping_sub(origin_index); - if window.read64(place3) == Some(bytes1) && place3 > 0 { + if window.has::<{ u64::MAX }>(place3, bytes1) && place3 > 0 { ip += 1; (place3, 8) } else { diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 767646046c63..1e3e41e9f8cf 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -387,13 +387,6 @@ impl<'a> Segments<'a> { spot.segment.read32(spot.pos) } - /// The 8 bytes at `place`, if they are in one segment. - #[inline(always)] - pub(super) fn read64(self, place: u32) -> Option { - let spot = self.spot(place); - spot.segment.read64(spot.pos) - } - /// Whether the bits of `MASK` are the same in `bytes` and in the 8 bytes at `place`, which are in one segment, as they /// are wherever a table leads to. #[inline(always)] From fe2ce75f2f4f54ffadb76e0738e9d31d8c131b7e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:07:38 +0000 Subject: [PATCH 056/199] zstd: cheaper searches in the tree of a dictionary, and long runs of btlazy2 insertions Most searches find nothing for their hash in the dictionary: its table is read before the search in the window, which also hides the wait for it, and the way down its tree is only set up when there is one. What is rarely called takes copies rather than references, so that what a search works on stays in registers. After a long match or in data that does not compress, btlazy2 adds many positions at once. They go through a loop of their own, without hints, which are of no use when no iteration waits for another. --- src/zstd/compress/internal.rs | 34 +++++-- src/zstd/compress/lazy.rs | 186 +++++++++++++++++++++++----------- src/zstd/compress/opt.rs | 23 +++-- 3 files changed, 171 insertions(+), 72 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index de3310cbefa3..117fc9bef81f 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1036,7 +1036,7 @@ impl<'a> TreeInput<'a> { /// [`Near::compare`] wherever the match is, for what [`Near::compare_chunks`] leaves. #[cold] #[inline(never)] - fn compare_rest(&self, ip: u32, match_index: u32, from: u32) -> Result { + fn compare_rest(self, ip: u32, match_index: u32, from: u32) -> Result { // A position of the older segment, as one of the current segment, only makes sense with a length that brings // it there. if u64::from(match_index) + u64::from(from) >= u64::from(self.dict_limit) { @@ -1140,6 +1140,7 @@ pub(crate) fn bt_walk( } /// The tree of a dictionary that a search goes on in. It is only read. +#[derive(Clone, Copy)] pub(crate) struct DictTree<'a> { dms: DictMatchState<'a>, bt: &'a Nodes, @@ -1177,12 +1178,28 @@ impl<'a> DictTree<'a> { }) } + /// The root of the tree of `dms` for `hash`, or 0 if there is none, and a hint that the way down is about to start. + #[inline(always)] + pub(crate) fn root(dms: DictMatchState<'_>, hash: usize) -> Result { + let root = dms.ms.hash_table.at(hash)?; + if root <= dms.ms.window.low_limit { + return Ok(0); + } + let (bt, _) = dms.ms.chain_table.as_chunks(); + bt_prefetch(bt, root); + prefetch_read( + dms.buf.as_ptr().wrapping_add(dms.pos_of(root)), + Locality::L1, + ); + Ok(root) + } + /// [`Near::compare`] for a match in the dictionary, for what [`Near::compare_chunks_with`] leaves. A match that /// reaches the end of the dictionary goes on at the start of the current segment. #[cold] #[inline(never)] fn compare_rest( - &self, + self, input: &TreeInput<'_>, ip: u32, dict_match_index: u32, @@ -1205,24 +1222,29 @@ impl<'a> DictTree<'a> { Ok(Common::followed_by(len, match_byte, byte)) } - /// Goes down the tree for the position `ip` of `input.near`, whose hash for the table of the dictionary is `hash`. - /// `visit` is called with each position that is met on the way, at most `nb_compares` of them, and what it has in + /// Goes down the tree for the position `ip` of `input.near`, from what [`Self::root`] returns for it. `visit` is called with each position that is met on the way, at most `nb_compares` of them, and what it has in /// common with `ip`, and returns whether to go on. #[inline(always)] pub(crate) fn walk( &self, input: &TreeInput<'_>, ip: u32, - hash: usize, + root: u32, mut nb_compares: u32, mut visit: impl FnMut(u32, Common) -> Result, ) -> Result<()> { - let mut dict_match_index = self.dms.ms.hash_table.at(hash)?; + let mut dict_match_index = root; let mut common_smaller = 0u32; let mut common_larger = 0u32; while nb_compares != 0 && dict_match_index > self.low_limit { let [smaller_child, larger_child] = bt_children(self.bt, dict_match_index); let from = common_smaller.min(common_larger); + // Whichever way it goes, the next step does not wait as long for memory. + for child in [smaller_child, larger_child] { + bt_prefetch(self.bt, child); + let pos = child.wrapping_sub(self.low_limit); + self.content.prefetch(pos.wrapping_add(from)); + } let match_pos = dict_match_index.wrapping_sub(self.low_limit); let chunks = input .near diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 20abe131b738..357323d1cb3e 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -92,6 +92,18 @@ impl<'a> Dubt<'a> { #[inline(always)] fn update(&mut self) -> Result<()> { let near = self.input.near; + if self.curr.saturating_sub(self.next_to_update) >= LONG_UPDATE { + let first = self.next_to_update..self.curr; + let dict_limit = self.input.dict_limit; + return update_many( + self.hash_table, + self.bt, + near, + self.hasher, + first, + dict_limit, + ); + } for idx in self.next_to_update..self.curr { let bytes = near.read64(idx.wrapping_sub(self.input.dict_limit)); let h = self.hasher.hash(bytes.ok_or(BUG)?); @@ -117,28 +129,19 @@ impl<'a> Dubt<'a> { self.low_limit }; if curr < self.input.dict_limit { - return self.insert_dubt1_in_dict(curr, nb_compares, bt_low, window_low); + return sort_dubt1_in_dict( + self.bt, + self.input, + self.low_limit, + curr, + nb_compares, + bt_low, + window_low, + ); } sort_dubt1(self.bt, &self.input, curr, nb_compares, bt_low, window_low) } - /// [`Self::insert_dubt1`] of a position of the older segment, which all that is before it is in too. - #[cold] - #[inline(never)] - fn insert_dubt1_in_dict( - &mut self, - curr: u32, - nb_compares: u32, - bt_low: u32, - window_low: u32, - ) -> Result<()> { - let no_dict = Prefix { buf: &[], base: 0 }; - let dict = self.input.dict; - let dict_end = dict.pos_of(self.input.dict_limit); - let input = TreeInput::new(dict, no_dict, self.low_limit, dict_end)?; - sort_dubt1(self.bt, &input, curr, nb_compares, bt_low, window_low) - } - /// `ZSTD_DUBT_findBestMatch`, but for the dictionary. Returns the length of the best match, or 0, and how many more /// positions could have been compared if the dictionary is to be searched. #[inline(always)] @@ -226,43 +229,99 @@ impl<'a> Dubt<'a> { } impl Dubt<'_> { - /// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary, with a search that found `found`. Returns - /// the length of the best match. - #[inline(never)] - fn find_better_dict_match( - &self, - dms: DictMatchState<'_>, - found: &DubtFound, - offset_ptr: &mut u32, - ) -> Result { - let curr = self.curr; - let tree = DictTree::new(dms, self.low_limit)?; + /// [`DictTree::root`] for the position that is searched for. + #[inline(always)] + fn dict_root(&self, dms: &DictMatchState<'_>) -> Result { let bytes = self.input.near.read64(self.ip).ok_or(BUG)?; - let hash = self.hasher.with_log(dms.ms.c_params.hash_log).hash(bytes); - let mut best_length = found.best_length; - tree.walk( - &self.input, - self.ip, - hash, - found.nb_compares, - |dict_match_index, common| { - if common.len > best_length { - let match_index = dict_match_index.wrapping_add(tree.index_delta); - if 4 * (common.len - best_length) as i32 - > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 - - highbit32(offset_ptr.wrapping_add(1)) as i32 - { - best_length = common.len; - *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); - } - // At the end, there is no way to know which is smaller. - return Ok(common.match_is_smaller.is_some()); - } - Ok(true) - }, - )?; - Ok(best_length) + let hasher = self.hasher.with_log(dms.ms.c_params.hash_log); + DictTree::root(*dms, hasher.hash(bytes)) + } +} + +/// From that many positions on, [`Dubt::update`] leaves them to [`update_many`]. +const LONG_UPDATE: u32 = 32; + +/// `ZSTD_updateDUBT` of the indices `indices`, which are in `near`, whose first index is `dict_limit`. There are enough +/// of them not to wait for memory. +#[cold] +#[inline(never)] +fn update_many( + hash_table: &mut [u32], + bt: &mut Nodes, + near: Near<'_>, + hasher: Hasher, + indices: core::ops::Range, + dict_limit: u32, +) -> Result<()> { + let first = indices.start.wrapping_sub(dict_limit) as usize; + let last = indices.end.wrapping_sub(dict_limit) as usize; + let bytes = near.bytes().sub(first..last + 7)?; + for (idx, bytes) in indices.zip(bytes.array_windows()) { + let h = hasher.hash(u64::from_le_bytes(*bytes)); + let match_index = table_get(hash_table, h); + table_put(hash_table, h, idx); + bt_set_children(bt, idx, [match_index, ZSTD_DUBT_UNSORTED_MARK]); } + Ok(()) +} + +/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary whose root is `root`, with a search for the +/// position `ip` of `input.near` that found `found`. `low_limit` is `window.lowLimit`. Returns the length of the best +/// match. +#[inline(never)] +fn find_better_dict_match( + dms: &DictMatchState<'_>, + input: TreeInput<'_>, + low_limit: u32, + ip: u32, + root: u32, + found: DubtFound, + offset_ptr: &mut u32, +) -> Result { + let tree = DictTree::new(*dms, low_limit)?; + let curr = ip.wrapping_add(input.dict_limit); + let mut best_length = found.best_length; + tree.walk( + &input, + ip, + root, + found.nb_compares, + |dict_match_index, common| { + if common.len > best_length { + let match_index = dict_match_index.wrapping_add(tree.index_delta); + if 4 * (common.len - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(offset_ptr.wrapping_add(1)) as i32 + { + best_length = common.len; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + // At the end, there is no way to know which is smaller. + return Ok(common.match_is_smaller.is_some()); + } + Ok(true) + }, + )?; + Ok(best_length) +} + +/// [`sort_dubt1`] of a position of the older segment of `input`, which starts at the index `low_limit`. All that is +/// before the position is in that segment too. +#[cold] +#[inline(never)] +fn sort_dubt1_in_dict( + bt: &mut Nodes, + input: TreeInput<'_>, + low_limit: u32, + curr: u32, + nb_compares: u32, + bt_low: u32, + window_low: u32, +) -> Result<()> { + let no_dict = Prefix { buf: &[], base: 0 }; + let dict_end = input.dict.pos_of(input.dict_limit); + let dict = TreeInput::new(input.dict, no_dict, low_limit, dict_end)?; + sort_dubt1(bt, &dict, curr, nb_compares, bt_low, window_low) } /// The body of `ZSTD_insertDUBT1`, for a position of `input.near`. @@ -292,6 +351,7 @@ fn sort_dubt1( } /// What [`Dubt::find_best_match`] returns. +#[derive(Clone, Copy)] struct DubtFound { best_length: u32, nb_compares: u32, @@ -313,12 +373,24 @@ fn bt_search( return Ok(0); } let mut dubt = Dubt::new(ms, src, ip, i_limit, mls)?; + let dms = src.dms.as_ref().filter(|_| search_dms); + let dict_root = match dms { + Some(dms) => dubt.dict_root(dms)?, + None => 0, + }; dubt.update()?; let found = dubt.find_best_match(off_base_ptr)?; - let best_length = if search_dms && found.nb_compares != 0 { - dubt.find_better_dict_match(src.dms.ok_or(BUG)?, &found, off_base_ptr)? - } else { - found.best_length + let best_length = match dms { + Some(dms) if dict_root != 0 && found.nb_compares != 0 => find_better_dict_match( + dms, + dubt.input, + dubt.low_limit, + dubt.ip, + dict_root, + found, + off_base_ptr, + )?, + _ => found.best_length, }; ms.next_to_update = dubt.next_to_update; Ok(best_length as usize) diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index b03d47367eae..d627397e3888 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -692,6 +692,13 @@ impl<'a> Trees<'a> { let min_match = self.min_match; let bytes = near.read64(ip).ok_or(BUG)?; + let mut dict_root = match self.dms { + Some(dms) => { + let hasher = self.hasher.with_log(dms.ms.c_params.hash_log); + DictTree::root(dms, hasher.hash(bytes))? + } + None => 0, + }; let window_low = self.lowest_match_index(curr); let mut mnum = 0u32; let mut best_length = min_match - 1; @@ -761,7 +768,6 @@ impl<'a> Trees<'a> { // The farthest position any match refers to, which detects repetitive patterns. let mut match_end_idx = curr.wrapping_add(8 + 1); - let mut search_dms = self.dms.is_some(); let nb_compares = self.walk(ip, bytes, match_low, |match_index, common| { if common.len > best_length { if common.len > match_end_idx.wrapping_sub(match_index) { @@ -779,20 +785,20 @@ impl<'a> Trees<'a> { if common.len > ZSTD_OPT_NUM as u32 || common.match_is_smaller.is_none() { // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. // The dictionary is not to be searched either. - search_dms = false; + dict_root = 0; return Ok(false); } } Ok(true) })?; - if search_dms { + if dict_root != 0 && nb_compares != 0 { let mut found = Found { mnum, best_length, match_end_idx, }; - self.search_dms(matches, ip, bytes, window_low, nb_compares, &mut found)?; + self.search_dms(matches, ip, dict_root, window_low, nb_compares, &mut found)?; mnum = found.mnum; match_end_idx = found.match_end_idx; } @@ -802,14 +808,14 @@ impl<'a> Trees<'a> { Ok(mnum) } - /// The end of `ZSTD_insertBtAndGetAllMatches`: goes on in the tree of the dictionary, for `nb_compares` positions at - /// most. + /// The end of `ZSTD_insertBtAndGetAllMatches`: goes on in the tree of the dictionary whose root is `root`, for + /// `nb_compares` positions at most. #[inline(never)] fn search_dms( &self, matches: &mut [Match], ip: u32, - bytes: u64, + root: u32, window_low: u32, nb_compares: u32, found: &mut Found, @@ -818,8 +824,7 @@ impl<'a> Trees<'a> { let input = self.input; let curr = ip.wrapping_add(input.dict_limit); let tree = DictTree::new(dms, window_low)?; - let hash = self.hasher.with_log(dms.ms.c_params.hash_log).hash(bytes); - tree.walk(&input, ip, hash, nb_compares, |dict_match_index, common| { + tree.walk(&input, ip, root, nb_compares, |dict_match_index, common| { if common.len > found.best_length { let match_index = dict_match_index.wrapping_add(tree.index_delta); if common.len > found.match_end_idx.wrapping_sub(match_index) { From b83480690b257cc594a4183862bfad1de2e8a6f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:14:51 +0000 Subject: [PATCH 057/199] zstd: Table::reset tells whether the memory is new It took two bools, one of which said to write to new memory right away. Only the tag table wants that, and now does it itself. The hash table of the other strategies is left alone until it is used, as it was. --- src/zstd/compress/cctx.rs | 20 +++++++++++--------- src/zstd/compress/internal.rs | 12 +++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 507c29106579..f9d4f083eac3 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -194,8 +194,7 @@ fn reset_match_state( // As long as indices go on, the ones in the tables are too low to be valid. let clean = force_reset_index == IndexResetPolicy::Reset && crp != CompResetPolicy::LeaveDirty; let use_rows = row_match_finder_used(c_params.strat(), use_row_match_finder); - // An entry of a row is written before it is read. - ms.hash_table.reset(sizes.h_size, clean, !use_rows)?; + ms.hash_table.reset(sizes.h_size, clean)?; reset_table(&mut ms.chain_table, sizes.chain_size, clean)?; reset_table(&mut ms.hash_table3, sizes.h3_size, clean)?; @@ -205,14 +204,17 @@ fn reset_match_state( ResetTarget::CDict => RowTags::Flat, }; let tag_table_size = lazy::row_tag_table_size(ms.row_tags, sizes.h_size); - if for_who == ResetTarget::CCtx { - // A new salt instead of erasing the tags. - ms.tag_table.reset(tag_table_size, false, true)?; - ms.hash_salt = bitmix(ms.hash_salt, 8) ^ bitmix(u64::from(ms.hash_salt_entropy), 4); - } else { - ms.tag_table.reset(tag_table_size, true, true)?; - ms.hash_salt = 0; + // In a context, a new salt instead of erasing the tags. + if (ms.tag_table).reset(tag_table_size, for_who == ResetTarget::CDict)? { + // Most of it is read before it is written, for which the system would provide each page twice. + ms.tag_table.fill(0); } + ms.hash_salt = match for_who { + ResetTarget::CCtx => { + bitmix(ms.hash_salt, 8) ^ bitmix(u64::from(ms.hash_salt_entropy), 4) + } + ResetTarget::CDict => 0, + }; let row_log = c_params.search_log.clamp(4, 6); ms.row_hash_log = c_params.hash_log - row_log; } else { diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 680e5322fcd2..5d6fc6e13eaa 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1333,11 +1333,9 @@ impl Table { } impl Table { - /// Gives it the length `len`. What it holds is 0, or if `clean` is false possibly what it held before. - /// - /// `read_first` tells that most of it is to be read before it is written. New memory is then written to right away: - /// else the system would provide each page twice, once to be read and once to be written. - pub(crate) fn reset(&mut self, len: usize, clean: bool, read_first: bool) -> Result<()> { + /// Gives it the length `len`. What it holds is 0, or if `clean` is false possibly what it held before. Returns + /// whether it is new memory. + pub(crate) fn reset(&mut self, len: usize, clean: bool) -> Result { let is_new = self.len != len; if is_new { // First, so that both are not allocated at once. @@ -1351,10 +1349,10 @@ impl Table { len, }; } - if if is_new { read_first } else { clean } { + if clean && !is_new { self.fill(T::default()); } - Ok(()) + Ok(is_new) } } From 16f007aad2f31961d34c3823ed456e764bda76b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:44:12 +0000 Subject: [PATCH 058/199] zstd: sequences are decoded by the batch, then executed --- src/zstd/common/bitstream.rs | 6 + src/zstd/decompress/block.rs | 362 ++++++++++++++++++------- src/zstd/decompress/mod.rs | 20 +- src/zstd/decompress/seq_table.rs | 444 +++++++++++++++++++++++-------- 4 files changed, 619 insertions(+), 213 deletions(-) diff --git a/src/zstd/common/bitstream.rs b/src/zstd/common/bitstream.rs index 6798ff23443d..2576cab18fed 100644 --- a/src/zstd/common/bitstream.rs +++ b/src/zstd/common/bitstream.rs @@ -142,6 +142,12 @@ impl<'a> BitDStream<'a> { }) } + /// The stream up to the end of what `container` was read from. + #[inline(always)] + pub(crate) fn bytes(&self) -> &'a [u8] { + self.src + } + /// `BIT_lookBits`: `nb_bits` is less than 32. #[inline(always)] pub(crate) fn look_bits(&self, nb_bits: u32) -> u64 { diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 63556b0307e3..ab821027f5e2 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -2,14 +2,14 @@ use super::huf::{self, HufDTable}; use super::seq_table::{ - LL_BASE, LL_DEFAULT_DTABLE, ML_BASE, ML_DEFAULT_DTABLE, OF_BASE, OF_BITS, OF_DEFAULT_DTABLE, - SeqTable, + Cell, LL_BASE, ML_BASE, OF_BASE, OF_BITS, OffsetTable, SeqTable, SeqTableSources, SeqTables, + TableSource, }; use crate::common::bitstream::BitDStream; use crate::common::entropy::read_ncount; use crate::common::{ BlockType, LL_BITS, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MIN_CBLOCK_SIZE, - MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, ML_BITS, ML_FSE_LOG, OFF_FSE_LOG, + MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, MINMATCH, ML_BITS, ML_FSE_LOG, OFF_FSE_LOG, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; use crate::error::{ErrorCode, Result}; @@ -50,9 +50,7 @@ pub(crate) fn get_c_block_size(src: &[u8]) -> Result<(usize, BlockProperties)> { /// `ZSTD_entropyDTables_t` #[derive(Clone)] pub(crate) struct EntropyDTables { - pub(crate) ll_table: SeqTable<{ 1 << LL_FSE_LOG }>, - pub(crate) of_table: SeqTable<{ 1 << OFF_FSE_LOG }>, - pub(crate) ml_table: SeqTable<{ 1 << ML_FSE_LOG }>, + pub(crate) seq: SeqTables, pub(crate) huf_table: HufDTable, pub(crate) rep: [u32; ZSTD_REP_NUM], } @@ -60,26 +58,13 @@ pub(crate) struct EntropyDTables { impl EntropyDTables { pub(crate) const fn new() -> Self { Self { - ll_table: SeqTable::new(), - of_table: SeqTable::new(), - ml_table: SeqTable::new(), + seq: SeqTables::new(), huf_table: HufDTable::new(), rep: REP_START_VALUE, } } } -/// Which table `LLTptr`, `MLTptr`, `OFTptr` or `HUFptr` points to. -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum TableSource { - /// The context's own. - Local, - /// The dictionary's. - Dict, - /// The one the format predefines. There is none for literals. - Predefined, -} - /// What precedes the contiguous output, for matches to reach into: `dictEnd` and `virtualStart`. #[derive(Clone, Copy)] pub(crate) enum ExtDict<'a> { @@ -173,9 +158,7 @@ struct Literals { /// The state of a `ZSTD_DCtx` that lasts from block to block. pub(crate) struct BlockDecoder { pub(crate) entropy: Box, - pub(crate) ll_source: TableSource, - pub(crate) of_source: TableSource, - pub(crate) ml_source: TableSource, + pub(crate) seq_sources: SeqTableSources, pub(crate) huf_source: TableSource, /// There is a Huffman table to repeat. pub(crate) lit_entropy: bool, @@ -190,9 +173,7 @@ impl BlockDecoder { pub(crate) fn new() -> Self { Self { entropy: Box::new(EntropyDTables::new()), - ll_source: TableSource::Local, - of_source: TableSource::Local, - ml_source: TableSource::Local, + seq_sources: SeqTableSources::all(TableSource::Local), huf_source: TableSource::Local, lit_entropy: false, fse_entropy: false, @@ -415,8 +396,8 @@ impl BlockDecoder { let fse_entropy = self.fse_entropy; ip += build_seq_table( - &mut self.entropy.ll_table, - &mut self.ll_source, + &mut self.entropy.seq.ll, + &mut self.seq_sources.ll, SymbolEncodingType::from_bits(modes >> 6), MAX_LL, LL_FSE_LOG, @@ -427,8 +408,8 @@ impl BlockDecoder { ) .map_err(|_| ErrorCode::CorruptionDetected)?; ip += build_seq_table( - &mut self.entropy.of_table, - &mut self.of_source, + &mut self.entropy.seq.of, + &mut self.seq_sources.of, SymbolEncodingType::from_bits(modes >> 4), MAX_OFF, OFF_FSE_LOG, @@ -439,8 +420,8 @@ impl BlockDecoder { ) .map_err(|_| ErrorCode::CorruptionDetected)?; ip += build_seq_table( - &mut self.entropy.ml_table, - &mut self.ml_source, + &mut self.entropy.seq.ml, + &mut self.seq_sources.ml, SymbolEncodingType::from_bits(modes >> 2), MAX_ML, ML_FSE_LOG, @@ -493,34 +474,14 @@ impl BlockDecoder { lit_buffer, .. } = self; - let EntropyDTables { - ll_table, - of_table, - ml_table, - rep, - .. - } = &mut **entropy; - let ll_table = match (self.ll_source, dict_entropy) { - (TableSource::Predefined, _) => &LL_DEFAULT_DTABLE, - (TableSource::Dict, Some(dict)) => &dict.ll_table, - _ => &*ll_table, - }; - let of_table = match (self.of_source, dict_entropy) { - (TableSource::Predefined, _) => &OF_DEFAULT_DTABLE, - (TableSource::Dict, Some(dict)) => &dict.of_table, - _ => &*of_table, - }; - let ml_table = match (self.ml_source, dict_entropy) { - (TableSource::Predefined, _) => &ML_DEFAULT_DTABLE, - (TableSource::Dict, Some(dict)) => &dict.ml_table, - _ => &*ml_table, - }; + let EntropyDTables { seq, rep, .. } = &mut **entropy; + let tables = seq.gather(&mut self.seq_sources, dict_entropy.map(|dict| &dict.seq)); // With matches far back in a long history, it pays to know the sequences ahead of time. Nothing is prefetched // here, but which of two problems in a damaged block is reported depends on the order. if !use_prefetch_decoder && total_history_size > (1 << 24) && nb_seq > 8 { const MIN_SHARE: u32 = 7; - use_prefetch_decoder = long_offset_share(of_table) >= MIN_SHARE; + use_prefetch_decoder = long_offset_share(&tables.of) >= MIN_SHARE; } self.ddict_is_cold = false; if nb_seq > 0 { @@ -557,7 +518,7 @@ impl BlockDecoder { }; if nb_seq > 0 { - let mut seq_state = SeqState::new(sequences, ll_table, of_table, ml_table, rep)?; + let mut seq_state = SeqState::new(sequences, tables, rep)?; if use_prefetch_decoder { exec.run_long(&mut seq_state, nb_seq)?; } else { @@ -571,8 +532,8 @@ impl BlockDecoder { } /// `ZSTD_buildSeqTable`: returns the size of the table's description in `src`. -fn build_seq_table( - table: &mut SeqTable, +fn build_seq_table( + table: &mut SeqTable, source: &mut TableSource, encoding_type: SymbolEncodingType, max: u32, @@ -625,7 +586,7 @@ fn build_seq_table( } /// `ZSTD_getOffsetInfo`: how many of 256 offset codes have more than 22 bits. -fn long_offset_share(of_table: &SeqTable<{ 1 << OFF_FSE_LOG }>) -> u32 { +fn long_offset_share(of_table: &OffsetTable) -> u32 { let count = of_table .symbols() .iter() @@ -642,6 +603,15 @@ struct Sequence { offset: usize, } +/// A sequence whose offset is as it is coded: 1 to 3 for one of the last offsets, or 3 more than the offset. +#[derive(Clone, Copy, Default)] +struct RawSequence { + offset_value: u32, + lit_length: u32, + /// Above `MINMATCH`. + match_length: u32, +} + /// `seqState_t` #[derive(Clone, Copy)] struct SeqState<'a> { @@ -649,20 +619,12 @@ struct SeqState<'a> { state_ll: usize, state_off: usize, state_ml: usize, - ll_table: &'a SeqTable<{ 1 << LL_FSE_LOG }>, - of_table: &'a SeqTable<{ 1 << OFF_FSE_LOG }>, - ml_table: &'a SeqTable<{ 1 << ML_FSE_LOG }>, + tables: &'a SeqTables, prev_offset: [usize; ZSTD_REP_NUM], } impl<'a> SeqState<'a> { - fn new( - src: &'a [u8], - ll_table: &'a SeqTable<{ 1 << LL_FSE_LOG }>, - of_table: &'a SeqTable<{ 1 << OFF_FSE_LOG }>, - ml_table: &'a SeqTable<{ 1 << ML_FSE_LOG }>, - rep: &[u32; ZSTD_REP_NUM], - ) -> Result { + fn new(src: &'a [u8], tables: &'a SeqTables, rep: &[u32; ZSTD_REP_NUM]) -> Result { let mut bit_d = BitDStream::new(src).map_err(|_| ErrorCode::CorruptionDetected)?; // `ZSTD_initFseState` let mut init = |table_log: u32| { @@ -670,28 +632,120 @@ impl<'a> SeqState<'a> { bit_d.reload(); state }; - let state_ll = init(ll_table.table_log); - let state_off = init(of_table.table_log); - let state_ml = init(ml_table.table_log); + let state_ll = init(tables.ll.table_log); + let state_off = init(tables.of.table_log); + let state_ml = init(tables.ml.table_log); Ok(Self { bit_d, state_ll, state_off, state_ml, - ll_table, - of_table, - ml_table, + tables, prev_offset: rep.map(|offset| offset as usize), }) } + /// Decodes sequences for as long as the start of the stream is far. Returns how many that is. + #[inline(never)] + fn decode_batch(&mut self, out: &mut [RawSequence]) -> Result { + /// The 64 bits that end at `position`, turned so that the next one to read is the highest. Those that are read + /// then go to the other end. + #[inline(always)] + fn load(containers: &[u8], position: u32) -> Option { + // Wraps around less than `FAR` bytes from the start. + let index = (position >> 3).wrapping_sub(FAR as u32) as usize; + let container = containers.get(index..index + 8)?; + let container = u64::from_le_bytes(container.try_into().ok()?); + Some(container.rotate_left(!position & 7)) + } + /// A sequence takes at most 12 bytes. + const FAR: usize = 24; + + let src = self.bit_d.bytes(); + // A block is not that large. + if src.len() < FAR || src.len() >= 1 << 28 || self.bit_d.bits_consumed > 64 { + return Ok(0); + } + // How many bits are left to read, and 7. + let mut position = src.len() as u32 * 8 + 7 - self.bit_d.bits_consumed; + let containers = src.get(FAR - 8..).unwrap_or_default(); + let tables = self.tables; + let mut state_ll = self.state_ll as u32 & 511; + let mut state_ml = self.state_ml as u32 & 511; + let mut state_off = self.state_off as u32 & 255; + + let room = out.len(); + let mut slots = out.iter_mut(); + let mut unused = 0; + for slot in slots.by_ref() { + let Some(mut bits) = load(containers, position) else { + unused = 1; + break; + }; + let ll_cell = tables.ll.cell(state_ll as usize); + let ml_cell = tables.ml.cell(state_ml as usize); + let of_cell = tables.of.cell(state_off as usize); + let of_bits = u32::from(of_cell.nb_additional_bits); + let ml_bits = u32::from(ml_cell.nb_additional_bits); + let ll_bits = u32::from(ll_cell.nb_additional_bits); + + let mut match_length = u32::from(ml_cell.base_value); + let mut lit_length = u32::from(ll_cell.base_value); + let first_value = of_cell.first_value; + bits = bits.rotate_left(of_bits); + position -= of_bits; + let offset_value = first_value + (bits as u32 & (first_value - 1)); + if ml_bits + ll_bits != 0 { + bits = bits.rotate_left(ml_bits); + position -= ml_bits; + match_length = ml_cell.base_value() + (bits as u32 & ((1 << ml_bits) - 1)); + // What follows has to fit in 57 bits with the 26 of the next states. + bits = load(src, position + (FAR as u32 - 8) * 8).unwrap_or(0); + bits = bits.rotate_left(ll_bits); + position -= ll_bits; + lit_length = ll_cell.base_value() + (bits as u32 & ((1 << ll_bits) - 1)); + } + *slot = RawSequence { + offset_value, + lit_length, + match_length, + }; + + let ll_nb_bits = u32::from(ll_cell.nb_bits); + let ml_nb_bits = u32::from(ml_cell.nb_bits); + let of_nb_bits = u32::from(of_cell.nb_bits); + position -= ll_nb_bits + ml_nb_bits + of_nb_bits; + bits = bits.rotate_left(ll_nb_bits); + state_ll = (u32::from(ll_cell.next_state) + + (bits as u32 & u32::from(ll_cell.state_mask))) + & 511; + bits = bits.rotate_left(ml_nb_bits); + state_ml = (u32::from(ml_cell.next_state) + + (bits as u32 & u32::from(ml_cell.state_mask))) + & 511; + bits = bits.rotate_left(of_nb_bits); + state_off = (u32::from(of_cell.next_state) + + (bits as u32 & u32::from(of_cell.state_mask))) + & 255; + } + + let count = room - slots.len() - unused; + if count > 0 { + self.bit_d = BitDStream::resume(src, (position >> 3) as usize - 8, !position & 7)?; + self.state_ll = state_ll as usize; + self.state_ml = state_ml as usize; + self.state_off = state_off as usize; + } + Ok(count) + } + /// `ZSTD_decodeSequence` #[inline(always)] fn decode_sequence(&mut self, is_last_seq: bool) -> Sequence { - let ll_d_info = self.ll_table.get(self.state_ll); - let ml_d_info = self.ml_table.get(self.state_ml); - let of_d_info = self.of_table.get(self.state_off); - let mut match_length = ml_d_info.base_value; + let ll_d_info = self.tables.ll.get(self.state_ll); + let ml_d_info = self.tables.ml.get(self.state_ml); + let of_d_info = self.tables.of.get(self.state_off); + let mut match_length = ml_d_info.base_value + MINMATCH; let mut lit_length = ll_d_info.base_value; let of_base = of_d_info.base_value as usize; let ll_bits = u32::from(ll_d_info.nb_additional_bits); @@ -763,6 +817,19 @@ fn copy_within(buf: &mut [u8], src: usize, dst: usize) -> Result Ok(()) } +/// The same for positions that do not overflow. Whether they are in `buf`. +#[inline(always)] +fn copy_16(buf: &mut [u8], src: usize, dst: usize) -> bool { + let Some(Ok(chunk)) = buf.get(src..src + 16).map(<[u8; 16]>::try_from) else { + return false; + }; + let Some(to) = buf.get_mut(dst..dst + 16) else { + return false; + }; + to.copy_from_slice(&chunk); + true +} + /// `buf[dst + i] = buf[src + i]`, one after the other, so that a match repeats itself where it overlaps its copy. /// `src` is less than `dst`. fn copy_match_exact(buf: &mut [u8], src: usize, dst: usize, len: usize) -> Result<()> { @@ -1023,25 +1090,136 @@ impl<'a> Executor<'_, 'a> { } } + /// Executes `sequences` from `index` on, for as long as nothing is close to its end and they are of the common kind. + /// Returns the index of the first that is not, whose offset value then is no reference to one of the last offsets: + /// it is 0 if that is not valid. + /// + /// The last offsets are those of the 3 sequences before, which are rewritten for that where it takes it. + #[inline(never)] + fn exec_batch(&mut self, sequences: &mut [RawSequence], mut index: usize) -> usize { + // Positions fit in 32 bits, so that the compiler sees that nothing added to them overflows. + const MAX_BACK: usize = 3 << 30; + let start = self.prefix_start.max(self.op.saturating_sub(MAX_BACK)); + let end = self.buf.len().min(start + u32::MAX as usize); + let buf = self.buf.get_mut(start..end).unwrap_or_default(); + let mut op = self.op.wrapping_sub(start) as u32; + let lits = self.lits.get(..u32::MAX as usize).unwrap_or(self.lits); + let mut lit_pos = 0u32; + + while let Some(raw) = sequences.get(index) { + let lit_length = raw.lit_length as usize; + let match_length = raw.match_length as usize + MINMATCH as usize; + let mut offset_value = raw.offset_value; + if offset_value < 16 + 3 { + if offset_value <= 3 { + let Some([last2, last1, last0, this]) = index + .checked_sub(3) + .and_then(|first| sequences.get_mut(first..)) + .and_then(<[RawSequence]>::first_chunk_mut) + else { + break; + }; + let rep0 = last0.offset_value; + let rep1 = last1.offset_value; + let rep2 = last2.offset_value; + let reps = match offset_value - 1 + u32::from(lit_length == 0) { + 0 => [rep0, rep1, rep2], + 1 => [rep1, rep0, rep2], + 2 => [rep2, rep0, rep1], + // An offset of 0 is not valid. + _ if rep0 == 1 + 3 => [0, rep0, rep1], + _ => [rep0 - 1, rep0, rep1], + }; + [this.offset_value, last0.offset_value, last1.offset_value] = reps; + [offset_value, ..] = reps; + } + if offset_value < 16 + 3 { + break; + } + } + + let o_lit_end = op as usize + lit_length; + let o_match_end = o_lit_end + match_length; + let lit_end = lit_pos as usize + lit_length; + if o_match_end + WILDCOPY_OVERLENGTH > buf.len() + || lit_end + WILDCOPY_OVERLENGTH > lits.len() + || lit_length > 32 + || match_length > 64 + || offset_value as usize > o_lit_end + 3 + { + break; + } + let match_pos = o_lit_end + 3 - offset_value as usize; + let (Some(to), Some(from)) = ( + buf.get_mut(op as usize..op as usize + 32), + lits.get(lit_pos as usize..lit_pos as usize + 32), + ) else { + break; + }; + to.copy_from_slice(from); + if !copy_16(buf, match_pos, o_lit_end) || !copy_16(buf, match_pos + 16, o_lit_end + 16) + { + break; + } + if match_length > 32 + && (!copy_16(buf, match_pos + 32, o_lit_end + 32) + || !copy_16(buf, match_pos + 48, o_lit_end + 48)) + { + break; + } + + op = o_match_end as u32; + lit_pos = lit_end as u32; + index += 1; + } + + self.op = op as usize + start; + self.lits = self.lits.get(lit_pos as usize..).unwrap_or_default(); + index + } + /// `ZSTD_decompressSequences_body` and `ZSTD_decompressSequences_bodySplitLitBuffer` fn run(&mut self, seq_state: &mut SeqState<'_>, nb_seq: usize) -> Result<()> { - let mut exec = self.fork(); - let mut state = *seq_state; + const BATCH: usize = 32; + // After 3 that only stand for the last offsets. + let mut batch = [RawSequence::default(); 3 + BATCH]; let mut remaining = nb_seq; - while exec.split && remaining > 0 { - let sequence = state.decode_sequence(remaining == 1); - remaining -= 1; - exec.exec_sequence_maybe_split(sequence)?; - } while remaining > 0 { - let sequence = state.decode_sequence(remaining == 1); + if !self.split { + let room = batch.span_mut(3, BATCH.min(remaining - 1))?; + let count = seq_state.decode_batch(room)?; + if count > 0 { + remaining -= count; + let sequences = batch.sub_mut(..3 + count)?; + for (last, offset) in + sequences.iter_mut().zip(seq_state.prev_offset.iter().rev()) + { + last.offset_value = *offset as u32 + 3; + } + let mut index = 3; + loop { + index = self.exec_batch(sequences, index); + let Some(raw) = sequences.get(index) else { + break; + }; + self.exec_sequence::(Sequence { + lit_length: raw.lit_length, + match_length: raw.match_length + MINMATCH, + offset: (raw.offset_value as usize).wrapping_sub(3), + })?; + index += 1; + } + for (last, offset) in sequences.iter().rev().zip(&mut seq_state.prev_offset) { + *offset = last.offset_value as usize - 3; + } + continue; + } + } + let sequence = seq_state.decode_sequence(remaining == 1); remaining -= 1; - exec.exec_sequence::(sequence)?; + self.exec_sequence_maybe_split(sequence)?; } - let exec = (exec.op, exec.lits, exec.split); - self.join(exec); - *seq_state = state; - if !state.bit_d.is_at_end() { + if !seq_state.bit_d.is_at_end() { return Err(ErrorCode::CorruptionDetected); } Ok(()) diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 88906b1c9d4c..6a4f80f933f1 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -6,10 +6,8 @@ mod seq_table; use std::borrow::Cow; -use block::{ - BlockDecoder, BlockOutput, EntropyDTables, ExtDict, Streaming, TableSource, get_c_block_size, -}; -use seq_table::{LL_BASE, ML_BASE, OF_BASE, OF_BITS}; +use block::{BlockDecoder, BlockOutput, EntropyDTables, ExtDict, Streaming, get_c_block_size}; +use seq_table::{LL_BASE, ML_BASE, OF_BASE, OF_BITS, SeqTableSources, TableSource}; use crate::common::entropy::read_ncount; use crate::common::xxhash::Xxh64; @@ -330,7 +328,7 @@ fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result { if header.max_symbol_value > MAX_OFF || header.table_log > OFF_FSE_LOG { return Err(CORRUPTED); } - entropy.of_table.build( + entropy.seq.of.build( ncount.sub(..header.max_symbol_value as usize + 1)?, &OF_BASE, &OF_BITS, @@ -344,7 +342,7 @@ fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result { if header.max_symbol_value > MAX_ML || header.table_log > ML_FSE_LOG { return Err(CORRUPTED); } - entropy.ml_table.build( + entropy.seq.ml.build( ncount.sub(..header.max_symbol_value as usize + 1)?, &ML_BASE, &ML_BITS, @@ -358,7 +356,7 @@ fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result { if header.max_symbol_value > MAX_LL || header.table_log > LL_FSE_LOG { return Err(CORRUPTED); } - entropy.ll_table.build( + entropy.seq.ll.build( ncount.sub(..header.max_symbol_value as usize + 1)?, &LL_BASE, &LL_BITS, @@ -676,9 +674,7 @@ impl DState { self.dict_id = 0; self.b_type = BlockType::Reserved; self.block.entropy.rep = REP_START_VALUE; - self.block.ll_source = TableSource::Local; - self.block.ml_source = TableSource::Local; - self.block.of_source = TableSource::Local; + self.block.seq_sources = SeqTableSources::all(TableSource::Local); self.block.huf_source = TableSource::Local; // `ZSTD_copyDDictParameters` @@ -688,9 +684,7 @@ impl DState { if let Some(entropy) = &ddict.entropy { self.block.lit_entropy = true; self.block.fse_entropy = true; - self.block.ll_source = TableSource::Dict; - self.block.ml_source = TableSource::Dict; - self.block.of_source = TableSource::Dict; + self.block.seq_sources = SeqTableSources::all(TableSource::Dict); self.block.huf_source = TableSource::Dict; self.block.entropy.rep = entropy.rep; } diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs index 54c99f97f3a6..c3e672e6ad25 100644 --- a/src/zstd/decompress/seq_table.rs +++ b/src/zstd/decompress/seq_table.rs @@ -32,15 +32,16 @@ pub(crate) const OF_BITS: [u8; MAX_OFF as usize + 1] = [ 24, 25, 26, 27, 28, 29, 30, 31, ]; +/// What a match length code starts at, above `MINMATCH`. #[rustfmt::skip] pub(crate) const ML_BASE: [u32; MAX_ML as usize + 1] = [ - 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, - 35, 37, 39, 41, 43, 47, 51, 59, - 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, - 0x1003, 0x2003, 0x4003, 0x8003, 0x10003, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 34, 36, 38, 40, 44, 48, 56, + 64, 80, 96, 0x80, 0x100, 0x200, 0x400, 0x800, + 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, ]; /// `ZSTD_seqSymbol` @@ -52,77 +53,142 @@ pub(crate) struct SeqSymbol { pub(crate) base_value: u32, } -const fn s(next_state: u16, nb_additional_bits: u8, nb_bits: u8, base_value: u32) -> SeqSymbol { - SeqSymbol { +/// How a table keeps a [`SeqSymbol`]. +pub(crate) trait Cell: Copy + 'static { + const EMPTY: Self; + const EMPTY_REF: &'static Self; + fn pack(symbol: SeqSymbol) -> Self; + fn unpack(self) -> SeqSymbol; +} + +/// A state of the table of literal lengths or of match lengths. +#[derive(Clone, Copy)] +pub(crate) struct LengthCell { + /// All of the base value but its bit 16, which is set if and only if there are 16 additional bits. + pub(crate) base_value: u16, + pub(crate) next_state: u16, + /// As many bits set as `nb_bits`. + pub(crate) state_mask: u16, + pub(crate) nb_additional_bits: u8, + pub(crate) nb_bits: u8, +} + +const fn l(next_state: u16, nb_additional_bits: u8, nb_bits: u8, base_value: u32) -> LengthCell { + LengthCell { + base_value: base_value as u16, next_state, + state_mask: (1u16 << nb_bits) - 1, nb_additional_bits, nb_bits, - base_value, } } -/// A table of at most `N` states, which is a power of 2. -#[derive(Clone)] -pub(crate) struct SeqTable { - pub(crate) table_log: u32, - table: [SeqSymbol; N], +impl LengthCell { + #[inline(always)] + pub(crate) fn base_value(self) -> u32 { + u32::from(self.base_value) + (u32::from(self.nb_additional_bits >> 4) << 16) + } } -impl SeqTable { - pub(crate) const fn new() -> Self { +impl Cell for LengthCell { + const EMPTY: Self = l(0, 0, 0, 0); + const EMPTY_REF: &'static Self = &Self::EMPTY; + + #[inline(always)] + fn pack(symbol: SeqSymbol) -> Self { Self { - table_log: 0, - table: [s(0, 0, 0, 0); N], + base_value: symbol.base_value as u16, + next_state: symbol.next_state, + state_mask: (1u16.wrapping_shl(u32::from(symbol.nb_bits))).wrapping_sub(1), + nb_additional_bits: symbol.nb_additional_bits, + nb_bits: symbol.nb_bits, } } - const fn predefined(table_log: u32, symbols: [SeqSymbol; M]) -> Self { - let mut table = [s(0, 0, 0, 0); N]; - let (head, _) = table.split_at_mut(M); - head.copy_from_slice(&symbols); - Self { table_log, table } + #[inline(always)] + fn unpack(self) -> SeqSymbol { + SeqSymbol { + next_state: self.next_state, + nb_additional_bits: self.nb_additional_bits, + nb_bits: self.nb_bits, + base_value: self.base_value(), + } } +} - #[inline(always)] - pub(crate) fn get(&self, state: usize) -> SeqSymbol { - self.table - .get(state & (N - 1)) - .copied() - .unwrap_or(s(0, 0, 0, 0)) +/// A state of the table of offsets. +#[derive(Clone, Copy)] +pub(crate) struct OffsetCell { + /// 1 with `nb_additional_bits` zeros after it. With the additional bits in their place, 1 to 3 stand for one of the + /// last offsets, and what is above for an offset 3 less than it. + pub(crate) first_value: u32, + pub(crate) next_state: u8, + /// As many bits set as `nb_bits`. + pub(crate) state_mask: u8, + pub(crate) nb_additional_bits: u8, + pub(crate) nb_bits: u8, +} + +const fn o(next_state: u8, nb_additional_bits: u8, nb_bits: u8) -> OffsetCell { + OffsetCell { + first_value: 1 << nb_additional_bits, + next_state, + state_mask: ((1u16 << nb_bits) - 1) as u8, + nb_additional_bits, + nb_bits, } +} - /// The states in use. - pub(crate) fn symbols(&self) -> &[SeqSymbol] { - self.table.get(..1 << self.table_log).unwrap_or_default() +impl Cell for OffsetCell { + const EMPTY: Self = o(0, 0, 0); + const EMPTY_REF: &'static Self = &Self::EMPTY; + + #[inline(always)] + fn pack(symbol: SeqSymbol) -> Self { + Self { + first_value: 1u32.wrapping_shl(u32::from(symbol.nb_additional_bits)), + next_state: symbol.next_state as u8, + state_mask: (1u16.wrapping_shl(u32::from(symbol.nb_bits))).wrapping_sub(1) as u8, + nb_additional_bits: symbol.nb_additional_bits, + nb_bits: symbol.nb_bits, + } } - /// `ZSTD_buildSeqTable_rle` - pub(crate) fn build_rle(&mut self, base_value: u32, nb_additional_bits: u8) { - self.table_log = 0; - if let Some(cell) = self.table.first_mut() { - *cell = s(0, nb_additional_bits, 0, base_value); + #[inline(always)] + fn unpack(self) -> SeqSymbol { + SeqSymbol { + next_state: u16::from(self.next_state), + nb_additional_bits: self.nb_additional_bits, + nb_bits: self.nb_bits, + // `OF_BASE` + base_value: if self.nb_additional_bits < 2 { + u32::from(self.nb_additional_bits) + } else { + self.first_value - 3 + }, } } +} - /// `ZSTD_buildFSETable`: `table_log` is such that the table fits, and `normalized_counter` adds up to its size. - pub(crate) fn build( - &mut self, - normalized_counter: &[i16], - base_value: &[u32], - nb_additional_bits: &[u8], - table_log: u32, - ) -> Result<()> { +const MAX_TABLE_SIZE: usize = 1 << 9; + +/// The first two passes of `ZSTD_buildFSETable`: which symbol each state decodes. +struct Spread { + symbols: [u8; MAX_TABLE_SIZE + 8], + symbol_next: [u16; MAX_SEQ as usize + 1], +} + +impl Spread { + fn new(normalized_counter: &[i16], table_log: u32) -> Result { let table_size = 1usize << table_log; - let table_decode = self.table.sub_mut(..table_size)?; + let mut symbols = [0u8; MAX_TABLE_SIZE + 8]; let mut symbol_next = [0u16; MAX_SEQ as usize + 1]; let mut high_threshold = table_size - 1; - // Until the last pass, `base_value` holds the symbol. - self.table_log = table_log; for (symbol, (&count, next)) in normalized_counter.iter().zip(&mut symbol_next).enumerate() { if count == -1 { - table_decode.at_mut(high_threshold)?.base_value = symbol as u32; + symbols.put(high_threshold, symbol as u8)?; high_threshold = high_threshold.wrapping_sub(1); *next = 1; } else { @@ -135,27 +201,27 @@ impl SeqTable { if high_threshold == table_size - 1 { // No symbol of low probability: lay the symbols down in order, then spread them in a second pass that // has no branch to mispredict. - let mut spread = [0u8; (1 << 9) + 8]; + let mut in_order = [0u8; MAX_TABLE_SIZE + 8]; let mut pos = 0usize; let mut sv = 0u64; for &count in normalized_counter { let n = count as usize; for i in (0..n.max(1)).step_by(8) { - write_le64(&mut spread, pos + i, sv)?; + write_le64(&mut in_order, pos + i, sv)?; } pos += n; sv += 0x0101_0101_0101_0101; } let mut position = 0usize; - for &symbol in spread.sub(..table_size)? { - table_decode.at_mut(position)?.base_value = u32::from(symbol); + for &symbol in in_order.sub(..table_size)? { + symbols.put(position, symbol)?; position = (position + step) & table_mask; } } else { let mut position = 0usize; for (symbol, &count) in normalized_counter.iter().enumerate() { for _ in 0..count { - table_decode.at_mut(position)?.base_value = symbol as u32; + symbols.put(position, symbol as u8)?; position = (position + step) & table_mask; while position > high_threshold { position = (position + step) & table_mask; @@ -163,79 +229,241 @@ impl SeqTable { } } } + Ok(Self { + symbols, + symbol_next, + }) + } +} + +/// A table of at most `N` states, which is a power of 2. +#[derive(Clone)] +pub(crate) struct SeqTable { + pub(crate) table_log: u32, + pub(crate) table: [C; N], +} + +pub(crate) type LengthTable = SeqTable; +pub(crate) type OffsetTable = SeqTable; + +impl SeqTable { + pub(crate) const fn new() -> Self { + Self { + table_log: 0, + table: [C::EMPTY; N], + } + } + + const fn predefined(table_log: u32, symbols: [C; M]) -> Self { + let mut table = [C::EMPTY; N]; + let (head, _) = table.split_at_mut(M); + head.copy_from_slice(&symbols); + Self { table_log, table } + } + + #[inline(always)] + pub(crate) fn get(&self, state: usize) -> SeqSymbol { + self.table + .get(state & (N - 1)) + .copied() + .unwrap_or(C::EMPTY) + .unpack() + } + + #[inline(always)] + pub(crate) fn cell(&self, state: usize) -> &C { + self.table.get(state & (N - 1)).unwrap_or(C::EMPTY_REF) + } + + /// The states in use. + pub(crate) fn symbols(&self) -> &[C] { + self.table.get(..1 << self.table_log).unwrap_or_default() + } + + fn copy_from(&mut self, other: &Self) { + self.table_log = other.table_log; + let symbols = other.symbols(); + if let Some(to) = self.table.get_mut(..symbols.len()) { + to.copy_from_slice(symbols); + } + } + + /// `ZSTD_buildSeqTable_rle` + pub(crate) fn build_rle(&mut self, base_value: u32, nb_additional_bits: u8) { + self.table_log = 0; + if let Some(cell) = self.table.first_mut() { + *cell = C::pack(SeqSymbol { + next_state: 0, + nb_additional_bits, + nb_bits: 0, + base_value, + }); + } + } + + /// `ZSTD_buildFSETable`: `table_log` is such that the table fits, and `normalized_counter` adds up to its size. + pub(crate) fn build( + &mut self, + normalized_counter: &[i16], + base_value: &[u32], + nb_additional_bits: &[u8], + table_log: u32, + ) -> Result<()> { + let table_size = 1usize << table_log; + let table_decode = self.table.sub_mut(..table_size)?; + self.table_log = table_log; + let Spread { + symbols, + mut symbol_next, + } = Spread::new(normalized_counter, table_log)?; - for cell in table_decode { - let symbol = cell.base_value as usize; + for (cell, &symbol) in table_decode.iter_mut().zip(&symbols) { + let symbol = usize::from(symbol); let next = symbol_next.at_mut(symbol)?; let next_state = u32::from(*next); *next = next.wrapping_add(1); let nb_bits = table_log.wrapping_sub(highbit32(next_state)); - *cell = SeqSymbol { + *cell = C::pack(SeqSymbol { next_state: (next_state.wrapping_shl(nb_bits) as usize).wrapping_sub(table_size) as u16, nb_additional_bits: nb_additional_bits.at(symbol)?, nb_bits: nb_bits as u8, base_value: base_value.at(symbol)?, - }; + }); } Ok(()) } } #[rustfmt::skip] -const LL_DEFAULT: [SeqSymbol; 64] = [ - s( 0, 0, 4, 0), s(16, 0, 4, 0), s(32, 0, 5, 1), s( 0, 0, 5, 3), - s( 0, 0, 5, 4), s( 0, 0, 5, 6), s( 0, 0, 5, 7), s( 0, 0, 5, 9), - s( 0, 0, 5, 10), s( 0, 0, 5, 12), s( 0, 0, 6, 14), s( 0, 1, 5, 16), - s( 0, 1, 5, 20), s( 0, 1, 5, 22), s( 0, 2, 5, 28), s( 0, 3, 5, 32), - s( 0, 4, 5, 48), s(32, 6, 5, 64), s( 0, 7, 5, 128), s( 0, 8, 6, 256), - s( 0, 10, 6, 1024), s( 0, 12, 6, 4096), s(32, 0, 4, 0), s( 0, 0, 4, 1), - s( 0, 0, 5, 2), s(32, 0, 5, 4), s( 0, 0, 5, 5), s(32, 0, 5, 7), - s( 0, 0, 5, 8), s(32, 0, 5, 10), s( 0, 0, 5, 11), s( 0, 0, 6, 13), - s(32, 1, 5, 16), s( 0, 1, 5, 18), s(32, 1, 5, 22), s( 0, 2, 5, 24), - s(32, 3, 5, 32), s( 0, 3, 5, 40), s( 0, 6, 4, 64), s(16, 6, 4, 64), - s(32, 7, 5, 128), s( 0, 9, 6, 512), s( 0, 11, 6, 2048), s(48, 0, 4, 0), - s(16, 0, 4, 1), s(32, 0, 5, 2), s(32, 0, 5, 3), s(32, 0, 5, 5), - s(32, 0, 5, 6), s(32, 0, 5, 8), s(32, 0, 5, 9), s(32, 0, 5, 11), - s(32, 0, 5, 12), s( 0, 0, 6, 15), s(32, 1, 5, 18), s(32, 1, 5, 20), - s(32, 2, 5, 24), s(32, 2, 5, 28), s(32, 3, 5, 40), s(32, 4, 5, 48), - s( 0, 16, 6, 65536), s( 0, 15, 6, 32768), s( 0, 14, 6, 16384), s( 0, 13, 6, 8192), +const LL_DEFAULT: [LengthCell; 64] = [ + l( 0, 0, 4, 0), l(16, 0, 4, 0), l(32, 0, 5, 1), l( 0, 0, 5, 3), + l( 0, 0, 5, 4), l( 0, 0, 5, 6), l( 0, 0, 5, 7), l( 0, 0, 5, 9), + l( 0, 0, 5, 10), l( 0, 0, 5, 12), l( 0, 0, 6, 14), l( 0, 1, 5, 16), + l( 0, 1, 5, 20), l( 0, 1, 5, 22), l( 0, 2, 5, 28), l( 0, 3, 5, 32), + l( 0, 4, 5, 48), l(32, 6, 5, 64), l( 0, 7, 5, 128), l( 0, 8, 6, 256), + l( 0, 10, 6, 1024), l( 0, 12, 6, 4096), l(32, 0, 4, 0), l( 0, 0, 4, 1), + l( 0, 0, 5, 2), l(32, 0, 5, 4), l( 0, 0, 5, 5), l(32, 0, 5, 7), + l( 0, 0, 5, 8), l(32, 0, 5, 10), l( 0, 0, 5, 11), l( 0, 0, 6, 13), + l(32, 1, 5, 16), l( 0, 1, 5, 18), l(32, 1, 5, 22), l( 0, 2, 5, 24), + l(32, 3, 5, 32), l( 0, 3, 5, 40), l( 0, 6, 4, 64), l(16, 6, 4, 64), + l(32, 7, 5, 128), l( 0, 9, 6, 512), l( 0, 11, 6, 2048), l(48, 0, 4, 0), + l(16, 0, 4, 1), l(32, 0, 5, 2), l(32, 0, 5, 3), l(32, 0, 5, 5), + l(32, 0, 5, 6), l(32, 0, 5, 8), l(32, 0, 5, 9), l(32, 0, 5, 11), + l(32, 0, 5, 12), l( 0, 0, 6, 15), l(32, 1, 5, 18), l(32, 1, 5, 20), + l(32, 2, 5, 24), l(32, 2, 5, 28), l(32, 3, 5, 40), l(32, 4, 5, 48), + l( 0, 16, 6, 65536), l( 0, 15, 6, 32768), l( 0, 14, 6, 16384), l( 0, 13, 6, 8192), ]; #[rustfmt::skip] -const OF_DEFAULT: [SeqSymbol; 32] = [ - s( 0, 0, 5, 0), s( 0, 6, 4, 61), s( 0, 9, 5, 509), s( 0, 15, 5, 32765), - s( 0, 21, 5, 2097149), s( 0, 3, 5, 5), s( 0, 7, 4, 125), s( 0, 12, 5, 4093), - s( 0, 18, 5, 262141), s( 0, 23, 5, 8388605), s( 0, 5, 5, 29), s( 0, 8, 4, 253), - s( 0, 14, 5, 16381), s( 0, 20, 5, 1048573), s( 0, 2, 5, 1), s(16, 7, 4, 125), - s( 0, 11, 5, 2045), s( 0, 17, 5, 131069), s( 0, 22, 5, 4194301), s( 0, 4, 5, 13), - s(16, 8, 4, 253), s( 0, 13, 5, 8189), s( 0, 19, 5, 524285), s( 0, 1, 5, 1), - s(16, 6, 4, 61), s( 0, 10, 5, 1021), s( 0, 16, 5, 65533), s( 0, 28, 5, 268435453), - s( 0, 27, 5, 134217725), s( 0, 26, 5, 67108861), s( 0, 25, 5, 33554429), s( 0, 24, 5, 16777213), +const OF_DEFAULT: [OffsetCell; 32] = [ + o( 0, 0, 5), o( 0, 6, 4), o( 0, 9, 5), o( 0, 15, 5), + o( 0, 21, 5), o( 0, 3, 5), o( 0, 7, 4), o( 0, 12, 5), + o( 0, 18, 5), o( 0, 23, 5), o( 0, 5, 5), o( 0, 8, 4), + o( 0, 14, 5), o( 0, 20, 5), o( 0, 2, 5), o(16, 7, 4), + o( 0, 11, 5), o( 0, 17, 5), o( 0, 22, 5), o( 0, 4, 5), + o(16, 8, 4), o( 0, 13, 5), o( 0, 19, 5), o( 0, 1, 5), + o(16, 6, 4), o( 0, 10, 5), o( 0, 16, 5), o( 0, 28, 5), + o( 0, 27, 5), o( 0, 26, 5), o( 0, 25, 5), o( 0, 24, 5), ]; #[rustfmt::skip] -const ML_DEFAULT: [SeqSymbol; 64] = [ - s( 0, 0, 6, 3), s( 0, 0, 4, 4), s(32, 0, 5, 5), s( 0, 0, 5, 6), - s( 0, 0, 5, 8), s( 0, 0, 5, 9), s( 0, 0, 5, 11), s( 0, 0, 6, 13), - s( 0, 0, 6, 16), s( 0, 0, 6, 19), s( 0, 0, 6, 22), s( 0, 0, 6, 25), - s( 0, 0, 6, 28), s( 0, 0, 6, 31), s( 0, 0, 6, 34), s( 0, 1, 6, 37), - s( 0, 1, 6, 41), s( 0, 2, 6, 47), s( 0, 3, 6, 59), s( 0, 4, 6, 83), - s( 0, 7, 6, 131), s( 0, 9, 6, 515), s(16, 0, 4, 4), s( 0, 0, 4, 5), - s(32, 0, 5, 6), s( 0, 0, 5, 7), s(32, 0, 5, 9), s( 0, 0, 5, 10), - s( 0, 0, 6, 12), s( 0, 0, 6, 15), s( 0, 0, 6, 18), s( 0, 0, 6, 21), - s( 0, 0, 6, 24), s( 0, 0, 6, 27), s( 0, 0, 6, 30), s( 0, 0, 6, 33), - s( 0, 1, 6, 35), s( 0, 1, 6, 39), s( 0, 2, 6, 43), s( 0, 3, 6, 51), - s( 0, 4, 6, 67), s( 0, 5, 6, 99), s( 0, 8, 6, 259), s(32, 0, 4, 4), - s(48, 0, 4, 4), s(16, 0, 4, 5), s(32, 0, 5, 7), s(32, 0, 5, 8), - s(32, 0, 5, 10), s(32, 0, 5, 11), s( 0, 0, 6, 14), s( 0, 0, 6, 17), - s( 0, 0, 6, 20), s( 0, 0, 6, 23), s( 0, 0, 6, 26), s( 0, 0, 6, 29), - s( 0, 0, 6, 32), s( 0, 16, 6, 65539), s( 0, 15, 6, 32771), s( 0, 14, 6, 16387), - s( 0, 13, 6, 8195), s( 0, 12, 6, 4099), s( 0, 11, 6, 2051), s( 0, 10, 6, 1027), +const ML_DEFAULT: [LengthCell; 64] = [ + l( 0, 0, 6, 0), l( 0, 0, 4, 1), l(32, 0, 5, 2), l( 0, 0, 5, 3), + l( 0, 0, 5, 5), l( 0, 0, 5, 6), l( 0, 0, 5, 8), l( 0, 0, 6, 10), + l( 0, 0, 6, 13), l( 0, 0, 6, 16), l( 0, 0, 6, 19), l( 0, 0, 6, 22), + l( 0, 0, 6, 25), l( 0, 0, 6, 28), l( 0, 0, 6, 31), l( 0, 1, 6, 34), + l( 0, 1, 6, 38), l( 0, 2, 6, 44), l( 0, 3, 6, 56), l( 0, 4, 6, 80), + l( 0, 7, 6, 128), l( 0, 9, 6, 512), l(16, 0, 4, 1), l( 0, 0, 4, 2), + l(32, 0, 5, 3), l( 0, 0, 5, 4), l(32, 0, 5, 6), l( 0, 0, 5, 7), + l( 0, 0, 6, 9), l( 0, 0, 6, 12), l( 0, 0, 6, 15), l( 0, 0, 6, 18), + l( 0, 0, 6, 21), l( 0, 0, 6, 24), l( 0, 0, 6, 27), l( 0, 0, 6, 30), + l( 0, 1, 6, 32), l( 0, 1, 6, 36), l( 0, 2, 6, 40), l( 0, 3, 6, 48), + l( 0, 4, 6, 64), l( 0, 5, 6, 96), l( 0, 8, 6, 256), l(32, 0, 4, 1), + l(48, 0, 4, 1), l(16, 0, 4, 2), l(32, 0, 5, 4), l(32, 0, 5, 5), + l(32, 0, 5, 7), l(32, 0, 5, 8), l( 0, 0, 6, 11), l( 0, 0, 6, 14), + l( 0, 0, 6, 17), l( 0, 0, 6, 20), l( 0, 0, 6, 23), l( 0, 0, 6, 26), + l( 0, 0, 6, 29), l( 0, 16, 6, 65536), l( 0, 15, 6, 32768), l( 0, 14, 6, 16384), + l( 0, 13, 6, 8192), l( 0, 12, 6, 4096), l( 0, 11, 6, 2048), l( 0, 10, 6, 1024), ]; -pub(crate) static LL_DEFAULT_DTABLE: SeqTable<512> = - SeqTable::predefined(LL_DEFAULT_NORM_LOG, LL_DEFAULT); -pub(crate) static OF_DEFAULT_DTABLE: SeqTable<256> = - SeqTable::predefined(OF_DEFAULT_NORM_LOG, OF_DEFAULT); -pub(crate) static ML_DEFAULT_DTABLE: SeqTable<512> = - SeqTable::predefined(ML_DEFAULT_NORM_LOG, ML_DEFAULT); + +/// Which table `LLTptr`, `MLTptr`, `OFTptr` or `HUFptr` points to. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TableSource { + /// The context's own. + Local, + /// The dictionary's. + Dict, + /// The one the format predefines. There is none for literals. + Predefined, +} + +/// The tables of literal lengths, offsets and match lengths. +#[derive(Clone)] +pub(crate) struct SeqTables { + pub(crate) ll: LengthTable, + pub(crate) of: OffsetTable, + pub(crate) ml: LengthTable, +} + +static DEFAULT_TABLES: SeqTables = SeqTables { + ll: SeqTable::predefined(LL_DEFAULT_NORM_LOG, LL_DEFAULT), + of: SeqTable::predefined(OF_DEFAULT_NORM_LOG, OF_DEFAULT), + ml: SeqTable::predefined(ML_DEFAULT_NORM_LOG, ML_DEFAULT), +}; + +/// `LLTptr`, `OFTptr` and `MLTptr` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct SeqTableSources { + pub(crate) ll: TableSource, + pub(crate) of: TableSource, + pub(crate) ml: TableSource, +} + +impl SeqTableSources { + pub(crate) const fn all(source: TableSource) -> Self { + Self { + ll: source, + of: source, + ml: source, + } + } +} + +impl SeqTables { + pub(crate) const fn new() -> Self { + Self { + ll: SeqTable::new(), + of: SeqTable::new(), + ml: SeqTable::new(), + } + } + + /// The three tables that `sources` designates, side by side. If they are not, those that are not in `self` are + /// copied there, which `sources` then says. + pub(crate) fn gather<'a>( + &'a mut self, + sources: &mut SeqTableSources, + dict: Option<&'a Self>, + ) -> &'a Self { + let from = |source| match (source, dict) { + (TableSource::Predefined, _) => Some(&DEFAULT_TABLES), + (TableSource::Dict, Some(dict)) => Some(dict), + _ => None, + }; + if *sources == SeqTableSources::all(sources.ll) { + return from(sources.ll).unwrap_or(self); + } + if let Some(other) = from(sources.ll) { + self.ll.copy_from(&other.ll); + } + if let Some(other) = from(sources.of) { + self.of.copy_from(&other.of); + } + if let Some(other) = from(sources.ml) { + self.ml.copy_from(&other.ml); + } + *sources = SeqTableSources::all(TableSource::Local); + self + } +} From 3d05caca25da9b02bfee0c29637b06bbf45bbc91 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:56:35 +0000 Subject: [PATCH 059/199] zstd: a Huffman lookup yields up to 3 literals, and streams that end where they should need not be decoded as C does --- src/zstd/decompress/block.rs | 2 +- src/zstd/decompress/huf.rs | 1247 ++++++++---------------------- src/zstd/decompress/huf_exact.rs | 1001 ++++++++++++++++++++++++ src/zstd/decompress/mod.rs | 1 + 4 files changed, 1341 insertions(+), 910 deletions(-) create mode 100644 src/zstd/decompress/huf_exact.rs diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index ab821027f5e2..9c150adaa8c6 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -1180,7 +1180,7 @@ impl<'a> Executor<'_, 'a> { /// `ZSTD_decompressSequences_body` and `ZSTD_decompressSequences_bodySplitLitBuffer` fn run(&mut self, seq_state: &mut SeqState<'_>, nb_seq: usize) -> Result<()> { - const BATCH: usize = 32; + const BATCH: usize = 128; // After 3 that only stand for the last offsets. let mut batch = [RawSequence::default(); 3 + BATCH]; let mut remaining = nb_seq; diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index 293a380e0c9f..ac89b2851ddb 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -1,964 +1,398 @@ -//! `huf_decompress.c`: Huffman decoding of literals, one symbol (X1) or up to two symbols (X2) per table lookup. +//! Huffman decoding of literals. +//! +//! Streams that end exactly where they should decode to the same thing whichever way it is done, so that is done in +//! the way that is fastest here. The others are left to [`huf_exact`](super::huf_exact). +use super::huf_exact::{self, TableType, select_decoder_x2}; use crate::common::bitstream::{BitDStream, DStreamStatus}; use crate::common::entropy::huf_read_stats; use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; use crate::error::{ErrorCode, Result}; -use crate::util::{BUG, SliceExt, highbit32, read_le16, read_le64}; +use crate::util::{BUG, SliceExt, read_le16}; -/// Tables of this log are decoded by the loops that keep four streams in registers. -const HUF_DECODER_FAST_TABLELOG: u32 = 11; - -const TABLE_SIZE: usize = 1 << HUF_TABLELOG_MAX; -const TABLE_MASK: usize = TABLE_SIZE - 1; const NB_SYMBOLS: usize = HUF_SYMBOLVALUE_MAX as usize + 1; const NB_RANKS: usize = HUF_TABLELOG_MAX as usize + 1; +const TABLE_SIZE: usize = 1 << HUF_TABLELOG_MAX; + +/// The description of a Huffman code. +#[derive(Clone)] +pub(super) struct Code { + /// A symbol of weight `w` has a code of `table_log + 1 - w` bits, or none if that is 0. + pub(super) weights: [u8; NB_SYMBOLS], + /// How many symbols have each weight. + pub(super) rank_stats: [u32; NB_RANKS], + pub(super) nb_symbols: u32, + pub(super) table_log: u32, +} + +/// What a lookup yields: up to 3 symbols, then a byte with how many bits they take in its low 6 bits and how many they +/// are above. +type Entry = [u8; 4]; -#[derive(Clone, Copy, PartialEq, Eq)] -enum TableType { - X1, - X2, +const COUNT_SHIFT: u32 = 6; + +/// As a number, what a symbol adds to an entry that has `index` symbols. +const fn entry(symbol: u8, index: u32, nb_bits: u32) -> u32 { + ((symbol as u32) << (8 * index)) | ((nb_bits + (1 << COUNT_SHIFT)) << 24) } -/// `HUF_DTable`, of capacity `ZSTD_HUFFDTABLE_CAPACITY_LOG`. +/// `HUF_DTable` #[derive(Clone)] pub(crate) struct HufDTable { + code: Code, + /// The kind of table that the C code has. table_type: TableType, - table_log: u32, - /// `HUF_DEltX1`: the number of bits in the low byte, and the symbol above it. - x1: [u16; TABLE_SIZE], - /// `HUF_DEltX2`: two symbols, then the number of bits, then how many of the symbols count. - x2: [u32; TABLE_SIZE], + /// How many bits `table` is indexed with. + log: u32, + table: [Entry; TABLE_SIZE], } impl HufDTable { pub(crate) const fn new() -> Self { Self { + code: Code { + weights: [0; NB_SYMBOLS], + rank_stats: [0; NB_RANKS], + nb_symbols: 0, + table_log: 0, + }, table_type: TableType::X1, - table_log: 0, - x1: [0; TABLE_SIZE], - x2: [0; TABLE_SIZE], - } - } -} - -#[inline(always)] -fn entry_x1(dt: &[u16; TABLE_SIZE], index: u64) -> u16 { - dt.get(index as usize & TABLE_MASK).copied().unwrap_or(0) -} - -#[inline(always)] -fn entry_x2(dt: &[u32; TABLE_SIZE], index: u64) -> u32 { - dt.get(index as usize & TABLE_MASK).copied().unwrap_or(0) -} - -// ── the four streams ────────────────────────────────────────────────────── - -/// Where the four streams of `src` start, after the jump table, and where the last one ends. -struct Streams { - start: [usize; 4], - length: [usize; 4], -} - -impl Streams { - /// `src` has at least 10 bytes. - fn new(src: &[u8]) -> Result { - let length1 = usize::from(read_le16(src, 0)?); - let length2 = usize::from(read_le16(src, 2)?); - let length3 = usize::from(read_le16(src, 4)?); - let length4 = src.len().wrapping_sub(length1 + length2 + length3 + 6); - let start1 = 6; - let start2 = start1 + length1; - let start3 = start2 + length2; - let start4 = start3 + length3; - Ok(Self { - start: [start1, start2, start3, start4], - length: [length1, length2, length3, length4], - }) - } - - fn open<'a>(&self, src: &'a [u8]) -> Result<[BitDStream<'a>; 4]> { - let [s1, s2, s3, s4] = self.start; - let [l1, l2, l3, l4] = self.length; - Ok([ - BitDStream::new(src.span(s1, l1)?)?, - BitDStream::new(src.span(s2, l2)?)?, - BitDStream::new(src.span(s3, l3)?)?, - BitDStream::new(src.span(s4, l4)?)?, - ]) - } -} - -/// `HUF_DecompressFastArgs`, with positions in the input and the output for pointers. -struct FastArgs { - ip: [usize; 4], - op: [usize; 4], - bits: [u64; 4], - /// Where each stream starts. - iend: [usize; 4], -} - -/// `HUF_initFastDStream`: the lowest set bit marks how much of the 64 bits is consumed. -#[inline] -fn init_fast_dstream(src: &[u8], ip: usize) -> Result { - let value = read_le64(src, ip)?; - let last_byte = (value >> 56) as u32; - let bits_consumed = if last_byte != 0 { - 8 - highbit32(last_byte) - } else { - 0 - }; - Ok((value | 1) << bits_consumed) -} - -impl FastArgs { - /// `HUF_DecompressFastArgs_init`. `None` if the input has to be decoded by the other loops. - fn new(dst_size: usize, src: &[u8], table_log: u32) -> Result> { - if dst_size == 0 { - return Ok(None); - } - if src.len() < 10 { - return Err(ErrorCode::CorruptionDetected); - } - if table_log != HUF_DECODER_FAST_TABLELOG { - return Ok(None); - } - let streams = Streams::new(src)?; - if streams.length.iter().any(|&length| length < 8) { - return Ok(None); - } - let [_, _, _, length4] = streams.length; - if length4 > src.len() { - return Err(ErrorCode::CorruptionDetected); + log: 0, + table: [[0; 4]; TABLE_SIZE], } - let [_, start2, start3, start4] = streams.start; - let ip = [start2 - 8, start3 - 8, start4 - 8, src.len() - 8]; - let segment = dst_size.div_ceil(4); - let op = [0, segment, 2 * segment, 3 * segment]; - if 3 * segment >= dst_size { - return Ok(None); - } - let [ip0, ip1, ip2, ip3] = ip; - Ok(Some(Self { - ip, - op, - bits: [ - init_fast_dstream(src, ip0)?, - init_fast_dstream(src, ip1)?, - init_fast_dstream(src, ip2)?, - init_fast_dstream(src, ip3)?, - ], - iend: streams.start, - })) } - /// `HUF_initRemainingDStream` - fn remaining_dstream<'a>( - &self, - src: &'a [u8], - stream: usize, - segment_end: usize, - ) -> Result> { - if self.op.at(stream)? > segment_end { - return Err(ErrorCode::CorruptionDetected); - } - let ip = self.ip.at(stream)?; - if ip + 8 < self.iend.at(stream)? { - return Err(ErrorCode::CorruptionDetected); + /// Reads the description of a table at the start of `src`, and returns its size. + fn read(&mut self, src: &[u8], table_type: TableType) -> Result { + let stats = huf_read_stats(&mut self.code.weights, &mut self.code.rank_stats, src)?; + self.code.nb_symbols = stats.nb_symbols; + self.code.table_log = stats.table_log; + self.table_type = table_type; + self.build(stats.table_log.max(11))?; + Ok(stats.size) + } + + /// Fills `table` for lookups of `log` bits, which is at least as many as the longest code has. + fn build(&mut self, log: u32) -> Result<()> { + let Code { + weights, + rank_stats, + nb_symbols, + table_log, + } = &self.code; + let max_weight = *table_log as usize; + self.log = log; + + // The symbols by increasing weight, which is the order of their codes. `class_start[w]` is where those of + // weight `w` start, and `position[w]` where their entries start in the table. + let mut sorted = [0u8; NB_SYMBOLS]; + let mut class_start = [0usize; NB_RANKS + 1]; + let mut position = [0usize; NB_RANKS + 1]; + let mut nb_sorted = 0usize; + let mut filled = 0usize; + for weight in 1..=max_weight + 1 { + class_start.put(weight, nb_sorted)?; + position.put(weight, filled)?; + let count = rank_stats.get(weight).copied().unwrap_or(0) as usize; + nb_sorted += count; + filled += count << (weight as u32 - 1 + log - table_log); } - BitDStream::resume(src, ip, self.bits.at(stream)?.trailing_zeros()) - } - - /// How many rounds of at most 7 bytes the first stream has input for. The others start above it. - #[inline] - fn input_rounds(&self) -> usize { - let [ip0, ..] = self.ip; - ip0 / 7 - } - - #[inline] - fn streams_are_ordered(&self) -> bool { - self.ip.is_sorted() - } -} - -/// What both fast loops do to a stream after a round. -#[inline(always)] -fn reload_fast(src: &[u8], ip: &mut usize, bits: &mut u64) -> Result<()> { - let ctz = bits.trailing_zeros(); - let nb_bits = ctz & 7; - let nb_bytes = (ctz >> 3) as usize; - *ip = ip.wrapping_sub(nb_bytes); - *bits = (read_le64(src, *ip)? | 1) << nb_bits; - Ok(()) -} - -/// Three parts of `segment_size` bytes, and the rest. -fn split_in_4(dst: &mut [u8], segment_size: usize) -> Result<[&mut [u8]; 4]> { - let (part0, rest) = dst.split_at_mut_checked(segment_size).ok_or(BUG)?; - let (part1, rest) = rest.split_at_mut_checked(segment_size).ok_or(BUG)?; - let (part2, part3) = rest.split_at_mut_checked(segment_size).ok_or(BUG)?; - Ok([part0, part1, part2, part3]) -} - -/// The end of each of the four parts of an output of `dst_size` bytes. -fn segment_ends(dst_size: usize) -> [usize; 4] { - let segment = dst_size.div_ceil(4); - [1, 2, 3, 4].map(|n| (n * segment).min(dst_size)) -} - -// ── X1 ──────────────────────────────────────────────────────────────────── - -/// `HUF_rescaleStats` -fn rescale_stats( - huff_weight: &mut [u8], - rank_val: &mut [u32; NB_RANKS], - table_log: u32, - target_table_log: u32, -) -> Result { - if table_log > target_table_log { - return Ok(table_log); - } - if table_log < target_table_log { - let scale = (target_table_log - table_log) as usize; - for weight in huff_weight { - if *weight != 0 { - *weight += scale as u8; + let mut cursor = class_start; + for (symbol, &weight) in weights.iter().enumerate().take(*nb_symbols as usize) { + if weight != 0 { + let at = cursor.at_mut(usize::from(weight))?; + sorted.put(*at, symbol as u8)?; + *at += 1; } } - let target = target_table_log as usize; - rank_val.move_within(1, scale + 1, target - scale)?; - rank_val.span_mut(1, scale)?.fill(0); - } - Ok(target_table_log) -} - -/// `HUF_readDTableX1_wksp`. Returns the size of the table's description in `src`. -fn read_dtable_x1(dtable: &mut HufDTable, src: &[u8]) -> Result { - let mut rank_val = [0u32; NB_RANKS]; - let mut rank_start = [0u32; NB_RANKS]; - let mut symbols = [0u8; NB_SYMBOLS]; - let mut huff_weight = [0u8; NB_SYMBOLS]; - - let stats = huf_read_stats(&mut huff_weight, &mut rank_val, src)?; - let weights = huff_weight.sub_mut(..stats.nb_symbols as usize)?; - - // An X1 table takes half the room of an X2 table, so its log may be one more than the capacity. - let max_table_log = HUF_TABLELOG_MAX + 1; - let target_table_log = max_table_log.min(HUF_DECODER_FAST_TABLELOG); - let table_log = rescale_stats(weights, &mut rank_val, stats.table_log, target_table_log)?; - if table_log > max_table_log { - return Err(ErrorCode::TableLogTooLarge); - } - dtable.table_type = TableType::X1; - dtable.table_log = table_log; - - // Sort the symbols by weight. - let mut next_rank_start = 0u32; - for (start, &count) in rank_start - .iter_mut() - .zip(&rank_val) - .take(table_log as usize + 1) - { - *start = next_rank_start; - next_rank_start += count; - } - for (symbol, &weight) in weights.iter().enumerate() { - let start = rank_start.at_mut(usize::from(weight))?; - symbols.put(*start as usize, symbol as u8)?; - *start += 1; - } - - let [rank0, ..] = rank_val; - let mut symbol = rank0 as usize; - let mut u_start = 0usize; - for w in 1..=table_log { - let symbol_count = rank_val.at(w as usize)? as usize; - let length = (1usize << w) >> 1; - let nb_bits = (table_log + 1 - w) as u16; - for &byte in symbols.span(symbol, symbol_count)? { - dtable - .x1 - .span_mut(u_start, length)? - .fill((u16::from(byte) << 8) | nb_bits); - u_start += length; - } - symbol += symbol_count; - } - Ok(stats.size) -} - -/// `HUF_decodeSymbolX1` -#[inline(always)] -fn decode_symbol_x1(bit_d: &mut BitDStream<'_>, dt: &[u16; TABLE_SIZE], dt_log: u32) -> u8 { - let entry = entry_x1(dt, bit_d.look_bits_fast(dt_log)); - bit_d.skip_bits(u32::from(entry & 0xFF)); - (entry >> 8) as u8 -} - -/// `HUF_decodeStreamX1`: fills `out`. -#[inline] -fn decode_stream_x1( - out: &mut [u8], - bit_d: &mut BitDStream<'_>, - dt: &[u16; TABLE_SIZE], - dt_log: u32, -) { - let end = out.len(); - let mut p = 0usize; - if end > 3 { - while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 3 < end) { - if let Ok(four) = out.array_at_mut::<4>(p) { - for byte in four { - *byte = decode_symbol_x1(bit_d, dt, dt_log); + let class = + |weight: usize| sorted.sub(class_start.at(weight)?..class_start.at(weight + 1)?); + let nb_bits = |weight: usize| table_log + 1 - weight as u32; + // The lowest weight of a code of at most `room` bits. + let min_weight = |room: u32| (max_weight + 1).saturating_sub(room as usize).max(1); + + // What follows a symbol in an index is an index into a smaller table of the same code, where the codes that are + // too long for it come first. For each size, from `size` on: what a third symbol adds to an entry. + let mut thirds = [0u32; TABLE_SIZE / 2]; + let shortest = nb_bits(max_weight); + for room in shortest..=log.saturating_sub(2 * shortest) { + let thirds = thirds.span_mut(1 << room, 1 << room)?; + for weight in min_weight(room)..=max_weight { + let run = 1usize << (room - nb_bits(weight)); + let symbols = class(weight)?; + let thirds = + thirds.span_mut(position.at(weight)? >> (log - room), run * symbols.len())?; + for (thirds, &symbol) in thirds.chunks_exact_mut(run).zip(symbols) { + thirds.fill(entry(symbol, 2, nb_bits(weight))); } } - p += 4; } - } else { - bit_d.reload(); - } - for byte in out.get_mut(p..).unwrap_or_default() { - *byte = decode_symbol_x1(bit_d, dt, dt_log); - } -} -/// `HUF_decompress1X1_usingDTable_internal_body` -fn decompress_1x1_using_dtable(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { - let mut bit_d = BitDStream::new(src)?; - decode_stream_x1(dst, &mut bit_d, &dtable.x1, dtable.table_log); - if !bit_d.is_at_end() { - return Err(ErrorCode::CorruptionDetected); - } - Ok(dst.len()) -} - -/// `HUF_decompress4X1_usingDTable_internal_body` -fn decompress_4x1_body(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { - if src.len() < 10 { - return Err(ErrorCode::CorruptionDetected); - } - if dst.len() < 6 { - return Err(ErrorCode::CorruptionDetected); - } - let dst_size = dst.len(); - let dt = &dtable.x1; - let dt_log = dtable.table_log; - let streams = Streams::new(src)?; - let [_, _, _, length4] = streams.length; - if length4 > src.len() { - return Err(ErrorCode::CorruptionDetected); - } - let segment_size = dst_size.div_ceil(4); - if 3 * segment_size > dst_size { - return Err(ErrorCode::CorruptionDetected); - } - let [mut bit_d1, mut bit_d2, mut bit_d3, mut bit_d4] = streams.open(src)?; + for weight in 1..=max_weight { + let Some((&first_symbol, other_symbols)) = class(weight)?.split_first() else { + continue; + }; + let first_bits = nb_bits(weight); + let room = log - first_bits; + let size = 1usize << room; + let entries = self + .table + .span_mut(position.at(weight)?, size * (1 + other_symbols.len()))?; + let (entries, other_entries) = entries.split_at_mut_checked(size).ok_or(BUG)?; + + let first = entry(first_symbol, 0, first_bits); + entries + .sub_mut(..position.at(min_weight(room))? >> first_bits)? + .fill(first.to_le_bytes()); + for second_weight in min_weight(room)..=max_weight { + let room = room - nb_bits(second_weight); + let size = 1usize << room; + let symbols = class(second_weight)?; + let thirds = thirds.span(size, size)?; + let entries = entries.span_mut( + position.at(second_weight)? >> first_bits, + size * symbols.len(), + )?; + for (entries, &symbol) in entries.chunks_exact_mut(size).zip(symbols) { + let two = first + entry(symbol, 1, nb_bits(second_weight)); + for (entry, third) in entries.iter_mut().zip(thirds) { + *entry = (two + third).to_le_bytes(); + } + } + } - let [out1, out2, out3, out4] = split_in_4(dst, segment_size)?; - // The four streams advance together, by `done`, and the last part is the shortest. - let mut done = 0usize; - if out4.len() >= 8 { - let mut end_signal = true; - while end_signal & (done + 3 < out4.len()) { - let four1 = out1.array_at_mut::<4>(done)?; - let four2 = out2.array_at_mut::<4>(done)?; - let four3 = out3.array_at_mut::<4>(done)?; - let four4 = out4.array_at_mut::<4>(done)?; - for (((byte1, byte2), byte3), byte4) in - four1.iter_mut().zip(four2).zip(four3).zip(four4) - { - *byte1 = decode_symbol_x1(&mut bit_d1, dt, dt_log); - *byte2 = decode_symbol_x1(&mut bit_d2, dt, dt_log); - *byte3 = decode_symbol_x1(&mut bit_d3, dt, dt_log); - *byte4 = decode_symbol_x1(&mut bit_d4, dt, dt_log); + for (copy, &symbol) in other_entries.chunks_exact_mut(size).zip(other_symbols) { + for (to, from) in copy.iter_mut().zip(&*entries) { + let [_, second, third, meta] = *from; + *to = [symbol, second, third, meta]; + } } - done += 4; - end_signal &= bit_d1.reload_fast() == DStreamStatus::Unfinished; - end_signal &= bit_d2.reload_fast() == DStreamStatus::Unfinished; - end_signal &= bit_d3.reload_fast() == DStreamStatus::Unfinished; - end_signal &= bit_d4.reload_fast() == DStreamStatus::Unfinished; } + Ok(()) } - decode_stream_x1(out1.sub_mut(done..)?, &mut bit_d1, dt, dt_log); - decode_stream_x1(out2.sub_mut(done..)?, &mut bit_d2, dt, dt_log); - decode_stream_x1(out3.sub_mut(done..)?, &mut bit_d3, dt, dt_log); - decode_stream_x1(out4.sub_mut(done..)?, &mut bit_d4, dt, dt_log); - - if !(bit_d1.is_at_end() & bit_d2.is_at_end() & bit_d3.is_at_end() & bit_d4.is_at_end()) { - return Err(ErrorCode::CorruptionDetected); - } - Ok(dst_size) -} - -/// `HUF_decompress4X1_usingDTable_internal_fast_c_loop` -fn fast_loop_4x1( - args: &mut FastArgs, - dst: &mut [u8], - src: &[u8], - dt: &[u16; TABLE_SIZE], -) -> Result<()> { - let [_, segment_size, ..] = args.op; - let [part0, part1, part2, part3] = split_in_4(dst, segment_size)?; - // How much of each part is written. It is the same for all of them. - let mut done = 0usize; - - let [mut ip0, mut ip1, mut ip2, mut ip3] = args.ip; - let [mut bits0, mut bits1, mut bits2, mut bits3] = args.bits; - #[inline(always)] - fn decode(bits: &mut u64, dt: &[u16; TABLE_SIZE]) -> u8 { - let entry = entry_x1(dt, *bits >> 53); - *bits <<= entry & 0x3F; - (entry >> 8) as u8 - } - - loop { - // Each round writes 5 bytes to each part, of which the last is the shortest, and reads at most 7 of each stream. - let rounds = ((part3.len() - done) / 5).min(ip0 / 7); - if rounds == 0 { - break; - } - if !(ip0 <= ip1 && ip1 <= ip2 && ip2 <= ip3) { - break; - } - let (out0, _) = part0.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); - let (out1, _) = part1.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); - let (out2, _) = part2.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); - let (out3, _) = part3.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); - for (((out0, out1), out2), out3) in out0.iter_mut().zip(out1).zip(out2).zip(out3) { - for (((byte0, byte1), byte2), byte3) in out0.iter_mut().zip(out1).zip(out2).zip(out3) { - *byte0 = decode(&mut bits0, dt); - *byte1 = decode(&mut bits1, dt); - *byte2 = decode(&mut bits2, dt); - *byte3 = decode(&mut bits3, dt); + fn entry(&self, index: u64) -> Entry { + self.table + .get(index as usize & (TABLE_SIZE - 1)) + .copied() + .unwrap_or_default() + } + + /// Fills `out` from a stream. Whether that takes exactly what is left of the stream. + fn decode_stream(&self, out: &mut [u8], bit_d: &mut BitDStream<'_>) -> bool { + let log = self.log; + let mut rest = &mut *out; + // Four lookups take at most 48 bits. + while rest.len() >= 16 && bit_d.reload() == DStreamStatus::Unfinished { + let mut done = 0usize; + for _ in 0..4 { + let entry = self.entry(bit_d.look_bits_fast(log)); + let [.., meta] = entry; + bit_d.skip_bits(u32::from(meta & 63)); + if let Some(to) = rest.get_mut(done..done + 4) { + to.copy_from_slice(&entry); + } + done += usize::from(meta >> COUNT_SHIFT); } - reload_fast(src, &mut ip0, &mut bits0)?; - reload_fast(src, &mut ip1, &mut bits1)?; - reload_fast(src, &mut ip2, &mut bits2)?; - reload_fast(src, &mut ip3, &mut bits3)?; + rest = rest.get_mut(done..).unwrap_or_default(); } - done += rounds * 5; - } - - args.ip = [ip0, ip1, ip2, ip3]; - args.bits = [bits0, bits1, bits2, bits3]; - args.op = args.op.map(|start| start + done); - Ok(()) -} - -/// `HUF_decompress4X1_usingDTable_internal_fast`. `None` if the input has to be decoded by the other loop. -fn decompress_4x1_fast(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result> { - let Some(mut args) = FastArgs::new(dst.len(), src, dtable.table_log)? else { - return Ok(None); - }; - fast_loop_4x1(&mut args, dst, src, &dtable.x1)?; - - for (stream, segment_end) in segment_ends(dst.len()).into_iter().enumerate() { - let mut bit_d = args.remaining_dstream(src, stream, segment_end)?; - let out = dst.sub_mut(args.op.at(stream)?..segment_end)?; - decode_stream_x1(out, &mut bit_d, &dtable.x1, HUF_DECODER_FAST_TABLELOG); - } - Ok(Some(dst.len())) -} - -/// `HUF_decompress4X1_usingDTable_internal` -fn decompress_4x1_using_dtable(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { - if let Some(size) = decompress_4x1_fast(dst, src, dtable)? { - return Ok(size); - } - decompress_4x1_body(dst, src, dtable) -} - -// ── X2 ──────────────────────────────────────────────────────────────────── - -type RankValCol = [u32; NB_RANKS]; - -/// `HUF_buildDEltX2U32` -#[inline(always)] -fn build_delt_x2(symbol: u32, nb_bits: u32, base_seq: u32, level: u32) -> u32 { - let seq = if level == 1 { - symbol - } else { - base_seq + (symbol << 8) - }; - seq + (nb_bits << 16) + (level << 24) -} - -/// `HUF_fillDTableX2ForWeight` -fn fill_dtable_x2_for_weight( - dtable_rank: &mut [u32], - symbols: &[u8], - nb_bits: u32, - table_log: u32, - base_seq: u32, - level: u32, -) -> Result<()> { - let length = 1usize << (table_log.wrapping_sub(nb_bits) & 0x1F); - let entries = dtable_rank.sub_mut(..length * symbols.len())?; - for (entries, &symbol) in entries.chunks_exact_mut(length).zip(symbols) { - entries.fill(build_delt_x2(u32::from(symbol), nb_bits, base_seq, level)); - } - Ok(()) -} - -/// `HUF_fillDTableX2Level2` -fn fill_dtable_x2_level2( - dtable: &mut [u32], - target_log: u32, - consumed_bits: u32, - rank_val: &RankValCol, - min_weight: usize, - max_weight1: usize, - sorted_symbols: &[u8; NB_SYMBOLS], - rank_start: &[u32; NB_RANKS + 2], - nb_bits_baseline: u32, - base_seq: u32, -) -> Result<()> { - // The codes that are too long for a second symbol to fit hold the first symbol alone. - if min_weight > 1 { - let length = 1usize << (target_log.wrapping_sub(consumed_bits) & 0x1F); - let skip_size = rank_val.at(min_weight)? as usize; - let filled = match length { - 2 | 4 => length, - _ => skip_size.next_multiple_of(8), - }; - let first_symbol = base_seq; - dtable - .sub_mut(..filled)? - .fill(build_delt_x2(first_symbol, consumed_bits, 0, 1)); - } - - for w in min_weight..max_weight1 { - let begin = rank_start.at(w)? as usize; - let end = rank_start.at(w + 1)? as usize; - let nb_bits = nb_bits_baseline - w as u32; - let total_bits = nb_bits + consumed_bits; - fill_dtable_x2_for_weight( - dtable.sub_mut(rank_val.at(w)? as usize..)?, - sorted_symbols.sub(begin..end)?, - total_bits, - target_log, - base_seq, - 2, - )?; - } - Ok(()) -} - -/// `HUF_fillDTableX2` -fn fill_dtable_x2( - dtable: &mut [u32; TABLE_SIZE], - target_log: u32, - sorted_list: &[u8; NB_SYMBOLS], - rank_start: &[u32; NB_RANKS + 2], - rank_val_origin: &[RankValCol; HUF_TABLELOG_MAX as usize], - max_weight: u32, - nb_bits_baseline: u32, -) -> Result<()> { - let [rank_val, ..] = rank_val_origin; - let scale_log = nb_bits_baseline as i32 - target_log as i32; - let min_bits = nb_bits_baseline - max_weight; - let w_end = max_weight as usize + 1; - - for w in 1..w_end { - let begin = rank_start.at(w)? as usize; - let end = rank_start.at(w + 1)? as usize; - let nb_bits = nb_bits_baseline - w as u32; - - if target_log.wrapping_sub(nb_bits) >= min_bits { - // Enough room for a second symbol. - let mut start = rank_val.at(w)? as usize; - let length = 1usize << (target_log.wrapping_sub(nb_bits) & 0x1F); - let min_weight = (nb_bits as i32 + scale_log).max(1) as usize; - for &symbol in sorted_list.sub(begin..end)? { - fill_dtable_x2_level2( - dtable.sub_mut(start..)?, - target_log, - nb_bits, - rank_val_origin.get(nb_bits as usize).ok_or(BUG)?, - min_weight, - w_end, - sorted_list, - rank_start, - nb_bits_baseline, - u32::from(symbol), - )?; - start += length; - } - } else { - fill_dtable_x2_for_weight( - dtable.sub_mut(rank_val.at(w)? as usize..)?, - sorted_list.sub(begin..end)?, - nb_bits, - target_log, - 0, - 1, - )?; + for byte in rest { + bit_d.reload(); + let [symbol, ..] = self.entry(bit_d.look_bits_fast(log)); + let weight = self + .code + .weights + .get(usize::from(symbol)) + .copied() + .unwrap_or(0); + bit_d.skip_bits(self.code.table_log + 1 - u32::from(weight)); + *byte = symbol; } + bit_d.is_at_end() } - Ok(()) -} - -/// `HUF_readDTableX2_wksp`. Returns the size of the table's description in `src`. -pub(crate) fn read_dtable_x2(dtable: &mut HufDTable, src: &[u8]) -> Result { - let mut rank_val = [[0u32; NB_RANKS]; HUF_TABLELOG_MAX as usize]; - let mut rank_stats = [0u32; NB_RANKS]; - // `rank_start0[w + 1]` is where the symbols of weight `w` start, until sorting moves it to where they end. - let mut rank_start0 = [0u32; NB_RANKS + 2]; - let mut sorted_symbol = [0u8; NB_SYMBOLS]; - let mut weight_list = [0u8; NB_SYMBOLS]; - let mut max_table_log = HUF_TABLELOG_MAX; - - let stats = huf_read_stats(&mut weight_list, &mut rank_stats, src)?; - let table_log = stats.table_log; - - if table_log > max_table_log { - return Err(ErrorCode::TableLogTooLarge); - } - if table_log <= HUF_DECODER_FAST_TABLELOG && max_table_log > HUF_DECODER_FAST_TABLELOG { - max_table_log = HUF_DECODER_FAST_TABLELOG; - } - - let mut max_w = table_log as usize; - while rank_stats.at(max_w)? == 0 { - max_w = max_w.checked_sub(1).ok_or(BUG)?; - } - - let mut next_rank_start = 0u32; - for w in 1..=max_w { - rank_start0.put(w + 1, next_rank_start)?; - next_rank_start += rank_stats.at(w)?; - } - // The symbols of weight 0 go to the end. - rank_start0.put(1, next_rank_start)?; - rank_start0.put(max_w + 2, next_rank_start)?; - for (symbol, &weight) in weight_list - .iter() - .enumerate() - .take(stats.nb_symbols as usize) - { - let start = rank_start0.at_mut(usize::from(weight) + 1)?; - sorted_symbol.put(*start as usize, symbol as u8)?; - *start += 1; + /// One stream. Whether it is exactly `dst.len()` symbols. + fn decode_1x(&self, dst: &mut [u8], src: &[u8]) -> bool { + match BitDStream::new(src) { + Ok(mut bit_d) => self.decode_stream(dst, &mut bit_d), + Err(_) => false, + } } - rank_start0.put(1, 0)?; - { - let rescale = (max_table_log - table_log) as i32 - 1; - let mut next_rank_val = 0u32; - let [rank_val0, ..] = &mut rank_val; - for w in 1..=max_w { - rank_val0.put(w, next_rank_val)?; - next_rank_val += rank_stats.at(w)? << (w as i32 + rescale); + /// Four streams after their sizes. Whether each is exactly its share of `dst.len()` symbols. + fn decode_4x(&self, dst: &mut [u8], src: &[u8]) -> bool { + let Some(mut streams) = Streams::new(dst, src) else { + return false; + }; + match self.log { + 11 => streams.decode_most::<11, 5>(&self.table), + 12 => streams.decode_most::<12, 4>(&self.table), + _ => {} } - let rank_val0 = *rank_val0; - let min_bits = table_log + 1 - max_w as u32; - for consumed in min_bits..(max_table_log - min_bits + 1) { - let rank_val_col = rank_val.get_mut(consumed as usize).ok_or(BUG)?; - for (shifted, &val) in rank_val_col.iter_mut().zip(&rank_val0).skip(1).take(max_w) { - *shifted = val >> consumed; - } + let mut exact = true; + for stream in streams.0 { + let Ok(mut bit_d) = + BitDStream::resume(stream.src, stream.ip as usize, stream.bits.trailing_zeros()) + else { + return false; + }; + let out = stream.out.get_mut(stream.op as usize..).unwrap_or_default(); + exact &= self.decode_stream(out, &mut bit_d); } + exact } - - fill_dtable_x2( - &mut dtable.x2, - max_table_log, - &sorted_symbol, - &rank_start0, - &rank_val, - max_w as u32, - table_log + 1, - )?; - - dtable.table_log = max_table_log; - dtable.table_type = TableType::X2; - Ok(stats.size) } -/// `HUF_decodeSymbolX2`: writes two bytes at `pos` and returns how many of them count. -#[inline(always)] -fn decode_symbol_x2( - out: &mut [u8], - pos: usize, - bit_d: &mut BitDStream<'_>, - dt: &[u32; TABLE_SIZE], - dt_log: u32, -) -> Result { - let entry = entry_x2(dt, bit_d.look_bits_fast(dt_log)); - *out.array_at_mut(pos)? = (entry as u16).to_le_bytes(); - bit_d.skip_bits((entry >> 16) & 0xFF); - Ok((entry >> 24) as usize) +/// A stream of at least 8 bytes, and where its symbols go. +struct Stream<'a> { + src: &'a [u8], + /// Where in `src` `bits` was read. + ip: u32, + /// What is left to read of 64 bits, from the highest bit on, then a bit set. + bits: u64, + out: &'a mut [u8], + /// How much of `out` is written. + op: u32, } -/// `HUF_decodeLastSymbolX2` -#[inline(always)] -fn decode_last_symbol_x2( - out: &mut [u8], - pos: usize, - bit_d: &mut BitDStream<'_>, - dt: &[u32; TABLE_SIZE], - dt_log: u32, -) -> Result<()> { - let entry = entry_x2(dt, bit_d.look_bits_fast(dt_log)); - out.put(pos, entry as u8)?; - let nb_bits = (entry >> 16) & 0xFF; - if entry >> 24 == 1 { - bit_d.skip_bits(nb_bits); - } else if bit_d.bits_consumed < u64::BITS { - bit_d.skip_bits(nb_bits); - // The code of the second symbol, which is not wanted, may be cut off. - if bit_d.bits_consumed > u64::BITS { - bit_d.bits_consumed = u64::BITS; +impl<'a> Stream<'a> { + fn new(src: &'a [u8], out: &'a mut [u8]) -> Option { + let container = u64::from_le_bytes(*src.last_chunk()?); + // The highest bit set marks the end of the stream. + if container >> 56 == 0 { + return None; } + Some(Self { + src, + ip: src.len() as u32 - 8, + bits: (container | 1) << (container.leading_zeros() + 1), + out, + op: 0, + }) } - Ok(()) } -/// `HUF_decodeStreamX2`: fills `out` and returns its length. -#[inline] -fn decode_stream_x2( - out: &mut [u8], - bit_d: &mut BitDStream<'_>, - dt: &[u32; TABLE_SIZE], - dt_log: u32, -) -> Result { - let end = out.len(); - let mut p = 0usize; - - if end >= 8 { - if dt_log <= 11 { - while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 9 < end) { - for _ in 0..5 { - p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; - } - } - } else { - while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 7 < end) { - for _ in 0..4 { - p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; +struct Streams<'a>([Stream<'a>; 4]); + +impl<'a> Streams<'a> { + /// `None` if there are no four streams of 8 bytes or more. + fn new(dst: &'a mut [u8], src: &'a [u8]) -> Option { + let (sizes, src) = src.split_first_chunk::<6>()?; + let (src0, src) = src.split_at_checked(usize::from(read_le16(sizes, 0).ok()?))?; + let (src1, src) = src.split_at_checked(usize::from(read_le16(sizes, 2).ok()?))?; + let (src2, src3) = src.split_at_checked(usize::from(read_le16(sizes, 4).ok()?))?; + let segment = dst.len().div_ceil(4); + let (dst0, dst) = dst.split_at_mut_checked(segment)?; + let (dst1, dst) = dst.split_at_mut_checked(segment)?; + let (dst2, dst3) = dst.split_at_mut_checked(segment)?; + Some(Self([ + Stream::new(src0, dst0)?, + Stream::new(src1, dst1)?, + Stream::new(src2, dst2)?, + Stream::new(src3, dst3)?, + ])) + } + + /// Decodes for as long as no stream is close to its start and no output to its end. `LOOKUPS` of `LOG` bits are at + /// most 57 bits. + fn decode_most(&mut self, table: &[Entry; TABLE_SIZE]) { + /// What `LOOKUPS` lookups can write, as far as the compiler can tell. + const WINDOW: usize = 32; + + #[inline(always)] + fn lookup( + table: &[Entry; TABLE_SIZE], + bits: &mut u64, + window: &mut [u8; WINDOW], + done: &mut usize, + ) { + if let Some(entry) = table.get((*bits >> (64 - LOG)) as usize) { + let [.., meta] = *entry; + *bits <<= meta & 63; + if let Some(to) = window.get_mut(*done..*done + 4) { + to.copy_from_slice(entry); } + *done += usize::from(meta >> COUNT_SHIFT); } } - } else { - bit_d.reload(); - } - if end - p >= 2 { - while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 2 <= end) { - p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; - } - while p + 2 <= end { - p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + #[inline(always)] + fn window(out: &mut [u8], op: u32) -> Option<&mut [u8; WINDOW]> { + out.get_mut(op as usize..op as usize + WINDOW)? + .as_mut_array() } - } - - if p < end { - decode_last_symbol_x2(out, p, bit_d, dt, dt_log)?; - p += 1; - } - Ok(p) -} - -/// `HUF_decompress1X2_usingDTable_internal_body` -fn decompress_1x2_using_dtable(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { - let mut bit_d = BitDStream::new(src)?; - decode_stream_x2(dst, &mut bit_d, &dtable.x2, dtable.table_log)?; - if !bit_d.is_at_end() { - return Err(ErrorCode::CorruptionDetected); - } - Ok(dst.len()) -} - -/// `HUF_decompress4X2_usingDTable_internal_body` -fn decompress_4x2_body(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { - if src.len() < 10 { - return Err(ErrorCode::CorruptionDetected); - } - if dst.len() < 6 { - return Err(ErrorCode::CorruptionDetected); - } - let oend = dst.len(); - let dt = &dtable.x2; - let dt_log = dtable.table_log; - let streams = Streams::new(src)?; - let [_, _, _, length4] = streams.length; - if length4 > src.len() { - return Err(ErrorCode::CorruptionDetected); - } - let segment_size = oend.div_ceil(4); - let op_start2 = segment_size; - let op_start3 = 2 * segment_size; - let op_start4 = 3 * segment_size; - if op_start4 > oend { - return Err(ErrorCode::CorruptionDetected); - } - let [mut bit_d1, mut bit_d2, mut bit_d3, mut bit_d4] = streams.open(src)?; - let (mut op1, mut op2, mut op3, mut op4) = (0, op_start2, op_start3, op_start4); - // A damaged stream can run over the start of the next part, so the parts are not split here. - if oend - op4 >= 8 { - let mut end_signal = true; - while end_signal & (op4 + 7 < oend) { - for _ in 0..4 { - op1 += decode_symbol_x2(dst, op1, &mut bit_d1, dt, dt_log)?; - } - for _ in 0..4 { - op2 += decode_symbol_x2(dst, op2, &mut bit_d2, dt, dt_log)?; - } - end_signal &= bit_d1.reload_fast() == DStreamStatus::Unfinished; - end_signal &= bit_d2.reload_fast() == DStreamStatus::Unfinished; - for _ in 0..4 { - op3 += decode_symbol_x2(dst, op3, &mut bit_d3, dt, dt_log)?; - } - for _ in 0..4 { - op4 += decode_symbol_x2(dst, op4, &mut bit_d4, dt, dt_log)?; - } - end_signal &= bit_d3.reload_fast() == DStreamStatus::Unfinished; - end_signal &= bit_d4.reload_fast() == DStreamStatus::Unfinished; + /// Whether there are 8 bytes to read from where the stream is. + #[inline(always)] + fn reload(src: &[u8], ip: &mut u32, bits: &mut u64) -> bool { + let consumed = bits.trailing_zeros(); + // Wraps around before the start. + let from = ip.wrapping_sub(consumed >> 3); + let Some(Ok(container)) = src + .get(from as usize..from as usize + 8) + .map(<[u8; 8]>::try_from) + else { + return false; + }; + *ip = from; + *bits = (u64::from_le_bytes(container) | 1) << (consumed & 7); + true } - } - - if op1 > op_start2 || op2 > op_start3 || op3 > op_start4 { - return Err(ErrorCode::CorruptionDetected); - } - - decode_stream_x2(dst.sub_mut(op1..op_start2)?, &mut bit_d1, dt, dt_log)?; - decode_stream_x2(dst.sub_mut(op2..op_start3)?, &mut bit_d2, dt, dt_log)?; - decode_stream_x2(dst.sub_mut(op3..op_start4)?, &mut bit_d3, dt, dt_log)?; - decode_stream_x2(dst.sub_mut(op4..oend)?, &mut bit_d4, dt, dt_log)?; - - if !(bit_d1.is_at_end() & bit_d2.is_at_end() & bit_d3.is_at_end() & bit_d4.is_at_end()) { - return Err(ErrorCode::CorruptionDetected); - } - Ok(oend) -} - -/// `HUF_decompress4X2_usingDTable_internal_fast_c_loop` -fn fast_loop_4x2( - args: &mut FastArgs, - dst: &mut [u8], - src: &[u8], - dt: &[u32; TABLE_SIZE], -) -> Result<()> { - let [_, end0, end1, end2] = args.op; - let oend = [end0, end1, end2, dst.len()]; - #[inline(always)] - fn decode( - dst: &mut [u8], - op: &mut usize, - bits: &mut u64, - dt: &[u32; TABLE_SIZE], - ) -> Result<()> { - let entry = entry_x2(dt, *bits >> 53); - *dst.array_at_mut(*op)? = (entry as u16).to_le_bytes(); - *bits <<= (entry >> 16) & 0x3F; - *op += (entry >> 24) as usize; - Ok(()) - } - - loop { - // Each round writes at most 10 bytes to each part and reads at most 7 of each stream. - let mut rounds = args.input_rounds(); - for (&end, &op) in oend.iter().zip(&args.op) { - rounds = rounds.min((end - op) / 10); - } - if rounds == 0 { - break; - } - if !args.streams_are_ordered() { - break; - } - let [mut op0, mut op1, mut op2, mut op3] = args.op; - let [mut ip0, mut ip1, mut ip2, mut ip3] = args.ip; - let [mut bits0, mut bits1, mut bits2, mut bits3] = args.bits; - let olimit = op3 + rounds * 5; + let [s0, s1, s2, s3] = &mut self.0; + let (mut bits0, mut bits1, mut bits2, mut bits3) = (s0.bits, s1.bits, s2.bits, s3.bits); + let (mut ip0, mut ip1, mut ip2, mut ip3) = (s0.ip, s1.ip, s2.ip, s3.ip); + let (mut op0, mut op1, mut op2, mut op3) = (s0.op, s1.op, s2.op, s3.op); loop { - for _ in 0..5 { - decode(dst, &mut op0, &mut bits0, dt)?; - decode(dst, &mut op1, &mut bits1, dt)?; - decode(dst, &mut op2, &mut bits2, dt)?; - decode(dst, &mut op3, &mut bits3, dt)?; + let (Some(window0), Some(window1), Some(window2), Some(window3)) = ( + window(s0.out, op0), + window(s1.out, op1), + window(s2.out, op2), + window(s3.out, op3), + ) else { + break; + }; + let (mut done0, mut done1, mut done2, mut done3) = (0, 0, 0, 0); + for _ in 0..LOOKUPS { + lookup::(table, &mut bits0, window0, &mut done0); + lookup::(table, &mut bits1, window1, &mut done1); + lookup::(table, &mut bits2, window2, &mut done2); + lookup::(table, &mut bits3, window3, &mut done3); } - reload_fast(src, &mut ip0, &mut bits0)?; - reload_fast(src, &mut ip1, &mut bits1)?; - reload_fast(src, &mut ip2, &mut bits2)?; - reload_fast(src, &mut ip3, &mut bits3)?; - if op3 >= olimit { + let mut more = reload(s0.src, &mut ip0, &mut bits0) + && reload(s1.src, &mut ip1, &mut bits1) + && reload(s2.src, &mut ip2, &mut bits2) + && reload(s3.src, &mut ip3, &mut bits3); + if more { + for _ in 0..LOOKUPS { + lookup::(table, &mut bits0, window0, &mut done0); + lookup::(table, &mut bits1, window1, &mut done1); + lookup::(table, &mut bits2, window2, &mut done2); + lookup::(table, &mut bits3, window3, &mut done3); + } + more = reload(s0.src, &mut ip0, &mut bits0) + && reload(s1.src, &mut ip1, &mut bits1) + && reload(s2.src, &mut ip2, &mut bits2) + && reload(s3.src, &mut ip3, &mut bits3); + } + op0 += done0 as u32; + op1 += done1 as u32; + op2 += done2 as u32; + op3 += done3 as u32; + if !more { break; } } - args.op = [op0, op1, op2, op3]; - args.ip = [ip0, ip1, ip2, ip3]; - args.bits = [bits0, bits1, bits2, bits3]; - } - Ok(()) -} - -/// `HUF_decompress4X2_usingDTable_internal_fast`. `None` if the input has to be decoded by the other loop. -fn decompress_4x2_fast(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result> { - let Some(mut args) = FastArgs::new(dst.len(), src, dtable.table_log)? else { - return Ok(None); - }; - fast_loop_4x2(&mut args, dst, src, &dtable.x2)?; - - for (stream, segment_end) in segment_ends(dst.len()).into_iter().enumerate() { - let mut bit_d = args.remaining_dstream(src, stream, segment_end)?; - let out = dst.sub_mut(args.op.at(stream)?..segment_end)?; - decode_stream_x2(out, &mut bit_d, &dtable.x2, HUF_DECODER_FAST_TABLELOG)?; + (s0.bits, s1.bits, s2.bits, s3.bits) = (bits0, bits1, bits2, bits3); + (s0.ip, s1.ip, s2.ip, s3.ip) = (ip0, ip1, ip2, ip3); + (s0.op, s1.op, s2.op, s3.op) = (op0, op1, op2, op3); } - Ok(Some(dst.len())) } -/// `HUF_decompress4X2_usingDTable_internal` -fn decompress_4x2_using_dtable(dst: &mut [u8], src: &[u8], dtable: &HufDTable) -> Result { - if let Some(size) = decompress_4x2_fast(dst, src, dtable)? { - return Ok(size); - } - decompress_4x2_body(dst, src, dtable) -} - -// ── choosing between X1 and X2 ──────────────────────────────────────────── - -/// `algoTime`: for each sixteenth of compression ratio, the time to build a table and to decode 256 bytes, for X1 and X2. -#[rustfmt::skip] -const ALGO_TIME: [[(u32, u32); 2]; 16] = [ - [( 0, 0), ( 1, 1)], - [( 0, 0), ( 1, 1)], - [( 150, 216), ( 381, 119)], - [( 170, 205), ( 514, 112)], - [( 177, 199), ( 539, 110)], - [( 197, 194), ( 644, 107)], - [( 221, 192), ( 735, 107)], - [( 256, 189), ( 881, 106)], - [( 359, 188), (1167, 109)], - [( 582, 187), (1570, 114)], - [( 688, 187), (1712, 122)], - [( 825, 186), (1965, 136)], - [( 976, 185), (2131, 150)], - [(1180, 186), (2070, 175)], - [(1377, 185), (1731, 202)], - [(1412, 185), (1695, 202)], -]; - -/// `HUF_selectDecoder`: whether X2 is expected to be the faster one. `dst_size` is not 0. -fn select_decoder_x2(dst_size: usize, src_size: usize) -> Result { - let q = if src_size >= dst_size { - 15 - } else { - src_size * 16 / dst_size - }; - let d256 = (dst_size >> 8) as u32; - let [(table0, decode0), (table1, decode1)] = ALGO_TIME.at(q)?; - let dtime0 = table0 + decode0 * d256; - let mut dtime1 = table1 + decode1 * d256; - // A small advantage to the algorithm that uses less memory. - dtime1 += dtime1 >> 5; - Ok(dtime1 < dtime0) +/// `HUF_readDTableX2_wksp`. Returns the size of the table's description in `src`. +pub(crate) fn read_dtable_x2(dtable: &mut HufDTable, src: &[u8]) -> Result { + dtable.read(src, TableType::X2) } /// `HUF_decompress1X_usingDTable` @@ -967,19 +401,19 @@ pub(crate) fn decompress_1x_using_dtable( src: &[u8], dtable: &HufDTable, ) -> Result { - match dtable.table_type { - TableType::X1 => decompress_1x1_using_dtable(dst, src, dtable), - TableType::X2 => decompress_1x2_using_dtable(dst, src, dtable), + if dtable.decode_1x(dst, src) { + return Ok(dst.len()); } + huf_exact::decompress_1x(dst, src, &dtable.code, dtable.table_type) } /// `HUF_decompress1X1_DCtx_wksp` pub(crate) fn decompress_1x1(dtable: &mut HufDTable, dst: &mut [u8], src: &[u8]) -> Result { - let h_size = read_dtable_x1(dtable, src)?; + let h_size = dtable.read(src, TableType::X1)?; if h_size >= src.len() { return Err(ErrorCode::SrcSizeWrong); } - decompress_1x1_using_dtable(dst, src.sub(h_size..)?, dtable) + decompress_1x_using_dtable(dst, src.sub(h_size..)?, dtable) } /// `HUF_decompress4X_usingDTable` @@ -988,10 +422,10 @@ pub(crate) fn decompress_4x_using_dtable( src: &[u8], dtable: &HufDTable, ) -> Result { - match dtable.table_type { - TableType::X1 => decompress_4x1_using_dtable(dst, src, dtable), - TableType::X2 => decompress_4x2_using_dtable(dst, src, dtable), + if dtable.decode_4x(dst, src) { + return Ok(dst.len()); } + huf_exact::decompress_4x(dst, src, &dtable.code, dtable.table_type) } /// `HUF_decompress4X_hufOnly_wksp` @@ -1006,19 +440,14 @@ pub(crate) fn decompress_4x_huf_only( if src.is_empty() { return Err(ErrorCode::CorruptionDetected); } - let x2 = select_decoder_x2(dst.len(), src.len())?; - let h_size = if x2 { - read_dtable_x2(dtable, src)? + let table_type = if select_decoder_x2(dst.len(), src.len())? { + TableType::X2 } else { - read_dtable_x1(dtable, src)? + TableType::X1 }; + let h_size = dtable.read(src, table_type)?; if h_size >= src.len() { return Err(ErrorCode::SrcSizeWrong); } - let body = src.sub(h_size..)?; - if x2 { - decompress_4x2_using_dtable(dst, body, dtable) - } else { - decompress_4x1_using_dtable(dst, body, dtable) - } + decompress_4x_using_dtable(dst, src.sub(h_size..)?, dtable) } diff --git a/src/zstd/decompress/huf_exact.rs b/src/zstd/decompress/huf_exact.rs new file mode 100644 index 000000000000..b5772355e3f4 --- /dev/null +++ b/src/zstd/decompress/huf_exact.rs @@ -0,0 +1,1001 @@ +//! `huf_decompress.c`: Huffman decoding of literals, one symbol (X1) or up to two symbols (X2) per table lookup, step by +//! step as the C code does it. What it makes of streams that do not end where they should depends on all of that. + +use super::huf::Code; +use crate::common::bitstream::{BitDStream, DStreamStatus}; +use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; +use crate::error::{ErrorCode, Result}; +use crate::util::{BUG, SliceExt, highbit32, read_le16, read_le64}; + +/// Tables of this log are decoded by the loops that keep four streams in registers. +const HUF_DECODER_FAST_TABLELOG: u32 = 11; + +const TABLE_SIZE: usize = 1 << HUF_TABLELOG_MAX; +const TABLE_MASK: usize = TABLE_SIZE - 1; +const NB_SYMBOLS: usize = HUF_SYMBOLVALUE_MAX as usize + 1; +const NB_RANKS: usize = HUF_TABLELOG_MAX as usize + 1; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum TableType { + X1, + X2, +} + +/// `HUF_DTable`, of capacity `ZSTD_HUFFDTABLE_CAPACITY_LOG`. +struct ExactTable { + table_type: TableType, + table_log: u32, + /// `HUF_DEltX1`: the number of bits in the low byte, and the symbol above it. + x1: [u16; TABLE_SIZE], + /// `HUF_DEltX2`: two symbols, then the number of bits, then how many of the symbols count. + x2: [u32; TABLE_SIZE], +} + +impl ExactTable { + const fn new() -> Self { + Self { + table_type: TableType::X1, + table_log: 0, + x1: [0; TABLE_SIZE], + x2: [0; TABLE_SIZE], + } + } +} + +#[inline(always)] +fn entry_x1(dt: &[u16; TABLE_SIZE], index: u64) -> u16 { + dt.get(index as usize & TABLE_MASK).copied().unwrap_or(0) +} + +#[inline(always)] +fn entry_x2(dt: &[u32; TABLE_SIZE], index: u64) -> u32 { + dt.get(index as usize & TABLE_MASK).copied().unwrap_or(0) +} + +// ── the four streams ────────────────────────────────────────────────────── + +/// Where the four streams of `src` start, after the jump table, and where the last one ends. +struct Streams { + start: [usize; 4], + length: [usize; 4], +} + +impl Streams { + /// `src` has at least 10 bytes. + fn new(src: &[u8]) -> Result { + let length1 = usize::from(read_le16(src, 0)?); + let length2 = usize::from(read_le16(src, 2)?); + let length3 = usize::from(read_le16(src, 4)?); + let length4 = src.len().wrapping_sub(length1 + length2 + length3 + 6); + let start1 = 6; + let start2 = start1 + length1; + let start3 = start2 + length2; + let start4 = start3 + length3; + Ok(Self { + start: [start1, start2, start3, start4], + length: [length1, length2, length3, length4], + }) + } + + fn open<'a>(&self, src: &'a [u8]) -> Result<[BitDStream<'a>; 4]> { + let [s1, s2, s3, s4] = self.start; + let [l1, l2, l3, l4] = self.length; + Ok([ + BitDStream::new(src.span(s1, l1)?)?, + BitDStream::new(src.span(s2, l2)?)?, + BitDStream::new(src.span(s3, l3)?)?, + BitDStream::new(src.span(s4, l4)?)?, + ]) + } +} + +/// `HUF_DecompressFastArgs`, with positions in the input and the output for pointers. +struct FastArgs { + ip: [usize; 4], + op: [usize; 4], + bits: [u64; 4], + /// Where each stream starts. + iend: [usize; 4], +} + +/// `HUF_initFastDStream`: the lowest set bit marks how much of the 64 bits is consumed. +#[inline] +fn init_fast_dstream(src: &[u8], ip: usize) -> Result { + let value = read_le64(src, ip)?; + let last_byte = (value >> 56) as u32; + let bits_consumed = if last_byte != 0 { + 8 - highbit32(last_byte) + } else { + 0 + }; + Ok((value | 1) << bits_consumed) +} + +impl FastArgs { + /// `HUF_DecompressFastArgs_init`. `None` if the input has to be decoded by the other loops. + fn new(dst_size: usize, src: &[u8], table_log: u32) -> Result> { + if dst_size == 0 { + return Ok(None); + } + if src.len() < 10 { + return Err(ErrorCode::CorruptionDetected); + } + if table_log != HUF_DECODER_FAST_TABLELOG { + return Ok(None); + } + let streams = Streams::new(src)?; + if streams.length.iter().any(|&length| length < 8) { + return Ok(None); + } + let [_, _, _, length4] = streams.length; + if length4 > src.len() { + return Err(ErrorCode::CorruptionDetected); + } + let [_, start2, start3, start4] = streams.start; + let ip = [start2 - 8, start3 - 8, start4 - 8, src.len() - 8]; + let segment = dst_size.div_ceil(4); + let op = [0, segment, 2 * segment, 3 * segment]; + if 3 * segment >= dst_size { + return Ok(None); + } + let [ip0, ip1, ip2, ip3] = ip; + Ok(Some(Self { + ip, + op, + bits: [ + init_fast_dstream(src, ip0)?, + init_fast_dstream(src, ip1)?, + init_fast_dstream(src, ip2)?, + init_fast_dstream(src, ip3)?, + ], + iend: streams.start, + })) + } + + /// `HUF_initRemainingDStream` + fn remaining_dstream<'a>( + &self, + src: &'a [u8], + stream: usize, + segment_end: usize, + ) -> Result> { + if self.op.at(stream)? > segment_end { + return Err(ErrorCode::CorruptionDetected); + } + let ip = self.ip.at(stream)?; + if ip + 8 < self.iend.at(stream)? { + return Err(ErrorCode::CorruptionDetected); + } + BitDStream::resume(src, ip, self.bits.at(stream)?.trailing_zeros()) + } + + /// How many rounds of at most 7 bytes the first stream has input for. The others start above it. + #[inline] + fn input_rounds(&self) -> usize { + let [ip0, ..] = self.ip; + ip0 / 7 + } + + #[inline] + fn streams_are_ordered(&self) -> bool { + self.ip.is_sorted() + } +} + +/// What both fast loops do to a stream after a round. +#[inline(always)] +fn reload_fast(src: &[u8], ip: &mut usize, bits: &mut u64) -> Result<()> { + let ctz = bits.trailing_zeros(); + let nb_bits = ctz & 7; + let nb_bytes = (ctz >> 3) as usize; + *ip = ip.wrapping_sub(nb_bytes); + *bits = (read_le64(src, *ip)? | 1) << nb_bits; + Ok(()) +} + +/// Three parts of `segment_size` bytes, and the rest. +fn split_in_4(dst: &mut [u8], segment_size: usize) -> Result<[&mut [u8]; 4]> { + let (part0, rest) = dst.split_at_mut_checked(segment_size).ok_or(BUG)?; + let (part1, rest) = rest.split_at_mut_checked(segment_size).ok_or(BUG)?; + let (part2, part3) = rest.split_at_mut_checked(segment_size).ok_or(BUG)?; + Ok([part0, part1, part2, part3]) +} + +/// The end of each of the four parts of an output of `dst_size` bytes. +fn segment_ends(dst_size: usize) -> [usize; 4] { + let segment = dst_size.div_ceil(4); + [1, 2, 3, 4].map(|n| (n * segment).min(dst_size)) +} + +// ── X1 ──────────────────────────────────────────────────────────────────── + +/// `HUF_rescaleStats` +fn rescale_stats( + huff_weight: &mut [u8], + rank_val: &mut [u32; NB_RANKS], + table_log: u32, + target_table_log: u32, +) -> Result { + if table_log > target_table_log { + return Ok(table_log); + } + if table_log < target_table_log { + let scale = (target_table_log - table_log) as usize; + for weight in huff_weight { + if *weight != 0 { + *weight += scale as u8; + } + } + let target = target_table_log as usize; + rank_val.move_within(1, scale + 1, target - scale)?; + rank_val.span_mut(1, scale)?.fill(0); + } + Ok(target_table_log) +} + +/// `HUF_readDTableX1_wksp`, once the description of the table is read. +fn build_dtable_x1(dtable: &mut ExactTable, stats: &Code) -> Result<()> { + let mut rank_val = stats.rank_stats; + let mut rank_start = [0u32; NB_RANKS]; + let mut symbols = [0u8; NB_SYMBOLS]; + let mut huff_weight = stats.weights; + let weights = huff_weight.sub_mut(..stats.nb_symbols as usize)?; + + // An X1 table takes half the room of an X2 table, so its log may be one more than the capacity. + let max_table_log = HUF_TABLELOG_MAX + 1; + let target_table_log = max_table_log.min(HUF_DECODER_FAST_TABLELOG); + let table_log = rescale_stats(weights, &mut rank_val, stats.table_log, target_table_log)?; + if table_log > max_table_log { + return Err(ErrorCode::TableLogTooLarge); + } + dtable.table_type = TableType::X1; + dtable.table_log = table_log; + + // Sort the symbols by weight. + let mut next_rank_start = 0u32; + for (start, &count) in rank_start + .iter_mut() + .zip(&rank_val) + .take(table_log as usize + 1) + { + *start = next_rank_start; + next_rank_start += count; + } + for (symbol, &weight) in weights.iter().enumerate() { + let start = rank_start.at_mut(usize::from(weight))?; + symbols.put(*start as usize, symbol as u8)?; + *start += 1; + } + + let [rank0, ..] = rank_val; + let mut symbol = rank0 as usize; + let mut u_start = 0usize; + for w in 1..=table_log { + let symbol_count = rank_val.at(w as usize)? as usize; + let length = (1usize << w) >> 1; + let nb_bits = (table_log + 1 - w) as u16; + for &byte in symbols.span(symbol, symbol_count)? { + dtable + .x1 + .span_mut(u_start, length)? + .fill((u16::from(byte) << 8) | nb_bits); + u_start += length; + } + symbol += symbol_count; + } + Ok(()) +} + +/// `HUF_decodeSymbolX1` +#[inline(always)] +fn decode_symbol_x1(bit_d: &mut BitDStream<'_>, dt: &[u16; TABLE_SIZE], dt_log: u32) -> u8 { + let entry = entry_x1(dt, bit_d.look_bits_fast(dt_log)); + bit_d.skip_bits(u32::from(entry & 0xFF)); + (entry >> 8) as u8 +} + +/// `HUF_decodeStreamX1`: fills `out`. +#[inline] +fn decode_stream_x1( + out: &mut [u8], + bit_d: &mut BitDStream<'_>, + dt: &[u16; TABLE_SIZE], + dt_log: u32, +) { + let end = out.len(); + let mut p = 0usize; + if end > 3 { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 3 < end) { + if let Ok(four) = out.array_at_mut::<4>(p) { + for byte in four { + *byte = decode_symbol_x1(bit_d, dt, dt_log); + } + } + p += 4; + } + } else { + bit_d.reload(); + } + for byte in out.get_mut(p..).unwrap_or_default() { + *byte = decode_symbol_x1(bit_d, dt, dt_log); + } +} + +/// `HUF_decompress1X1_usingDTable_internal_body` +fn decompress_1x1_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { + let mut bit_d = BitDStream::new(src)?; + decode_stream_x1(dst, &mut bit_d, &dtable.x1, dtable.table_log); + if !bit_d.is_at_end() { + return Err(ErrorCode::CorruptionDetected); + } + Ok(dst.len()) +} + +/// `HUF_decompress4X1_usingDTable_internal_body` +fn decompress_4x1_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { + if src.len() < 10 { + return Err(ErrorCode::CorruptionDetected); + } + if dst.len() < 6 { + return Err(ErrorCode::CorruptionDetected); + } + let dst_size = dst.len(); + let dt = &dtable.x1; + let dt_log = dtable.table_log; + let streams = Streams::new(src)?; + let [_, _, _, length4] = streams.length; + if length4 > src.len() { + return Err(ErrorCode::CorruptionDetected); + } + let segment_size = dst_size.div_ceil(4); + if 3 * segment_size > dst_size { + return Err(ErrorCode::CorruptionDetected); + } + let [mut bit_d1, mut bit_d2, mut bit_d3, mut bit_d4] = streams.open(src)?; + + let [out1, out2, out3, out4] = split_in_4(dst, segment_size)?; + // The four streams advance together, by `done`, and the last part is the shortest. + let mut done = 0usize; + if out4.len() >= 8 { + let mut end_signal = true; + while end_signal & (done + 3 < out4.len()) { + let four1 = out1.array_at_mut::<4>(done)?; + let four2 = out2.array_at_mut::<4>(done)?; + let four3 = out3.array_at_mut::<4>(done)?; + let four4 = out4.array_at_mut::<4>(done)?; + for (((byte1, byte2), byte3), byte4) in + four1.iter_mut().zip(four2).zip(four3).zip(four4) + { + *byte1 = decode_symbol_x1(&mut bit_d1, dt, dt_log); + *byte2 = decode_symbol_x1(&mut bit_d2, dt, dt_log); + *byte3 = decode_symbol_x1(&mut bit_d3, dt, dt_log); + *byte4 = decode_symbol_x1(&mut bit_d4, dt, dt_log); + } + done += 4; + end_signal &= bit_d1.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d2.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d3.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d4.reload_fast() == DStreamStatus::Unfinished; + } + } + + decode_stream_x1(out1.sub_mut(done..)?, &mut bit_d1, dt, dt_log); + decode_stream_x1(out2.sub_mut(done..)?, &mut bit_d2, dt, dt_log); + decode_stream_x1(out3.sub_mut(done..)?, &mut bit_d3, dt, dt_log); + decode_stream_x1(out4.sub_mut(done..)?, &mut bit_d4, dt, dt_log); + + if !(bit_d1.is_at_end() & bit_d2.is_at_end() & bit_d3.is_at_end() & bit_d4.is_at_end()) { + return Err(ErrorCode::CorruptionDetected); + } + Ok(dst_size) +} + +/// `HUF_decompress4X1_usingDTable_internal_fast_c_loop` +fn fast_loop_4x1( + args: &mut FastArgs, + dst: &mut [u8], + src: &[u8], + dt: &[u16; TABLE_SIZE], +) -> Result<()> { + let [_, segment_size, ..] = args.op; + let [part0, part1, part2, part3] = split_in_4(dst, segment_size)?; + // How much of each part is written. It is the same for all of them. + let mut done = 0usize; + + let [mut ip0, mut ip1, mut ip2, mut ip3] = args.ip; + let [mut bits0, mut bits1, mut bits2, mut bits3] = args.bits; + + #[inline(always)] + fn decode(bits: &mut u64, dt: &[u16; TABLE_SIZE]) -> u8 { + let entry = entry_x1(dt, *bits >> 53); + *bits <<= entry & 0x3F; + (entry >> 8) as u8 + } + + loop { + // Each round writes 5 bytes to each part, of which the last is the shortest, and reads at most 7 of each stream. + let rounds = ((part3.len() - done) / 5).min(ip0 / 7); + if rounds == 0 { + break; + } + if !(ip0 <= ip1 && ip1 <= ip2 && ip2 <= ip3) { + break; + } + let (out0, _) = part0.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + let (out1, _) = part1.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + let (out2, _) = part2.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + let (out3, _) = part3.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + for (((out0, out1), out2), out3) in out0.iter_mut().zip(out1).zip(out2).zip(out3) { + for (((byte0, byte1), byte2), byte3) in out0.iter_mut().zip(out1).zip(out2).zip(out3) { + *byte0 = decode(&mut bits0, dt); + *byte1 = decode(&mut bits1, dt); + *byte2 = decode(&mut bits2, dt); + *byte3 = decode(&mut bits3, dt); + } + reload_fast(src, &mut ip0, &mut bits0)?; + reload_fast(src, &mut ip1, &mut bits1)?; + reload_fast(src, &mut ip2, &mut bits2)?; + reload_fast(src, &mut ip3, &mut bits3)?; + } + done += rounds * 5; + } + + args.ip = [ip0, ip1, ip2, ip3]; + args.bits = [bits0, bits1, bits2, bits3]; + args.op = args.op.map(|start| start + done); + Ok(()) +} + +/// `HUF_decompress4X1_usingDTable_internal_fast`. `None` if the input has to be decoded by the other loop. +fn decompress_4x1_fast(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result> { + let Some(mut args) = FastArgs::new(dst.len(), src, dtable.table_log)? else { + return Ok(None); + }; + fast_loop_4x1(&mut args, dst, src, &dtable.x1)?; + + for (stream, segment_end) in segment_ends(dst.len()).into_iter().enumerate() { + let mut bit_d = args.remaining_dstream(src, stream, segment_end)?; + let out = dst.sub_mut(args.op.at(stream)?..segment_end)?; + decode_stream_x1(out, &mut bit_d, &dtable.x1, HUF_DECODER_FAST_TABLELOG); + } + Ok(Some(dst.len())) +} + +/// `HUF_decompress4X1_usingDTable_internal` +fn decompress_4x1_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { + if let Some(size) = decompress_4x1_fast(dst, src, dtable)? { + return Ok(size); + } + decompress_4x1_body(dst, src, dtable) +} + +// ── X2 ──────────────────────────────────────────────────────────────────── + +type RankValCol = [u32; NB_RANKS]; + +/// `HUF_buildDEltX2U32` +#[inline(always)] +fn build_delt_x2(symbol: u32, nb_bits: u32, base_seq: u32, level: u32) -> u32 { + let seq = if level == 1 { + symbol + } else { + base_seq + (symbol << 8) + }; + seq + (nb_bits << 16) + (level << 24) +} + +/// `HUF_fillDTableX2ForWeight` +fn fill_dtable_x2_for_weight( + dtable_rank: &mut [u32], + symbols: &[u8], + nb_bits: u32, + table_log: u32, + base_seq: u32, + level: u32, +) -> Result<()> { + let length = 1usize << (table_log.wrapping_sub(nb_bits) & 0x1F); + let entries = dtable_rank.sub_mut(..length * symbols.len())?; + for (entries, &symbol) in entries.chunks_exact_mut(length).zip(symbols) { + entries.fill(build_delt_x2(u32::from(symbol), nb_bits, base_seq, level)); + } + Ok(()) +} + +/// `HUF_fillDTableX2Level2` +fn fill_dtable_x2_level2( + dtable: &mut [u32], + target_log: u32, + consumed_bits: u32, + rank_val: &RankValCol, + min_weight: usize, + max_weight1: usize, + sorted_symbols: &[u8; NB_SYMBOLS], + rank_start: &[u32; NB_RANKS + 2], + nb_bits_baseline: u32, + base_seq: u32, +) -> Result<()> { + // The codes that are too long for a second symbol to fit hold the first symbol alone. + if min_weight > 1 { + let length = 1usize << (target_log.wrapping_sub(consumed_bits) & 0x1F); + let skip_size = rank_val.at(min_weight)? as usize; + let filled = match length { + 2 | 4 => length, + _ => skip_size.next_multiple_of(8), + }; + let first_symbol = base_seq; + dtable + .sub_mut(..filled)? + .fill(build_delt_x2(first_symbol, consumed_bits, 0, 1)); + } + + for w in min_weight..max_weight1 { + let begin = rank_start.at(w)? as usize; + let end = rank_start.at(w + 1)? as usize; + let nb_bits = nb_bits_baseline - w as u32; + let total_bits = nb_bits + consumed_bits; + fill_dtable_x2_for_weight( + dtable.sub_mut(rank_val.at(w)? as usize..)?, + sorted_symbols.sub(begin..end)?, + total_bits, + target_log, + base_seq, + 2, + )?; + } + Ok(()) +} + +/// `HUF_fillDTableX2` +fn fill_dtable_x2( + dtable: &mut [u32; TABLE_SIZE], + target_log: u32, + sorted_list: &[u8; NB_SYMBOLS], + rank_start: &[u32; NB_RANKS + 2], + rank_val_origin: &[RankValCol; HUF_TABLELOG_MAX as usize], + max_weight: u32, + nb_bits_baseline: u32, +) -> Result<()> { + let [rank_val, ..] = rank_val_origin; + let scale_log = nb_bits_baseline as i32 - target_log as i32; + let min_bits = nb_bits_baseline - max_weight; + let w_end = max_weight as usize + 1; + + for w in 1..w_end { + let begin = rank_start.at(w)? as usize; + let end = rank_start.at(w + 1)? as usize; + let nb_bits = nb_bits_baseline - w as u32; + + if target_log.wrapping_sub(nb_bits) >= min_bits { + // Enough room for a second symbol. + let mut start = rank_val.at(w)? as usize; + let length = 1usize << (target_log.wrapping_sub(nb_bits) & 0x1F); + let min_weight = (nb_bits as i32 + scale_log).max(1) as usize; + for &symbol in sorted_list.sub(begin..end)? { + fill_dtable_x2_level2( + dtable.sub_mut(start..)?, + target_log, + nb_bits, + rank_val_origin.get(nb_bits as usize).ok_or(BUG)?, + min_weight, + w_end, + sorted_list, + rank_start, + nb_bits_baseline, + u32::from(symbol), + )?; + start += length; + } + } else { + fill_dtable_x2_for_weight( + dtable.sub_mut(rank_val.at(w)? as usize..)?, + sorted_list.sub(begin..end)?, + nb_bits, + target_log, + 0, + 1, + )?; + } + } + Ok(()) +} + +/// `HUF_readDTableX2_wksp`, once the description of the table is read. +fn build_dtable_x2(dtable: &mut ExactTable, stats: &Code) -> Result<()> { + let mut rank_val = [[0u32; NB_RANKS]; HUF_TABLELOG_MAX as usize]; + let rank_stats = stats.rank_stats; + // `rank_start0[w + 1]` is where the symbols of weight `w` start, until sorting moves it to where they end. + let mut rank_start0 = [0u32; NB_RANKS + 2]; + let mut sorted_symbol = [0u8; NB_SYMBOLS]; + let weight_list = stats.weights; + let mut max_table_log = HUF_TABLELOG_MAX; + let table_log = stats.table_log; + + if table_log > max_table_log { + return Err(ErrorCode::TableLogTooLarge); + } + if table_log <= HUF_DECODER_FAST_TABLELOG && max_table_log > HUF_DECODER_FAST_TABLELOG { + max_table_log = HUF_DECODER_FAST_TABLELOG; + } + + let mut max_w = table_log as usize; + while rank_stats.at(max_w)? == 0 { + max_w = max_w.checked_sub(1).ok_or(BUG)?; + } + + let mut next_rank_start = 0u32; + for w in 1..=max_w { + rank_start0.put(w + 1, next_rank_start)?; + next_rank_start += rank_stats.at(w)?; + } + // The symbols of weight 0 go to the end. + rank_start0.put(1, next_rank_start)?; + rank_start0.put(max_w + 2, next_rank_start)?; + + for (symbol, &weight) in weight_list + .iter() + .enumerate() + .take(stats.nb_symbols as usize) + { + let start = rank_start0.at_mut(usize::from(weight) + 1)?; + sorted_symbol.put(*start as usize, symbol as u8)?; + *start += 1; + } + rank_start0.put(1, 0)?; + + { + let rescale = (max_table_log - table_log) as i32 - 1; + let mut next_rank_val = 0u32; + let [rank_val0, ..] = &mut rank_val; + for w in 1..=max_w { + rank_val0.put(w, next_rank_val)?; + next_rank_val += rank_stats.at(w)? << (w as i32 + rescale); + } + let rank_val0 = *rank_val0; + let min_bits = table_log + 1 - max_w as u32; + for consumed in min_bits..(max_table_log - min_bits + 1) { + let rank_val_col = rank_val.get_mut(consumed as usize).ok_or(BUG)?; + for (shifted, &val) in rank_val_col.iter_mut().zip(&rank_val0).skip(1).take(max_w) { + *shifted = val >> consumed; + } + } + } + + fill_dtable_x2( + &mut dtable.x2, + max_table_log, + &sorted_symbol, + &rank_start0, + &rank_val, + max_w as u32, + table_log + 1, + )?; + + dtable.table_log = max_table_log; + dtable.table_type = TableType::X2; + Ok(()) +} + +/// `HUF_decodeSymbolX2`: writes two bytes at `pos` and returns how many of them count. +#[inline(always)] +fn decode_symbol_x2( + out: &mut [u8], + pos: usize, + bit_d: &mut BitDStream<'_>, + dt: &[u32; TABLE_SIZE], + dt_log: u32, +) -> Result { + let entry = entry_x2(dt, bit_d.look_bits_fast(dt_log)); + *out.array_at_mut(pos)? = (entry as u16).to_le_bytes(); + bit_d.skip_bits((entry >> 16) & 0xFF); + Ok((entry >> 24) as usize) +} + +/// `HUF_decodeLastSymbolX2` +#[inline(always)] +fn decode_last_symbol_x2( + out: &mut [u8], + pos: usize, + bit_d: &mut BitDStream<'_>, + dt: &[u32; TABLE_SIZE], + dt_log: u32, +) -> Result<()> { + let entry = entry_x2(dt, bit_d.look_bits_fast(dt_log)); + out.put(pos, entry as u8)?; + let nb_bits = (entry >> 16) & 0xFF; + if entry >> 24 == 1 { + bit_d.skip_bits(nb_bits); + } else if bit_d.bits_consumed < u64::BITS { + bit_d.skip_bits(nb_bits); + // The code of the second symbol, which is not wanted, may be cut off. + if bit_d.bits_consumed > u64::BITS { + bit_d.bits_consumed = u64::BITS; + } + } + Ok(()) +} + +/// `HUF_decodeStreamX2`: fills `out` and returns its length. +#[inline] +fn decode_stream_x2( + out: &mut [u8], + bit_d: &mut BitDStream<'_>, + dt: &[u32; TABLE_SIZE], + dt_log: u32, +) -> Result { + let end = out.len(); + let mut p = 0usize; + + if end >= 8 { + if dt_log <= 11 { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 9 < end) { + for _ in 0..5 { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + } + } else { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 7 < end) { + for _ in 0..4 { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + } + } + } else { + bit_d.reload(); + } + + if end - p >= 2 { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 2 <= end) { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + while p + 2 <= end { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + } + + if p < end { + decode_last_symbol_x2(out, p, bit_d, dt, dt_log)?; + p += 1; + } + Ok(p) +} + +/// `HUF_decompress1X2_usingDTable_internal_body` +fn decompress_1x2_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { + let mut bit_d = BitDStream::new(src)?; + decode_stream_x2(dst, &mut bit_d, &dtable.x2, dtable.table_log)?; + if !bit_d.is_at_end() { + return Err(ErrorCode::CorruptionDetected); + } + Ok(dst.len()) +} + +/// `HUF_decompress4X2_usingDTable_internal_body` +fn decompress_4x2_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { + if src.len() < 10 { + return Err(ErrorCode::CorruptionDetected); + } + if dst.len() < 6 { + return Err(ErrorCode::CorruptionDetected); + } + let oend = dst.len(); + let dt = &dtable.x2; + let dt_log = dtable.table_log; + let streams = Streams::new(src)?; + let [_, _, _, length4] = streams.length; + if length4 > src.len() { + return Err(ErrorCode::CorruptionDetected); + } + let segment_size = oend.div_ceil(4); + let op_start2 = segment_size; + let op_start3 = 2 * segment_size; + let op_start4 = 3 * segment_size; + if op_start4 > oend { + return Err(ErrorCode::CorruptionDetected); + } + let [mut bit_d1, mut bit_d2, mut bit_d3, mut bit_d4] = streams.open(src)?; + let (mut op1, mut op2, mut op3, mut op4) = (0, op_start2, op_start3, op_start4); + + // A damaged stream can run over the start of the next part, so the parts are not split here. + if oend - op4 >= 8 { + let mut end_signal = true; + while end_signal & (op4 + 7 < oend) { + for _ in 0..4 { + op1 += decode_symbol_x2(dst, op1, &mut bit_d1, dt, dt_log)?; + } + for _ in 0..4 { + op2 += decode_symbol_x2(dst, op2, &mut bit_d2, dt, dt_log)?; + } + end_signal &= bit_d1.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d2.reload_fast() == DStreamStatus::Unfinished; + for _ in 0..4 { + op3 += decode_symbol_x2(dst, op3, &mut bit_d3, dt, dt_log)?; + } + for _ in 0..4 { + op4 += decode_symbol_x2(dst, op4, &mut bit_d4, dt, dt_log)?; + } + end_signal &= bit_d3.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d4.reload_fast() == DStreamStatus::Unfinished; + } + } + + if op1 > op_start2 || op2 > op_start3 || op3 > op_start4 { + return Err(ErrorCode::CorruptionDetected); + } + + decode_stream_x2(dst.sub_mut(op1..op_start2)?, &mut bit_d1, dt, dt_log)?; + decode_stream_x2(dst.sub_mut(op2..op_start3)?, &mut bit_d2, dt, dt_log)?; + decode_stream_x2(dst.sub_mut(op3..op_start4)?, &mut bit_d3, dt, dt_log)?; + decode_stream_x2(dst.sub_mut(op4..oend)?, &mut bit_d4, dt, dt_log)?; + + if !(bit_d1.is_at_end() & bit_d2.is_at_end() & bit_d3.is_at_end() & bit_d4.is_at_end()) { + return Err(ErrorCode::CorruptionDetected); + } + Ok(oend) +} + +/// `HUF_decompress4X2_usingDTable_internal_fast_c_loop` +fn fast_loop_4x2( + args: &mut FastArgs, + dst: &mut [u8], + src: &[u8], + dt: &[u32; TABLE_SIZE], +) -> Result<()> { + let [_, end0, end1, end2] = args.op; + let oend = [end0, end1, end2, dst.len()]; + + #[inline(always)] + fn decode( + dst: &mut [u8], + op: &mut usize, + bits: &mut u64, + dt: &[u32; TABLE_SIZE], + ) -> Result<()> { + let entry = entry_x2(dt, *bits >> 53); + *dst.array_at_mut(*op)? = (entry as u16).to_le_bytes(); + *bits <<= (entry >> 16) & 0x3F; + *op += (entry >> 24) as usize; + Ok(()) + } + + loop { + // Each round writes at most 10 bytes to each part and reads at most 7 of each stream. + let mut rounds = args.input_rounds(); + for (&end, &op) in oend.iter().zip(&args.op) { + rounds = rounds.min((end - op) / 10); + } + if rounds == 0 { + break; + } + if !args.streams_are_ordered() { + break; + } + let [mut op0, mut op1, mut op2, mut op3] = args.op; + let [mut ip0, mut ip1, mut ip2, mut ip3] = args.ip; + let [mut bits0, mut bits1, mut bits2, mut bits3] = args.bits; + let olimit = op3 + rounds * 5; + loop { + for _ in 0..5 { + decode(dst, &mut op0, &mut bits0, dt)?; + decode(dst, &mut op1, &mut bits1, dt)?; + decode(dst, &mut op2, &mut bits2, dt)?; + decode(dst, &mut op3, &mut bits3, dt)?; + } + reload_fast(src, &mut ip0, &mut bits0)?; + reload_fast(src, &mut ip1, &mut bits1)?; + reload_fast(src, &mut ip2, &mut bits2)?; + reload_fast(src, &mut ip3, &mut bits3)?; + if op3 >= olimit { + break; + } + } + args.op = [op0, op1, op2, op3]; + args.ip = [ip0, ip1, ip2, ip3]; + args.bits = [bits0, bits1, bits2, bits3]; + } + Ok(()) +} + +/// `HUF_decompress4X2_usingDTable_internal_fast`. `None` if the input has to be decoded by the other loop. +fn decompress_4x2_fast(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result> { + let Some(mut args) = FastArgs::new(dst.len(), src, dtable.table_log)? else { + return Ok(None); + }; + fast_loop_4x2(&mut args, dst, src, &dtable.x2)?; + + for (stream, segment_end) in segment_ends(dst.len()).into_iter().enumerate() { + let mut bit_d = args.remaining_dstream(src, stream, segment_end)?; + let out = dst.sub_mut(args.op.at(stream)?..segment_end)?; + decode_stream_x2(out, &mut bit_d, &dtable.x2, HUF_DECODER_FAST_TABLELOG)?; + } + Ok(Some(dst.len())) +} + +/// `HUF_decompress4X2_usingDTable_internal` +fn decompress_4x2_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { + if let Some(size) = decompress_4x2_fast(dst, src, dtable)? { + return Ok(size); + } + decompress_4x2_body(dst, src, dtable) +} + +// ── choosing between X1 and X2 ──────────────────────────────────────────── + +/// `algoTime`: for each sixteenth of compression ratio, the time to build a table and to decode 256 bytes, for X1 and X2. +#[rustfmt::skip] +const ALGO_TIME: [[(u32, u32); 2]; 16] = [ + [( 0, 0), ( 1, 1)], + [( 0, 0), ( 1, 1)], + [( 150, 216), ( 381, 119)], + [( 170, 205), ( 514, 112)], + [( 177, 199), ( 539, 110)], + [( 197, 194), ( 644, 107)], + [( 221, 192), ( 735, 107)], + [( 256, 189), ( 881, 106)], + [( 359, 188), (1167, 109)], + [( 582, 187), (1570, 114)], + [( 688, 187), (1712, 122)], + [( 825, 186), (1965, 136)], + [( 976, 185), (2131, 150)], + [(1180, 186), (2070, 175)], + [(1377, 185), (1731, 202)], + [(1412, 185), (1695, 202)], +]; + +/// `HUF_selectDecoder`: whether X2 is expected to be the faster one. `dst_size` is not 0. +pub(super) fn select_decoder_x2(dst_size: usize, src_size: usize) -> Result { + let q = if src_size >= dst_size { + 15 + } else { + src_size * 16 / dst_size + }; + let d256 = (dst_size >> 8) as u32; + let [(table0, decode0), (table1, decode1)] = ALGO_TIME.at(q)?; + let dtime0 = table0 + decode0 * d256; + let mut dtime1 = table1 + decode1 * d256; + // A small advantage to the algorithm that uses less memory. + dtime1 += dtime1 >> 5; + Ok(dtime1 < dtime0) +} + +impl ExactTable { + fn build(code: &Code, table_type: TableType) -> Result { + let mut table = Self::new(); + match table_type { + TableType::X1 => build_dtable_x1(&mut table, code)?, + TableType::X2 => build_dtable_x2(&mut table, code)?, + } + Ok(table) + } +} + +/// `HUF_decompress1X_usingDTable` +#[cold] +#[inline(never)] +pub(super) fn decompress_1x( + dst: &mut [u8], + src: &[u8], + code: &Code, + table_type: TableType, +) -> Result { + let dtable = ExactTable::build(code, table_type)?; + match dtable.table_type { + TableType::X1 => decompress_1x1_using_dtable(dst, src, &dtable), + TableType::X2 => decompress_1x2_using_dtable(dst, src, &dtable), + } +} + +/// `HUF_decompress4X_usingDTable` +#[cold] +#[inline(never)] +pub(super) fn decompress_4x( + dst: &mut [u8], + src: &[u8], + code: &Code, + table_type: TableType, +) -> Result { + let dtable = ExactTable::build(code, table_type)?; + match dtable.table_type { + TableType::X1 => decompress_4x1_using_dtable(dst, src, &dtable), + TableType::X2 => decompress_4x2_using_dtable(dst, src, &dtable), + } +} diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 6a4f80f933f1..0dbe4952288b 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -2,6 +2,7 @@ mod block; mod huf; +mod huf_exact; mod seq_table; use std::borrow::Cow; From 68e6f8e57d00d9ecef27654bc602083a744458b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:00:06 +0000 Subject: [PATCH 060/199] zstd: a streaming decoder touches no more of its buffers than it uses --- src/zstd/decompress/block.rs | 101 ++++++++++++++++++++++++++-------- src/zstd/decompress/mod.rs | 103 ++++++++++++++++++++++++++--------- src/zstd/util.rs | 2 +- 3 files changed, 155 insertions(+), 51 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 9c150adaa8c6..cc33bff2a779 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -94,10 +94,45 @@ impl ExtDict<'_> { } } +/// A buffer that output goes to. +pub(crate) enum OutputBuffer<'a> { + Whole(&'a mut [u8]), + /// What has been written so far of a buffer. It has the capacity for all that a block is let to write. + Used(&'a mut Vec), +} + +impl OutputBuffer<'_> { + #[inline(always)] + fn bytes(&mut self) -> &mut [u8] { + match self { + Self::Whole(bytes) => bytes, + Self::Used(bytes) => bytes, + } + } + + #[inline(always)] + fn reborrow(&mut self) -> OutputBuffer<'_> { + match self { + Self::Whole(bytes) => OutputBuffer::Whole(bytes), + Self::Used(bytes) => OutputBuffer::Used(bytes), + } + } + + /// Makes `bytes` reach `end`, and somewhat further if `limit` allows. + fn extend_to(&mut self, end: usize, limit: usize) { + if let Self::Used(bytes) = self + && bytes.len() < end + { + let ahead = (bytes.len() / 2).clamp(1 << 12, 1 << 17) + 2 * WILDCOPY_OVERLENGTH; + bytes.resize((end + ahead).min(limit), 0); + } + } +} + /// Where a block is written. pub(crate) struct BlockOutput<'a> { /// Holds the contiguous output so far, from `prefix_start` to `start`. - pub(crate) buf: &'a mut [u8], + pub(crate) buf: OutputBuffer<'a>, pub(crate) prefix_start: usize, pub(crate) start: usize, /// How many bytes from `start` on may be written. @@ -505,7 +540,11 @@ impl BlockDecoder { literals.size }; let mut exec = Executor { - buf: buf.sub_mut(..oend)?, + out: match buf { + OutputBuffer::Whole(bytes) => OutputBuffer::Whole(bytes.sub_mut(..oend)?), + used => used, + }, + oend, op: start, prefix_start, ext_dict, @@ -849,8 +888,10 @@ fn copy_match_exact(buf: &mut [u8], src: usize, dst: usize, len: usize) -> Resul /// Turns sequences into output. struct Executor<'buf, 'a> { - /// Ends where the block has to stop writing: `oend`. - buf: &'buf mut [u8], + /// Ends no further than `oend`. + out: OutputBuffer<'buf>, + /// Where the block has to stop writing. + oend: usize, op: usize, prefix_start: usize, ext_dict: ExtDict<'a>, @@ -876,7 +917,7 @@ impl<'a> Executor<'_, 'a> { #[inline(always)] fn fork(&mut self) -> Executor<'_, 'a> { Executor { - buf: &mut *self.buf, + out: self.out.reborrow(), ..*self } } @@ -905,9 +946,11 @@ impl<'a> Executor<'_, 'a> { let offset = sequence.offset; let mut op = self.op; - if lit_length + match_length > self.buf.len() - op { + if lit_length + match_length > self.oend - op { return Err(ErrorCode::DstSizeTooSmall); } + self.out + .extend_to(op + lit_length + match_length, self.oend); if lit_length > self.lits_left() { return Err(ErrorCode::CorruptionDetected); } @@ -918,7 +961,9 @@ impl<'a> Executor<'_, 'a> { } } - self.buf.write_at(op, self.lits.sub(..lit_length)?)?; + self.out + .bytes() + .write_at(op, self.lits.sub(..lit_length)?)?; self.lits = self.lits.sub(lit_length..)?; op += lit_length; self.op = op + match_length; @@ -931,15 +976,17 @@ impl<'a> Executor<'_, 'a> { } let back = offset - in_prefix; if back >= match_length { - return self.ext_dict.copy_to(self.buf, op, back, match_length); + return self + .ext_dict + .copy_to(self.out.bytes(), op, back, match_length); } // The match spans the two segments. - self.ext_dict.copy_to(self.buf, op, back, back)?; + self.ext_dict.copy_to(self.out.bytes(), op, back, back)?; op += back; match_length -= back; match_pos = self.prefix_start; } - copy_match_exact(self.buf, match_pos, op, match_length) + copy_match_exact(self.out.bytes(), match_pos, op, match_length) } /// `ZSTD_execSequence` and `ZSTD_execSequenceSplitLitBuffer`, if there is room to copy in chunks and the match is @@ -953,7 +1000,7 @@ impl<'a> Executor<'_, 'a> { let offset = sequence.offset; let op = self.op; - let room = self.buf.get_mut(op..)?; + let room = self.out.bytes().get_mut(op..)?; if lit_length + match_length + WILDCOPY_OVERLENGTH > room.len() || lit_length + WILDCOPY_OVERLENGTH > self.lits.len() { @@ -979,7 +1026,7 @@ impl<'a> Executor<'_, 'a> { if offset > o_lit_end - self.prefix_start { return None; } - let (before, room) = self.buf.split_at_mut_checked(o_lit_end)?; + let (before, room) = self.out.bytes().split_at_mut_checked(o_lit_end)?; let from = before.get(before.len().checked_sub(offset)?..)?; if offset >= 16 { // `ZSTD_wildcopy(ZSTD_no_overlap)` @@ -993,7 +1040,7 @@ impl<'a> Executor<'_, 'a> { let mut done = 16; while done < match_length { let to = o_lit_end + done; - copy_within::<16>(self.buf, to - offset, to).ok()?; + copy_within::<16>(self.out.bytes(), to - offset, to).ok()?; done += 16; } } @@ -1022,21 +1069,21 @@ impl<'a> Executor<'_, 'a> { const DEC32_TABLE: [usize; 8] = [0, 1, 2, 1, 4, 4, 4, 4]; const DEC64_TABLE: [usize; 8] = [8, 8, 8, 7, 8, 9, 10, 11]; for i in 0..4 { - let byte = self.buf.at(match_pos + i)?; - self.buf.put(op + i, byte)?; + let byte = self.out.bytes().at(match_pos + i)?; + self.out.bytes().put(op + i, byte)?; } match_pos += DEC32_TABLE.at(offset)?; - copy_within::<4>(self.buf, match_pos, op + 4)?; + copy_within::<4>(self.out.bytes(), match_pos, op + 4)?; match_pos += 8; match_pos -= DEC64_TABLE.at(offset)?; } else { - copy_within::<8>(self.buf, match_pos, op)?; + copy_within::<8>(self.out.bytes(), match_pos, op)?; match_pos += 8; } op += 8; let mut done = 8usize; while done < match_length { - copy_within::<8>(self.buf, match_pos, op)?; + copy_within::<8>(self.out.bytes(), match_pos, op)?; match_pos += 8; op += 8; done += 8; @@ -1067,10 +1114,13 @@ impl<'a> Executor<'_, 'a> { fn exec_sequence_leaving_split_cold(&mut self, mut sequence: Sequence) -> Result<()> { let leftover_lit = self.lits_left(); if leftover_lit > 0 { - if leftover_lit > self.buf.len() - self.op { + if leftover_lit > self.oend - self.op { return Err(ErrorCode::DstSizeTooSmall); } - self.buf.write_at(self.op, self.lits.sub(..leftover_lit)?)?; + self.out.extend_to(self.op + leftover_lit, self.oend); + self.out + .bytes() + .write_at(self.op, self.lits.sub(..leftover_lit)?)?; sequence.lit_length -= leftover_lit as u32; self.op += leftover_lit; } @@ -1100,8 +1150,9 @@ impl<'a> Executor<'_, 'a> { // Positions fit in 32 bits, so that the compiler sees that nothing added to them overflows. const MAX_BACK: usize = 3 << 30; let start = self.prefix_start.max(self.op.saturating_sub(MAX_BACK)); - let end = self.buf.len().min(start + u32::MAX as usize); - let buf = self.buf.get_mut(start..end).unwrap_or_default(); + let buf = self.out.bytes(); + let end = buf.len().min(start + u32::MAX as usize); + let buf = buf.get_mut(start..end).unwrap_or_default(); let mut op = self.op.wrapping_sub(start) as u32; let lits = self.lits.get(..u32::MAX as usize).unwrap_or(self.lits); let mut lit_pos = 0u32; @@ -1255,11 +1306,13 @@ impl<'a> Executor<'_, 'a> { let parts = [self.lits, extra] .map(|part| part.get(..part.len().saturating_sub(WILDCOPY_OVERLENGTH))); let parts = parts.map(Option::unwrap_or_default); - if parts.iter().map(|part| part.len()).sum::() > self.buf.len() - self.op { + let size = parts.iter().map(|part| part.len()).sum::(); + if size > self.oend - self.op { return Err(ErrorCode::DstSizeTooSmall); } + self.out.extend_to(self.op + size, self.op + size); for part in parts { - self.buf.write_at(self.op, part)?; + self.out.bytes().write_at(self.op, part)?; self.op += part.len(); } Ok(()) diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 0dbe4952288b..7c123e5fd52a 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -7,7 +7,9 @@ mod seq_table; use std::borrow::Cow; -use block::{BlockDecoder, BlockOutput, EntropyDTables, ExtDict, Streaming, get_c_block_size}; +use block::{ + BlockDecoder, BlockOutput, EntropyDTables, ExtDict, OutputBuffer, Streaming, get_c_block_size, +}; use seq_table::{LL_BASE, ML_BASE, OF_BASE, OF_BITS, SeqTableSources, TableSource}; use crate::common::entropy::read_ncount; @@ -22,7 +24,7 @@ use crate::common::{ ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, }; use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, read_le16, read_le32, read_le64, try_copy, try_zeroed}; +use crate::util::{SliceExt, read_le16, read_le32, read_le64, try_copy}; use crate::{InBuffer, OutBuffer}; /// `ZSTD_FRAMEHEADERSIZE_PREFIX`: what it takes to know the size of a frame header. @@ -470,6 +472,45 @@ enum ExtKind { Within { end: usize, len: usize }, } +/// A buffer whose size a frame header dictates. It takes up no more memory than has been used of it. +struct Buffer { + /// What has been used so far. There is capacity for `size`. + bytes: Vec, + size: usize, +} + +impl Buffer { + const fn new() -> Self { + Self { + bytes: Vec::new(), + size: 0, + } + } + + fn with_size(size: usize) -> Result { + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(size) + .map_err(|_| ErrorCode::MemoryAllocation)?; + Ok(Self { bytes, size }) + } + + /// The `len` bytes from `start` on, which are within `size`. + fn span_mut(&mut self, start: usize, len: usize) -> Result<&mut [u8]> { + let end = start + len; + if self.bytes.len() < end && end <= self.size { + self.bytes.resize(end, 0); + } + self.bytes.span_mut(start, len) + } +} + +impl Default for Buffer { + fn default() -> Self { + Self::new() + } +} + /// All of a `ZSTD_DCtx` but the dictionary it owns. struct DState { block: BlockDecoder, @@ -493,10 +534,10 @@ struct DState { dict_id: u32, stream_stage: DStreamStage, - in_buff: Vec, + in_buff: Buffer, in_pos: usize, max_window_size: usize, - out_buff: Vec, + out_buff: Buffer, out_start: usize, out_end: usize, /// The last byte of a frame's input is held back until its output is flushed. @@ -542,10 +583,10 @@ impl DCtx { rle_size: 0, dict_id: 0, stream_stage: DStreamStage::Init, - in_buff: Vec::new(), + in_buff: Buffer::new(), in_pos: 0, max_window_size: ZSTD_MAXWINDOWSIZE_DEFAULT, - out_buff: Vec::new(), + out_buff: Buffer::new(), out_start: 0, out_end: 0, hostage_byte: false, @@ -726,7 +767,7 @@ impl DState { /// One compressed block, to `buf` from `start` on. fn decompress_block( &mut self, - buf: &mut [u8], + buf: OutputBuffer<'_>, start: usize, capacity: usize, src: &[u8], @@ -787,7 +828,14 @@ impl DState { let decoded_size = match properties.block_type { BlockType::Compressed => { let capacity = dst.len() - op; - self.decompress_block(dst, op, capacity, block, ddict, Streaming::No)? + self.decompress_block( + OutputBuffer::Whole(dst), + op, + capacity, + block, + ddict, + Streaming::No, + )? } BlockType::Raw => copy_raw_block(dst.sub_mut(op..)?, block)?, BlockType::Rle => set_rle_block( @@ -925,7 +973,7 @@ impl DState { BlockType::Compressed => { self.expected = 0; self.decompress_block( - &mut out_buff, + OutputBuffer::Used(&mut out_buff.bytes), dst, dst_capacity, src, @@ -934,13 +982,13 @@ impl DState { ) } BlockType::Raw => out_buff - .span_mut(dst, dst_capacity) + .span_mut(dst, dst_capacity.min(src.len())) .and_then(|dst| copy_raw_block(dst, src)) .inspect(|&r_size| self.expected -= r_size), BlockType::Rle => { self.expected = 0; out_buff - .span_mut(dst, dst_capacity) + .span_mut(dst, dst_capacity.min(self.rle_size)) .and_then(|dst| set_rle_block(dst, src.at(0)?, self.rle_size)) } BlockType::Reserved => Err(ErrorCode::CorruptionDetected), @@ -952,7 +1000,8 @@ impl DState { } self.decoded_size += r_size as u64; if self.validate_checksum { - self.xxh_state.update(self.out_buff.span(dst, r_size)?); + self.xxh_state + .update(self.out_buff.bytes.span(dst, r_size)?); } self.previous_dst_end = Some(dst + r_size); @@ -1012,7 +1061,7 @@ impl DState { let dst_size = if is_skip_frame { 0 } else { - self.out_buff.len() - self.out_start + self.out_buff.size - self.out_start }; let decoded_size = self.decompress_continue(dst_size, src, ddict)?; if decoded_size == 0 && !is_skip_frame { @@ -1035,7 +1084,7 @@ impl DState { ); // `ZSTD_DCtx_updateOversizedDuration` - if self.in_buff.len() + self.out_buff.len() + if self.in_buff.size + self.out_buff.size >= (needed_in_buff_size + needed_out_buff_size) * ZSTD_WORKSPACETOOLARGE_FACTOR { self.oversized_duration += 1; @@ -1044,13 +1093,13 @@ impl DState { } let too_small = - self.in_buff.len() < needed_in_buff_size || self.out_buff.len() < needed_out_buff_size; + self.in_buff.size < needed_in_buff_size || self.out_buff.size < needed_out_buff_size; let too_large = self.oversized_duration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; if too_small || too_large { - self.in_buff = Vec::new(); - self.out_buff = Vec::new(); - self.in_buff = try_zeroed(needed_in_buff_size)?; - self.out_buff = try_zeroed(needed_out_buff_size)?; + self.in_buff = Buffer::new(); + self.out_buff = Buffer::new(); + self.in_buff = Buffer::with_size(needed_in_buff_size)?; + self.out_buff = Buffer::with_size(needed_out_buff_size)?; } Ok(()) } @@ -1178,11 +1227,13 @@ impl DState { let to_load = needed_in_size - self.in_pos; let loaded_size = to_load.min(iend - ip); if self.stage != DStage::SkipFrame { - if to_load > self.in_buff.len() - self.in_pos { + if to_load > self.in_buff.size - self.in_pos { return Err(ErrorCode::CorruptionDetected); } + self.in_buff.bytes.truncate(self.in_pos); self.in_buff - .write_at(self.in_pos, src.span(ip, loaded_size)?)?; + .bytes + .extend_from_slice(src.span(ip, loaded_size)?); } ip += loaded_size; self.in_pos += loaded_size; @@ -1191,7 +1242,7 @@ impl DState { } self.in_pos = 0; - let in_buff = core::mem::take(&mut self.in_buff); + let in_buff = core::mem::take(&mut self.in_buff.bytes); // Nothing of a skippable frame is read, or kept. let loaded = in_buff.get(..needed_in_size).unwrap_or_default(); let result = if self.stage == DStage::SkipFrame { @@ -1199,7 +1250,7 @@ impl DState { } else { self.decompress_continue_stream(loaded, ddict) }; - self.in_buff = in_buff; + self.in_buff.bytes = in_buff; result?; continue 'work; } @@ -1209,13 +1260,13 @@ impl DState { let flushed_size = to_flush_size.min(oend - op); output .dst - .write_at(op, self.out_buff.span(self.out_start, flushed_size)?)?; + .write_at(op, self.out_buff.bytes.span(self.out_start, flushed_size)?)?; op += flushed_size; self.out_start += flushed_size; if flushed_size == to_flush_size { self.stream_stage = DStreamStage::Read; - if (self.out_buff.len() as u64) < self.f_params.frame_content_size - && self.out_start + self.f_params.block_size_max > self.out_buff.len() + if (self.out_buff.size as u64) < self.f_params.frame_content_size + && self.out_start + self.f_params.block_size_max > self.out_buff.size { self.out_start = 0; self.out_end = 0; diff --git a/src/zstd/util.rs b/src/zstd/util.rs index 85b00622f155..145a299ca936 100644 --- a/src/zstd/util.rs +++ b/src/zstd/util.rs @@ -161,7 +161,7 @@ pub(crate) const fn highbit32(value: u32) -> u32 { 31 - (value | 1).leading_zeros() } -/// `len` zeros, or [`ErrorCode::MemoryAllocation`]. A large one costs nothing until it is written to. +/// `len` zeros, or [`ErrorCode::MemoryAllocation`]. An allocator that reuses memory has to clear all of it. pub(crate) fn try_zeroed(len: usize) -> Result> { bun_pod::try_zeroed_vec(len).ok_or(ErrorCode::MemoryAllocation) } From 6196cf2035e1114ab5e9082fe7daa7406c82f5cd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:00:34 +0000 Subject: [PATCH 061/199] zstd: a skippable frame that goes on past the input has an extent too --- src/zstd/decompress/mod.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 7c123e5fd52a..dc432f2f8c15 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -215,8 +215,8 @@ pub fn get_frame_content_size(src: &[u8]) -> u64 { } } -/// `readSkippableFrameSize` -fn read_skippable_frame_size(src: &[u8]) -> Result { +/// The size of the skippable frame whose header is at the start of `src`. +fn skippable_frame_size(src: &[u8]) -> Result { if src.len() < ZSTD_SKIPPABLEHEADERSIZE { return Err(ErrorCode::SrcSizeWrong); } @@ -227,7 +227,12 @@ fn read_skippable_frame_size(src: &[u8]) -> Result { { return Err(ErrorCode::FrameParameterUnsupported); } - let skippable_size = ZSTD_SKIPPABLEHEADERSIZE + size_u32 as usize; + Ok(ZSTD_SKIPPABLEHEADERSIZE + size_u32 as usize) +} + +/// `readSkippableFrameSize` +fn read_skippable_frame_size(src: &[u8]) -> Result { + let skippable_size = skippable_frame_size(src)?; if skippable_size > src.len() { return Err(ErrorCode::SrcSizeWrong); } @@ -245,7 +250,12 @@ pub enum FrameExtent { /// How far the frame at the start of `src` goes, for when where it ends is not known in advance. pub fn frame_extent(src: &[u8]) -> Result { if src.len() >= ZSTD_SKIPPABLEHEADERSIZE && is_skippable_magic(read_le32(src, 0)?) { - return read_skippable_frame_size(src).map(FrameExtent::Whole); + let size = skippable_frame_size(src)?; + return Ok(if size <= src.len() { + FrameExtent::Whole(size) + } else { + FrameExtent::AtLeast(size) + }); } let header = match get_frame_header(src)? { HeaderStatus::Complete(header) => header, From da99d074bf1f0962d977c4c3d86564f1f09a6222 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:11:44 +0000 Subject: [PATCH 062/199] zstd: far matches are prefetched while a batch of sequences is decoded --- src/zstd/decompress/block.rs | 215 ++++++++++++++++++++++++----------- 1 file changed, 148 insertions(+), 67 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index cc33bff2a779..484eb4fda40c 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -5,6 +5,8 @@ use super::seq_table::{ Cell, LL_BASE, ML_BASE, OF_BASE, OF_BITS, OffsetTable, SeqTable, SeqTableSources, SeqTables, TableSource, }; +use core::hint::{Locality, prefetch_read}; + use crate::common::bitstream::BitDStream; use crate::common::entropy::read_ncount; use crate::common::{ @@ -559,9 +561,9 @@ impl BlockDecoder { if nb_seq > 0 { let mut seq_state = SeqState::new(sequences, tables, rep)?; if use_prefetch_decoder { - exec.run_long(&mut seq_state, nb_seq)?; + exec.run::(&mut seq_state, nb_seq)?; } else { - exec.run(&mut seq_state, nb_seq)?; + exec.run::(&mut seq_state, nb_seq)?; } *rep = seq_state.prev_offset.map(|offset| offset as u32); } @@ -651,6 +653,36 @@ struct RawSequence { match_length: u32, } +/// Where matches are, for `ZSTD_prefetchMatch`. +struct History { + prefix_start: *const u8, + dict_end: *const u8, + /// How long the prefix is. + position: usize, +} + +impl History { + /// Of the match of the next sequence, if its offset is not one of the last. + #[inline(always)] + fn prefetch(&mut self, lit_length: u32, match_length: u32, offset_value: u32) { + self.position += lit_length as usize; + let offset = offset_value.saturating_sub(3) as usize; + let base = if offset > self.position { + self.dict_end + } else { + self.prefix_start + }; + let from = if offset == 0 { + self.dict_end + } else { + base.wrapping_add(self.position).wrapping_sub(offset) + }; + prefetch_read(from, Locality::L1); + prefetch_read(from.wrapping_add(64), Locality::L1); + self.position += (match_length + MINMATCH) as usize; + } +} + /// `seqState_t` #[derive(Clone, Copy)] struct SeqState<'a> { @@ -685,8 +717,13 @@ impl<'a> SeqState<'a> { } /// Decodes sequences for as long as the start of the stream is far. Returns how many that is. + /// With `PREFETCH`, `history` follows the output, and what matches refer to is on its way to the cache. #[inline(never)] - fn decode_batch(&mut self, out: &mut [RawSequence]) -> Result { + fn decode_batch( + &mut self, + out: &mut [RawSequence], + history: &mut History, + ) -> Result { /// The 64 bits that end at `position`, turned so that the next one to read is the highest. Those that are read /// then go to the other end. #[inline(always)] @@ -749,6 +786,9 @@ impl<'a> SeqState<'a> { lit_length, match_length, }; + if PREFETCH { + history.prefetch(lit_length, match_length, offset_value); + } let ll_nb_bits = u32::from(ll_cell.nb_bits); let ml_nb_bits = u32::from(ml_cell.nb_bits); @@ -886,6 +926,33 @@ fn copy_match_exact(buf: &mut [u8], src: usize, dst: usize, len: usize) -> Resul Ok(()) } +/// Makes the offset value of `sequences[index]`, which refers to one of the last offsets, what it refers to, or 0 if that +/// is not valid, and returns it. The last offsets are those of the 3 sequences before, which are rewritten so that the +/// same holds for the next sequence. +#[inline(always)] +fn resolve_offset(sequences: &mut [RawSequence], index: usize) -> u32 { + let Some([last2, last1, last0, this]) = index + .checked_sub(3) + .and_then(|first| sequences.get_mut(first..)) + .and_then(<[RawSequence]>::first_chunk_mut) + else { + return 0; + }; + let rep0 = last0.offset_value; + let rep1 = last1.offset_value; + let rep2 = last2.offset_value; + let reps = match this.offset_value - 1 + u32::from(this.lit_length == 0) { + 0 => [rep0, rep1, rep2], + 1 => [rep1, rep0, rep2], + 2 => [rep2, rep0, rep1], + // An offset of 0 is not valid. + _ if rep0 == 1 + 3 => [0, rep0, rep1], + _ => [rep0 - 1, rep0, rep1], + }; + [this.offset_value, last0.offset_value, last1.offset_value] = reps; + this.offset_value +} + /// Turns sequences into output. struct Executor<'buf, 'a> { /// Ends no further than `oend`. @@ -1163,26 +1230,7 @@ impl<'a> Executor<'_, 'a> { let mut offset_value = raw.offset_value; if offset_value < 16 + 3 { if offset_value <= 3 { - let Some([last2, last1, last0, this]) = index - .checked_sub(3) - .and_then(|first| sequences.get_mut(first..)) - .and_then(<[RawSequence]>::first_chunk_mut) - else { - break; - }; - let rep0 = last0.offset_value; - let rep1 = last1.offset_value; - let rep2 = last2.offset_value; - let reps = match offset_value - 1 + u32::from(lit_length == 0) { - 0 => [rep0, rep1, rep2], - 1 => [rep1, rep0, rep2], - 2 => [rep2, rep0, rep1], - // An offset of 0 is not valid. - _ if rep0 == 1 + 3 => [0, rep0, rep1], - _ => [rep0 - 1, rep0, rep1], - }; - [this.offset_value, last0.offset_value, last1.offset_value] = reps; - [offset_value, ..] = reps; + offset_value = resolve_offset(sequences, index); } if offset_value < 16 + 3 { break; @@ -1229,40 +1277,55 @@ impl<'a> Executor<'_, 'a> { index } - /// `ZSTD_decompressSequences_body` and `ZSTD_decompressSequences_bodySplitLitBuffer` - fn run(&mut self, seq_state: &mut SeqState<'_>, nb_seq: usize) -> Result<()> { + /// Executes `sequences[index]` and those after it that are not for [`Self::exec_batch`]. Returns the index of the + /// first that is. + #[inline(never)] + fn exec_others(&mut self, sequences: &mut [RawSequence], first: usize) -> Result { + let mut exec = self.fork(); + let mut index = first; + while let Some(&raw) = sequences.get(index) { + let mut offset_value = raw.offset_value; + if (1..=3).contains(&offset_value) { + offset_value = resolve_offset(sequences, index); + } + if index != first + && raw.lit_length <= 32 + && raw.match_length <= 64 - MINMATCH + && offset_value >= 16 + 3 + { + break; + } + exec.exec_sequence::(Sequence { + lit_length: raw.lit_length, + match_length: raw.match_length + MINMATCH, + offset: (offset_value as usize).wrapping_sub(3), + })?; + index += 1; + } + let exec = (exec.op, exec.lits, exec.split); + self.join(exec); + Ok(index) + } + + /// `ZSTD_decompressSequences_body`, `ZSTD_decompressSequences_bodySplitLitBuffer` and, with `LONG`, + /// `ZSTD_decompressSequencesLong_body`. + fn run(&mut self, seq_state: &mut SeqState<'_>, nb_seq: usize) -> Result<()> { const BATCH: usize = 128; // After 3 that only stand for the last offsets. let mut batch = [RawSequence::default(); 3 + BATCH]; + // The C code then decodes 8 sequences ahead, so it sees whether the stream ends where it should before it + // executes the last ones. + let mut ahead = [Sequence::default(); 8]; + let ahead = ahead.sub_mut(..if LONG { nb_seq.min(8) } else { 0 })?; + let mut remaining = nb_seq; - while remaining > 0 { + while remaining > ahead.len() { if !self.split { - let room = batch.span_mut(3, BATCH.min(remaining - 1))?; - let count = seq_state.decode_batch(room)?; + let room = batch.span_mut(3, BATCH.min(remaining - ahead.len().max(1)))?; + let count = seq_state.decode_batch::(room, &mut self.history())?; if count > 0 { remaining -= count; - let sequences = batch.sub_mut(..3 + count)?; - for (last, offset) in - sequences.iter_mut().zip(seq_state.prev_offset.iter().rev()) - { - last.offset_value = *offset as u32 + 3; - } - let mut index = 3; - loop { - index = self.exec_batch(sequences, index); - let Some(raw) = sequences.get(index) else { - break; - }; - self.exec_sequence::(Sequence { - lit_length: raw.lit_length, - match_length: raw.match_length + MINMATCH, - offset: (raw.offset_value as usize).wrapping_sub(3), - })?; - index += 1; - } - for (last, offset) in sequences.iter().rev().zip(&mut seq_state.prev_offset) { - *offset = last.offset_value as usize - 3; - } + self.exec_all(batch.sub_mut(..3 + count)?, &mut seq_state.prev_offset)?; continue; } } @@ -1270,32 +1333,50 @@ impl<'a> Executor<'_, 'a> { remaining -= 1; self.exec_sequence_maybe_split(sequence)?; } + for sequence in &mut *ahead { + *sequence = seq_state.decode_sequence(remaining == 1); + remaining -= 1; + } if !seq_state.bit_d.is_at_end() { return Err(ErrorCode::CorruptionDetected); } + for &sequence in &*ahead { + self.exec_sequence_maybe_split(sequence)?; + } Ok(()) } - /// `ZSTD_decompressSequencesLong_body`: sequences are decoded 8 ahead of the one that is executed. - fn run_long(&mut self, seq_state: &mut SeqState<'_>, nb_seq: usize) -> Result<()> { - const STORED_SEQS: usize = 8; - const STORED_SEQS_MASK: usize = STORED_SEQS - 1; - let mut sequences = [Sequence::default(); STORED_SEQS]; - let seq_advance = nb_seq.min(STORED_SEQS); - - for (seq_nb, stored) in sequences.iter_mut().enumerate().take(seq_advance) { - *stored = seq_state.decode_sequence(seq_nb == nb_seq - 1); + fn history(&mut self) -> History { + let buf = self.out.bytes().as_ptr(); + History { + prefix_start: buf.wrapping_add(self.prefix_start), + dict_end: match self.ext_dict { + ExtDict::Dict(dict) => dict.as_ptr_range().end, + ExtDict::Within { end, .. } => buf.wrapping_add(end), + }, + position: self.op - self.prefix_start, } - for seq_nb in seq_advance..nb_seq { - let sequence = seq_state.decode_sequence(seq_nb == nb_seq - 1); - let stored = sequences.at_mut(seq_nb & STORED_SEQS_MASK)?; - self.exec_sequence_maybe_split(core::mem::replace(stored, sequence))?; + } + + /// Executes `sequences` but the first 3, which are there to stand for the last offsets. + fn exec_all( + &mut self, + sequences: &mut [RawSequence], + prev_offset: &mut [usize; ZSTD_REP_NUM], + ) -> Result<()> { + for (last, offset) in sequences.iter_mut().zip(prev_offset.iter().rev()) { + last.offset_value = *offset as u32 + 3; } - if !seq_state.bit_d.is_at_end() { - return Err(ErrorCode::CorruptionDetected); + let mut index = 3; + loop { + index = self.exec_batch(sequences, index); + if index >= sequences.len() { + break; + } + index = self.exec_others(sequences, index)?; } - for seq_nb in nb_seq - seq_advance..nb_seq { - self.exec_sequence_maybe_split(sequences.at(seq_nb & STORED_SEQS_MASK)?)?; + for (last, offset) in sequences.iter().rev().zip(prev_offset) { + *offset = last.offset_value as usize - 3; } Ok(()) } From edc24dacbadcbc526652fd15972b177aae87b7a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:16:42 +0000 Subject: [PATCH 063/199] zstd: the decoder adapts to long matches, close matches, lengths with additional bits and few literals --- src/zstd/decompress/block.rs | 101 ++++++++++-------- src/zstd/decompress/huf.rs | 169 ++++++++++++++++++++++++------- src/zstd/decompress/seq_table.rs | 31 +++++- 3 files changed, 218 insertions(+), 83 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 484eb4fda40c..baf2d326101f 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -292,7 +292,10 @@ impl BlockDecoder { let huf_result = if lit_enc_type == SymbolEncodingType::Repeat { let table = match (huf_source, dict_entropy) { (TableSource::Dict, Some(dict)) => &dict.huf_table, - _ => &entropy.huf_table, + _ => { + entropy.huf_table.prepare(lit_size)?; + &entropy.huf_table + } }; if single_stream { huf::decompress_1x_using_dtable(dst, compressed, table) @@ -718,8 +721,10 @@ impl<'a> SeqState<'a> { /// Decodes sequences for as long as the start of the stream is far. Returns how many that is. /// With `PREFETCH`, `history` follows the output, and what matches refer to is on its way to the cache. + /// + /// `MORE_BITS` is for when it cannot be predicted which lengths come with additional bits. #[inline(never)] - fn decode_batch( + fn decode_batch( &mut self, out: &mut [RawSequence], history: &mut History, @@ -771,12 +776,14 @@ impl<'a> SeqState<'a> { bits = bits.rotate_left(of_bits); position -= of_bits; let offset_value = first_value + (bits as u32 & (first_value - 1)); - if ml_bits + ll_bits != 0 { + if MORE_BITS || ml_bits + ll_bits != 0 { bits = bits.rotate_left(ml_bits); position -= ml_bits; match_length = ml_cell.base_value() + (bits as u32 & ((1 << ml_bits) - 1)); - // What follows has to fit in 57 bits with the 26 of the next states. - bits = load(src, position + (FAR as u32 - 8) * 8).unwrap_or(0); + // All has to fit in 57 bits with the 26 of the next states. + if of_bits + ml_bits + ll_bits >= 31 { + bits = load(src, position + (FAR as u32 - 8) * 8).unwrap_or(0); + } bits = bits.rotate_left(ll_bits); position -= ll_bits; lit_length = ll_cell.base_value() + (bits as u32 & ((1 << ll_bits) - 1)); @@ -1104,12 +1111,8 @@ impl<'a> Executor<'_, 'a> { room.get_mut(16..rounded)? .copy_from_slice(from.get(16..rounded)?); } else { - let mut done = 16; - while done < match_length { - let to = o_lit_end + done; - copy_within::<16>(self.out.bytes(), to - offset, to).ok()?; - done += 16; - } + let buf = self.out.bytes(); + copy_match_exact(buf, o_lit_end - offset, o_lit_end, match_length).ok()?; } } } else { @@ -1122,38 +1125,30 @@ impl<'a> Executor<'_, 'a> { Some(()) } - /// `ZSTD_overlapCopy8` and `ZSTD_wildcopy(ZSTD_overlap_src_before_dst)`: a match less than 16 bytes back. + /// `ZSTD_overlapCopy8` and `ZSTD_wildcopy(ZSTD_overlap_src_before_dst)`: a match less than 16 bytes back, which is + /// not 0. Up to 16 bytes are written past its end. #[inline(always)] - fn copy_close_match( - &mut self, - mut op: usize, - offset: usize, - match_length: usize, - ) -> Result<()> { - let mut match_pos = op - offset; - // After 8 bytes, the source is at least 8 bytes behind. - if offset < 8 { - const DEC32_TABLE: [usize; 8] = [0, 1, 2, 1, 4, 4, 4, 4]; - const DEC64_TABLE: [usize; 8] = [8, 8, 8, 7, 8, 9, 10, 11]; - for i in 0..4 { - let byte = self.out.bytes().at(match_pos + i)?; - self.out.bytes().put(op + i, byte)?; - } - match_pos += DEC32_TABLE.at(offset)?; - copy_within::<4>(self.out.bytes(), match_pos, op + 4)?; - match_pos += 8; - match_pos -= DEC64_TABLE.at(offset)?; - } else { - copy_within::<8>(self.out.bytes(), match_pos, op)?; - match_pos += 8; + fn copy_close_match(&mut self, op: usize, offset: usize, match_length: usize) -> Result<()> { + /// The largest multiple of an offset that is at most 16. + const STEPS: [u8; 16] = [16, 16, 16, 15, 16, 15, 12, 14, 16, 9, 10, 11, 12, 13, 14, 15]; + let buf = self.out.bytes(); + // The `offset` bytes before the match, again and again. + let mut pattern = u128::from_le_bytes(*buf.array_at(op - offset)?); + let mut filled = offset as u32 * 8; + pattern &= !(u128::MAX << filled); + while filled < 128 { + pattern |= pattern << filled; + filled *= 2; } - op += 8; - let mut done = 8usize; - while done < match_length { - copy_within::<8>(self.out.bytes(), match_pos, op)?; - match_pos += 8; - op += 8; - done += 8; + let pattern = pattern.to_le_bytes(); + let step = usize::from(STEPS.at(offset)?); + let room = buf.span_mut(op, match_length + 16)?; + let mut done = 0; + while let Some(to) = room.get_mut(done..done + 16) + && done < match_length + { + to.copy_from_slice(&pattern); + done += step; } Ok(()) } @@ -1318,11 +1313,33 @@ impl<'a> Executor<'_, 'a> { let mut ahead = [Sequence::default(); 8]; let ahead = ahead.sub_mut(..if LONG { nb_seq.min(8) } else { 0 })?; + // A branch that goes wrong costs about 10 times what it saves. + let tables = seq_state.tables; + let more_bits = tables.ll.with_more_bits + tables.ml.with_more_bits > 512 / 10; + let mut remaining = nb_seq; + // Long copies go on while the next sequences are decoded, if that is what comes next. + if tables.ml.with_more_bits > 512 / 2 && !self.split { + let mut exec = self.fork(); + let mut state = *seq_state; + while remaining > ahead.len() { + let sequence = state.decode_sequence(remaining == 1); + remaining -= 1; + exec.exec_sequence::(sequence)?; + } + let exec = (exec.op, exec.lits, exec.split); + self.join(exec); + *seq_state = state; + } while remaining > ahead.len() { if !self.split { let room = batch.span_mut(3, BATCH.min(remaining - ahead.len().max(1)))?; - let count = seq_state.decode_batch::(room, &mut self.history())?; + let history = &mut self.history(); + let count = if more_bits { + seq_state.decode_batch::(room, history)? + } else { + seq_state.decode_batch::(room, history)? + }; if count > 0 { remaining -= count; self.exec_all(batch.sub_mut(..3 + count)?, &mut seq_state.prev_offset)?; diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index ac89b2851ddb..4675bc0c0eb8 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -25,6 +25,65 @@ pub(super) struct Code { pub(super) table_log: u32, } +/// The loop that decodes four streams at once takes a table of this log, or one more. +const FAST_LOG: u32 = 11; +/// From how many literals on that loop makes up for the time that it takes to fill such a table. +const FAST_MIN_LITERALS: usize = 768; +/// From how many literals on the larger table is considered. +const LARGE_MIN_LITERALS: usize = 1 << 14; + +impl Code { + /// How many symbols a lookup of `log` bits yields, in 65536ths, if symbols are as frequent as their codes are short. + fn symbols_per_lookup(&self, log: u32) -> u64 { + let table_log = self.table_log as usize; + // For each number of bits, out of `1 << table_log`, how many symbols have a code no longer than that. + let mut one = [0u64; NB_RANKS + 1]; + let mut share = [0u64; NB_RANKS + 1]; + let mut total = 0; + for nb_bits in 1..=log as usize { + let weight = (table_log + 1).wrapping_sub(nb_bits); + let count = self.rank_stats.get(weight).copied().unwrap_or(0); + let of_these = if weight == 0 { 0 } else { u64::from(count) << (weight - 1) }; + total += of_these; + if let (Some(share), Some(one)) = (share.get_mut(nb_bits), one.get_mut(nb_bits)) { + *share = of_these; + *one = total; + } + } + // The same for two symbols and for three, out of that squared and cubed. + let after = |fits: &[u64; NB_RANKS + 1], room: usize| -> u64 { + (1..room) + .map(|nb_bits| { + share.get(nb_bits).copied().unwrap_or(0) + * fits.get(room - nb_bits).copied().unwrap_or(0) + }) + .sum() + }; + let mut two = [0u64; NB_RANKS + 1]; + for (room, two) in two.iter_mut().enumerate() { + *two = after(&one, room); + } + let three = after(&two, log as usize); + let two = two.get(log as usize).copied().unwrap_or(0); + (1 << 16) + ((two << 16) >> (2 * table_log)) + ((three << 16) >> (3 * table_log)) + } + + /// How many bits to look up at once to decode `nb_literals`. + fn best_log(&self, nb_literals: usize) -> u32 { + if nb_literals < FAST_MIN_LITERALS { + return self.table_log; + } + // The larger table takes twice the time to fill, half of the cache, and a refill for 4 lookups and not 5. + if self.table_log > FAST_LOG + || nb_literals >= LARGE_MIN_LITERALS + && self.symbols_per_lookup(FAST_LOG + 1) * 4 > self.symbols_per_lookup(FAST_LOG) * 5 + { + return FAST_LOG + 1; + } + FAST_LOG + } +} + /// What a lookup yields: up to 3 symbols, then a byte with how many bits they take in its low 6 bits and how many they /// are above. type Entry = [u8; 4]; @@ -62,16 +121,25 @@ impl HufDTable { } } - /// Reads the description of a table at the start of `src`, and returns its size. - fn read(&mut self, src: &[u8], table_type: TableType) -> Result { + /// Reads the description of a table at the start of `src`, and returns its size. The table is to decode + /// `nb_literals` with. + fn read(&mut self, src: &[u8], table_type: TableType, nb_literals: usize) -> Result { let stats = huf_read_stats(&mut self.code.weights, &mut self.code.rank_stats, src)?; self.code.nb_symbols = stats.nb_symbols; self.code.table_log = stats.table_log; self.table_type = table_type; - self.build(stats.table_log.max(11))?; + self.build(self.code.best_log(nb_literals))?; Ok(stats.size) } + /// Makes the table fit to decode `nb_literals` with. + pub(crate) fn prepare(&mut self, nb_literals: usize) -> Result<()> { + if self.log < FAST_LOG && nb_literals >= FAST_MIN_LITERALS { + self.build(self.code.table_log.max(FAST_LOG))?; + } + Ok(()) + } + /// Fills `table` for lookups of `log` bits, which is at least as many as the longest code has. fn build(&mut self, log: u32) -> Result<()> { let Code { @@ -222,7 +290,10 @@ impl HufDTable { /// Four streams after their sizes. Whether each is exactly its share of `dst.len()` symbols. fn decode_4x(&self, dst: &mut [u8], src: &[u8]) -> bool { - let Some(mut streams) = Streams::new(dst, src) else { + let Some(parts) = Part::split(dst, src) else { + return false; + }; + let Some(mut streams) = Streams::new(parts) else { return false; }; match self.log { @@ -232,9 +303,7 @@ impl HufDTable { } let mut exact = true; for stream in streams.0 { - let Ok(mut bit_d) = - BitDStream::resume(stream.src, stream.ip as usize, stream.bits.trailing_zeros()) - else { + let Ok(mut bit_d) = stream.bit_d() else { return false; }; let out = stream.out.get_mut(stream.op as usize..).unwrap_or_default(); @@ -244,12 +313,34 @@ impl HufDTable { } } -/// A stream of at least 8 bytes, and where its symbols go. +/// A stream, and where its symbols go. +struct Part<'a> { + src: &'a [u8], + out: &'a mut [u8], +} + +impl<'a> Part<'a> { + /// `None` if the sizes at the start of `src` are not those of four streams. + fn split(dst: &'a mut [u8], src: &'a [u8]) -> Option<[Self; 4]> { + let (sizes, src) = src.split_first_chunk::<6>()?; + let (src0, src) = src.split_at_checked(usize::from(read_le16(sizes, 0).ok()?))?; + let (src1, src) = src.split_at_checked(usize::from(read_le16(sizes, 2).ok()?))?; + let (src2, src3) = src.split_at_checked(usize::from(read_le16(sizes, 4).ok()?))?; + let segment = dst.len().div_ceil(4); + let (dst0, dst) = dst.split_at_mut_checked(segment)?; + let (dst1, dst) = dst.split_at_mut_checked(segment)?; + let (dst2, dst3) = dst.split_at_mut_checked(segment)?; + Some([(src0, dst0), (src1, dst1), (src2, dst2), (src3, dst3)].map(|(src, out)| Self { src, out })) + } +} + +/// A stream that is being decoded. struct Stream<'a> { src: &'a [u8], /// Where in `src` `bits` was read. ip: u32, - /// What is left to read of 64 bits, from the highest bit on, then a bit set. + /// What is left to read of 64 bits, from the highest bit on, then a bit set. 0 if the stream is shorter than that: + /// nothing is read of it yet. bits: u64, out: &'a mut [u8], /// How much of `out` is written. @@ -257,40 +348,43 @@ struct Stream<'a> { } impl<'a> Stream<'a> { - fn new(src: &'a [u8], out: &'a mut [u8]) -> Option { - let container = u64::from_le_bytes(*src.last_chunk()?); - // The highest bit set marks the end of the stream. - if container >> 56 == 0 { - return None; - } - Some(Self { + fn new(Part { src, out }: Part<'a>) -> Option { + let mut stream = Self { src, - ip: src.len() as u32 - 8, - bits: (container | 1) << (container.leading_zeros() + 1), + ip: 0, + bits: 0, out, op: 0, - }) + }; + if let Some(container) = src.last_chunk() { + let container = u64::from_le_bytes(*container); + // The highest bit set marks the end of the stream. + if container >> 56 == 0 { + return None; + } + stream.ip = src.len() as u32 - 8; + stream.bits = (container | 1) << (container.leading_zeros() + 1); + } + Some(stream) + } + + fn bit_d(&self) -> Result> { + if self.bits == 0 { + return BitDStream::new(self.src); + } + BitDStream::resume(self.src, self.ip as usize, self.bits.trailing_zeros()) } } struct Streams<'a>([Stream<'a>; 4]); impl<'a> Streams<'a> { - /// `None` if there are no four streams of 8 bytes or more. - fn new(dst: &'a mut [u8], src: &'a [u8]) -> Option { - let (sizes, src) = src.split_first_chunk::<6>()?; - let (src0, src) = src.split_at_checked(usize::from(read_le16(sizes, 0).ok()?))?; - let (src1, src) = src.split_at_checked(usize::from(read_le16(sizes, 2).ok()?))?; - let (src2, src3) = src.split_at_checked(usize::from(read_le16(sizes, 4).ok()?))?; - let segment = dst.len().div_ceil(4); - let (dst0, dst) = dst.split_at_mut_checked(segment)?; - let (dst1, dst) = dst.split_at_mut_checked(segment)?; - let (dst2, dst3) = dst.split_at_mut_checked(segment)?; + fn new([part0, part1, part2, part3]: [Part<'a>; 4]) -> Option { Some(Self([ - Stream::new(src0, dst0)?, - Stream::new(src1, dst1)?, - Stream::new(src2, dst2)?, - Stream::new(src3, dst3)?, + Stream::new(part0)?, + Stream::new(part1)?, + Stream::new(part2)?, + Stream::new(part3)?, ])) } @@ -340,6 +434,9 @@ impl<'a> Streams<'a> { true } + if self.0.iter().any(|stream| stream.bits == 0) { + return; + } let [s0, s1, s2, s3] = &mut self.0; let (mut bits0, mut bits1, mut bits2, mut bits3) = (s0.bits, s1.bits, s2.bits, s3.bits); let (mut ip0, mut ip1, mut ip2, mut ip3) = (s0.ip, s1.ip, s2.ip, s3.ip); @@ -392,7 +489,7 @@ impl<'a> Streams<'a> { /// `HUF_readDTableX2_wksp`. Returns the size of the table's description in `src`. pub(crate) fn read_dtable_x2(dtable: &mut HufDTable, src: &[u8]) -> Result { - dtable.read(src, TableType::X2) + dtable.read(src, TableType::X2, usize::MAX) } /// `HUF_decompress1X_usingDTable` @@ -409,7 +506,7 @@ pub(crate) fn decompress_1x_using_dtable( /// `HUF_decompress1X1_DCtx_wksp` pub(crate) fn decompress_1x1(dtable: &mut HufDTable, dst: &mut [u8], src: &[u8]) -> Result { - let h_size = dtable.read(src, TableType::X1)?; + let h_size = dtable.read(src, TableType::X1, dst.len())?; if h_size >= src.len() { return Err(ErrorCode::SrcSizeWrong); } @@ -445,7 +542,7 @@ pub(crate) fn decompress_4x_huf_only( } else { TableType::X1 }; - let h_size = dtable.read(src, table_type)?; + let h_size = dtable.read(src, table_type, dst.len())?; if h_size >= src.len() { return Err(ErrorCode::SrcSizeWrong); } diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs index c3e672e6ad25..3ca90fbcbc1b 100644 --- a/src/zstd/decompress/seq_table.rs +++ b/src/zstd/decompress/seq_table.rs @@ -240,6 +240,8 @@ impl Spread { #[derive(Clone)] pub(crate) struct SeqTable { pub(crate) table_log: u32, + /// How many of 512 symbols are expected to come with additional bits. + pub(crate) with_more_bits: u32, pub(crate) table: [C; N], } @@ -250,15 +252,25 @@ impl SeqTable { pub(crate) const fn new() -> Self { Self { table_log: 0, + with_more_bits: 0, table: [C::EMPTY; N], } } - const fn predefined(table_log: u32, symbols: [C; M]) -> Self { + /// `with_more_bits` of the `symbols` have additional bits. + const fn predefined( + table_log: u32, + with_more_bits: u32, + symbols: [C; M], + ) -> Self { let mut table = [C::EMPTY; N]; let (head, _) = table.split_at_mut(M); head.copy_from_slice(&symbols); - Self { table_log, table } + Self { + table_log, + with_more_bits: with_more_bits << (9 - table_log), + table, + } } #[inline(always)] @@ -282,6 +294,7 @@ impl SeqTable { fn copy_from(&mut self, other: &Self) { self.table_log = other.table_log; + self.with_more_bits = other.with_more_bits; let symbols = other.symbols(); if let Some(to) = self.table.get_mut(..symbols.len()) { to.copy_from_slice(symbols); @@ -291,6 +304,7 @@ impl SeqTable { /// `ZSTD_buildSeqTable_rle` pub(crate) fn build_rle(&mut self, base_value: u32, nb_additional_bits: u8) { self.table_log = 0; + self.with_more_bits = u32::from(nb_additional_bits != 0) << 9; if let Some(cell) = self.table.first_mut() { *cell = C::pack(SeqSymbol { next_state: 0, @@ -312,6 +326,13 @@ impl SeqTable { let table_size = 1usize << table_log; let table_decode = self.table.sub_mut(..table_size)?; self.table_log = table_log; + self.with_more_bits = normalized_counter + .iter() + .zip(nb_additional_bits) + .filter(|&(_, &nb_bits)| nb_bits != 0) + .map(|(&count, _)| u32::from(count.unsigned_abs())) + .sum::() + << (9 - table_log.min(9)); let Spread { symbols, mut symbol_next, @@ -407,9 +428,9 @@ pub(crate) struct SeqTables { } static DEFAULT_TABLES: SeqTables = SeqTables { - ll: SeqTable::predefined(LL_DEFAULT_NORM_LOG, LL_DEFAULT), - of: SeqTable::predefined(OF_DEFAULT_NORM_LOG, OF_DEFAULT), - ml: SeqTable::predefined(ML_DEFAULT_NORM_LOG, ML_DEFAULT), + ll: SeqTable::predefined(LL_DEFAULT_NORM_LOG, 32, LL_DEFAULT), + of: SeqTable::predefined(OF_DEFAULT_NORM_LOG, 31, OF_DEFAULT), + ml: SeqTable::predefined(ML_DEFAULT_NORM_LOG, 21, ML_DEFAULT), }; /// `LLTptr`, `OFTptr` and `MLTptr` From 1bcb9f739303a9dbce396222bf9d2b89536cb689 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:18:02 +0000 Subject: [PATCH 064/199] zstd: four Huffman streams are decoded at a time until the last one is done --- src/zstd/decompress/block.rs | 4 +- src/zstd/decompress/huf.rs | 109 +++++++++++++++++++++++++---------- 2 files changed, 83 insertions(+), 30 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index baf2d326101f..62ba1eccc9cd 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -1130,7 +1130,9 @@ impl<'a> Executor<'_, 'a> { #[inline(always)] fn copy_close_match(&mut self, op: usize, offset: usize, match_length: usize) -> Result<()> { /// The largest multiple of an offset that is at most 16. - const STEPS: [u8; 16] = [16, 16, 16, 15, 16, 15, 12, 14, 16, 9, 10, 11, 12, 13, 14, 15]; + const STEPS: [u8; 16] = [ + 16, 16, 16, 15, 16, 15, 12, 14, 16, 9, 10, 11, 12, 13, 14, 15, + ]; let buf = self.out.bytes(); // The `offset` bytes before the match, again and again. let mut pattern = u128::from_le_bytes(*buf.array_at(op - offset)?); diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index 4675bc0c0eb8..d352a4538f90 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -43,7 +43,11 @@ impl Code { for nb_bits in 1..=log as usize { let weight = (table_log + 1).wrapping_sub(nb_bits); let count = self.rank_stats.get(weight).copied().unwrap_or(0); - let of_these = if weight == 0 { 0 } else { u64::from(count) << (weight - 1) }; + let of_these = if weight == 0 { + 0 + } else { + u64::from(count) << (weight - 1) + }; total += of_these; if let (Some(share), Some(one)) = (share.get_mut(nb_bits), one.get_mut(nb_bits)) { *share = of_these; @@ -296,10 +300,9 @@ impl HufDTable { let Some(mut streams) = Streams::new(parts) else { return false; }; - match self.log { - 11 => streams.decode_most::<11, 5>(&self.table), - 12 => streams.decode_most::<12, 4>(&self.table), - _ => {} + let mut no_output = [[0u8; 512]; 4]; + if self.log >= FAST_LOG { + self.decode_most(&mut streams, &mut no_output); } let mut exact = true; for stream in streams.0 { @@ -311,6 +314,46 @@ impl HufDTable { } exact } + + /// Decodes each stream until it is close to its start, or its output to its end. + fn decode_most<'a>(&self, streams: &mut Streams<'a>, no_output: &'a mut [[u8; 512]; 4]) { + // Some streams get there before others, which go on four at a time all the same: with these. + static NO_STREAM: [u8; 256] = [0x55; 256]; + let mut decoding = Streams(no_output.each_mut().map(|out| Stream { + src: &NO_STREAM, + ip: 0, + bits: 0, + out, + op: 0, + })); + for (slot, stream) in decoding.0.iter_mut().zip(&mut streams.0) { + core::mem::swap(slot, stream); + } + // `streams[i]` is in `decoding`. + let mut left = [true; 4]; + loop { + for ((stream, done), left) in decoding.0.iter_mut().zip(&mut streams.0).zip(&mut left) { + if stream.has_most_left() { + continue; + } + if *left { + *left = false; + core::mem::swap(stream, done); + } + stream.ip = NO_STREAM.len() as u32 - 8; + stream.bits = 1; + stream.op = 0; + stream.has_most_left(); + } + if left == [false; 4] { + break; + } + match self.log { + 11 => decoding.decode_most::<11, 5>(&self.table), + _ => decoding.decode_most::<12, 4>(&self.table), + } + } + } } /// A stream, and where its symbols go. @@ -330,10 +373,33 @@ impl<'a> Part<'a> { let (dst0, dst) = dst.split_at_mut_checked(segment)?; let (dst1, dst) = dst.split_at_mut_checked(segment)?; let (dst2, dst3) = dst.split_at_mut_checked(segment)?; - Some([(src0, dst0), (src1, dst1), (src2, dst2), (src3, dst3)].map(|(src, out)| Self { src, out })) + Some( + [(src0, dst0), (src1, dst1), (src2, dst2), (src3, dst3)] + .map(|(src, out)| Self { src, out }), + ) } } +/// What two rounds of lookups between reloads can write, as far as the compiler can tell. +const WINDOW: usize = 32; + +/// Reads the 64 bits from where a stream is. Whether there are as many. +#[inline(always)] +fn reload(src: &[u8], ip: &mut u32, bits: &mut u64) -> bool { + let consumed = bits.trailing_zeros(); + // Wraps around before the start. + let from = ip.wrapping_sub(consumed >> 3); + let Some(Ok(container)) = src + .get(from as usize..from as usize + 8) + .map(<[u8; 8]>::try_from) + else { + return false; + }; + *ip = from; + *bits = (u64::from_le_bytes(container) | 1) << (consumed & 7); + true +} + /// A stream that is being decoded. struct Stream<'a> { src: &'a [u8], @@ -368,6 +434,14 @@ impl<'a> Stream<'a> { Some(stream) } + /// Whether the stream is far enough from its start and its output from its end for `Streams::decode_most`, for + /// which it then is ready. + fn has_most_left(&mut self) -> bool { + self.bits != 0 + && self.op as usize + WINDOW <= self.out.len() + && reload(self.src, &mut self.ip, &mut self.bits) + } + fn bit_d(&self) -> Result> { if self.bits == 0 { return BitDStream::new(self.src); @@ -391,9 +465,6 @@ impl<'a> Streams<'a> { /// Decodes for as long as no stream is close to its start and no output to its end. `LOOKUPS` of `LOG` bits are at /// most 57 bits. fn decode_most(&mut self, table: &[Entry; TABLE_SIZE]) { - /// What `LOOKUPS` lookups can write, as far as the compiler can tell. - const WINDOW: usize = 32; - #[inline(always)] fn lookup( table: &[Entry; TABLE_SIZE], @@ -417,26 +488,6 @@ impl<'a> Streams<'a> { .as_mut_array() } - /// Whether there are 8 bytes to read from where the stream is. - #[inline(always)] - fn reload(src: &[u8], ip: &mut u32, bits: &mut u64) -> bool { - let consumed = bits.trailing_zeros(); - // Wraps around before the start. - let from = ip.wrapping_sub(consumed >> 3); - let Some(Ok(container)) = src - .get(from as usize..from as usize + 8) - .map(<[u8; 8]>::try_from) - else { - return false; - }; - *ip = from; - *bits = (u64::from_le_bytes(container) | 1) << (consumed & 7); - true - } - - if self.0.iter().any(|stream| stream.bits == 0) { - return; - } let [s0, s1, s2, s3] = &mut self.0; let (mut bits0, mut bits1, mut bits2, mut bits3) = (s0.bits, s1.bits, s2.bits, s3.bits); let (mut ip0, mut ip1, mut ip2, mut ip3) = (s0.ip, s1.ip, s2.ip, s3.ip); From a8e5dae0b142f89453113d6e709cd65dc28eaecd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:36:33 +0000 Subject: [PATCH 065/199] zstd: the ends of streams are decoded from padded copies, and tables take fewer steps to build --- src/zstd/decompress/block.rs | 184 +++++++--- src/zstd/decompress/huf.rs | 602 ++++++++++++++++++++++--------- src/zstd/decompress/mod.rs | 28 +- src/zstd/decompress/seq_table.rs | 318 ++++++++++------ 4 files changed, 790 insertions(+), 342 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 62ba1eccc9cd..cbda3c5fa2b4 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -2,16 +2,17 @@ use super::huf::{self, HufDTable}; use super::seq_table::{ - Cell, LL_BASE, ML_BASE, OF_BASE, OF_BITS, OffsetTable, SeqTable, SeqTableSources, SeqTables, - TableSource, + Cell, Codes, LL_CODES, ML_CODES, OF_CODES, OffsetTable, SeqTable, SeqTableSources, SeqTables, + TableSource, Workspace, }; +use core::cell::OnceCell; use core::hint::{Locality, prefetch_read}; use crate::common::bitstream::BitDStream; use crate::common::entropy::read_ncount; use crate::common::{ - BlockType, LL_BITS, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MIN_CBLOCK_SIZE, - MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, MINMATCH, ML_BITS, ML_FSE_LOG, OFF_FSE_LOG, + BlockType, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MIN_CBLOCK_SIZE, + MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, MINMATCH, ML_FSE_LOG, OFF_FSE_LOG, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; use crate::error::{ErrorCode, Result}; @@ -55,6 +56,7 @@ pub(crate) struct EntropyDTables { pub(crate) seq: SeqTables, pub(crate) huf_table: HufDTable, pub(crate) rep: [u32; ZSTD_REP_NUM], + pub(crate) workspace: Workspace, } impl EntropyDTables { @@ -63,6 +65,7 @@ impl EntropyDTables { seq: SeqTables::new(), huf_table: HufDTable::new(), rep: REP_START_VALUE, + workspace: Workspace::new(), } } } @@ -442,8 +445,8 @@ impl BlockDecoder { MAX_LL, LL_FSE_LOG, src.sub(ip..)?, - &LL_BASE, - &LL_BITS, + &LL_CODES, + &mut self.entropy.workspace, fse_entropy, ) .map_err(|_| ErrorCode::CorruptionDetected)?; @@ -454,8 +457,8 @@ impl BlockDecoder { MAX_OFF, OFF_FSE_LOG, src.sub(ip..)?, - &OF_BASE, - &OF_BITS, + &OF_CODES, + &mut self.entropy.workspace, fse_entropy, ) .map_err(|_| ErrorCode::CorruptionDetected)?; @@ -466,8 +469,8 @@ impl BlockDecoder { MAX_ML, ML_FSE_LOG, src.sub(ip..)?, - &ML_BASE, - &ML_BITS, + &ML_CODES, + &mut self.entropy.workspace, fse_entropy, ) .map_err(|_| ErrorCode::CorruptionDetected)?; @@ -562,13 +565,13 @@ impl BlockDecoder { }; if nb_seq > 0 { - let mut seq_state = SeqState::new(sequences, tables, rep)?; - if use_prefetch_decoder { - exec.run::(&mut seq_state, nb_seq)?; + let seq_state = SeqState::new(sequences, tables, rep)?; + let prev_offset = if use_prefetch_decoder { + exec.run::(&seq_state, nb_seq)? } else { - exec.run::(&mut seq_state, nb_seq)?; - } - *rep = seq_state.prev_offset.map(|offset| offset as u32); + exec.run::(&seq_state, nb_seq)? + }; + *rep = prev_offset.map(|offset| offset as u32); } exec.last_literals()?; Ok(exec.op - start) @@ -583,8 +586,8 @@ fn build_seq_table( max: u32, max_log: u32, src: &[u8], - base_value: &[u32], - nb_additional_bits: &[u8], + codes: &Codes, + workspace: &mut Workspace, flag_repeat_table: bool, ) -> Result { match encoding_type { @@ -595,8 +598,7 @@ fn build_seq_table( if u32::from(symbol) > max { return Err(ErrorCode::CorruptionDetected); } - let symbol = usize::from(symbol); - table.build_rle(base_value.at(symbol)?, nb_additional_bits.at(symbol)?); + table.build_rle(codes.at(usize::from(symbol))?); *source = TableSource::Local; Ok(1) } @@ -619,9 +621,9 @@ fn build_seq_table( } table.build( norm.sub(..header.max_symbol_value as usize + 1)?, - base_value, - nb_additional_bits, + codes, header.table_log, + workspace, )?; *source = TableSource::Local; Ok(header.size) @@ -686,6 +688,51 @@ impl History { } } +/// A copy of the start of a stream after zeroes, where it can be read as if its start were far. That it was not shows +/// afterwards. +struct PaddedStart(OnceCell<[u8; Self::PADDING + Self::MAX_SIZE]>); + +impl PaddedStart { + const PADDING: usize = 32; + const MAX_SIZE: usize = 32; + + const fn new() -> Self { + Self(OnceCell::new()) + } + + /// `bit_d` on the copy, if little enough is left of it. + fn set(&self, bit_d: &BitDStream<'_>) -> Option> { + let src = bit_d.bytes(); + if src.len() > Self::MAX_SIZE || bit_d.bits_consumed > 64 { + return None; + } + let copy = self.0.get_or_init(|| { + let mut copy = [0; _]; + if let Some(start) = copy.get_mut(Self::PADDING..Self::PADDING + src.len()) { + start.copy_from_slice(src); + } + copy + }); + // A stream of less than 8 bytes counts what it lacks as consumed. + let lacking = 8usize.saturating_sub(src.len()) as u32 * 8; + let pos = Self::PADDING + src.len() - 8; + BitDStream::resume(copy, pos, bit_d.bits_consumed.checked_sub(lacking)?).ok() + } + + /// How many bits are left to read of the copy and of the zeroes. + fn bits_left(bit_d: &BitDStream<'_>) -> usize { + (bit_d.bytes().len() * 8).saturating_sub(bit_d.bits_consumed as usize) + } + + fn is_reached(bit_d: &BitDStream<'_>) -> bool { + Self::bits_left(bit_d) == Self::PADDING * 8 + } + + fn is_passed(bit_d: &BitDStream<'_>) -> bool { + Self::bits_left(bit_d) < Self::PADDING * 8 + } +} + /// `seqState_t` #[derive(Clone, Copy)] struct SeqState<'a> { @@ -896,14 +943,7 @@ impl<'a> SeqState<'a> { } } -#[inline(always)] -fn copy_within(buf: &mut [u8], src: usize, dst: usize) -> Result<()> { - let chunk: [u8; N] = *buf.array_at(src)?; - *buf.array_at_mut(dst)? = chunk; - Ok(()) -} - -/// The same for positions that do not overflow. Whether they are in `buf`. +/// Copies 16 bytes within `buf`, between positions that do not overflow. Whether they are in `buf`. #[inline(always)] fn copy_16(buf: &mut [u8], src: usize, dst: usize) -> bool { let Some(Ok(chunk)) = buf.get(src..src + 16).map(<[u8; 16]>::try_from) else { @@ -1305,15 +1345,32 @@ impl<'a> Executor<'_, 'a> { } /// `ZSTD_decompressSequences_body`, `ZSTD_decompressSequences_bodySplitLitBuffer` and, with `LONG`, - /// `ZSTD_decompressSequencesLong_body`. - fn run(&mut self, seq_state: &mut SeqState<'_>, nb_seq: usize) -> Result<()> { - const BATCH: usize = 128; - // After 3 that only stand for the last offsets. - let mut batch = [RawSequence::default(); 3 + BATCH]; + /// `ZSTD_decompressSequencesLong_body`. Returns the last offsets. + fn run( + &mut self, + seq_state: &SeqState<'_>, + nb_seq: usize, + ) -> Result<[usize; ZSTD_REP_NUM]> { + // Sequences are decoded that many at a time, after 3 that only stand for the last offsets. + if nb_seq < 32 { + self.run_by_batch::(seq_state, nb_seq, &mut [RawSequence::default(); 3 + 32]) + } else { + self.run_by_batch::(seq_state, nb_seq, &mut [RawSequence::default(); 3 + 128]) + } + } + + fn run_by_batch( + &mut self, + seq_state: &SeqState<'_>, + nb_seq: usize, + batch: &mut [RawSequence], + ) -> Result<[usize; ZSTD_REP_NUM]> { // The C code then decodes 8 sequences ahead, so it sees whether the stream ends where it should before it // executes the last ones. let mut ahead = [Sequence::default(); 8]; let ahead = ahead.sub_mut(..if LONG { nb_seq.min(8) } else { 0 })?; + let start = PaddedStart::new(); + let mut seq_state: SeqState<'_> = *seq_state; // A branch that goes wrong costs about 10 times what it saves. let tables = seq_state.tables; @@ -1323,46 +1380,79 @@ impl<'a> Executor<'_, 'a> { // Long copies go on while the next sequences are decoded, if that is what comes next. if tables.ml.with_more_bits > 512 / 2 && !self.split { let mut exec = self.fork(); - let mut state = *seq_state; while remaining > ahead.len() { - let sequence = state.decode_sequence(remaining == 1); + let sequence = seq_state.decode_sequence(remaining == 1); remaining -= 1; exec.exec_sequence::(sequence)?; } let exec = (exec.op, exec.lits, exec.split); self.join(exec); - *seq_state = state; } + // Where decoding went on from a copy of the start of the stream, if it did. + let mut left_at = None; + let mut can_leave = !LONG; while remaining > ahead.len() { + let mut count = 0; if !self.split { - let room = batch.span_mut(3, BATCH.min(remaining - ahead.len().max(1)))?; + let room = (batch.len() - 3).min(remaining - ahead.len().max(1)); + let room = batch.span_mut(3, room)?; let history = &mut self.history(); - let count = if more_bits { + count = if more_bits { seq_state.decode_batch::(room, history)? } else { seq_state.decode_batch::(room, history)? }; - if count > 0 { + if count == 0 + && can_leave + && let Some(bit_d) = start.set(&seq_state.bit_d) + { + can_leave = false; + left_at = Some((seq_state, remaining)); + seq_state.bit_d = bit_d; + continue; + } + } + let mut sequence = None; + if count == 0 { + sequence = Some(seq_state.decode_sequence(remaining == 1)); + } + if let Some((state, at)) = left_at + && PaddedStart::is_passed(&seq_state.bit_d) + { + // What the C code makes of that depends on more than the bits. + seq_state = state; + for _ in remaining..at { + seq_state.decode_sequence(false); + } + left_at = None; + continue; + } + match sequence { + Some(sequence) => { + remaining -= 1; + self.exec_sequence_maybe_split(sequence)?; + } + None => { remaining -= count; self.exec_all(batch.sub_mut(..3 + count)?, &mut seq_state.prev_offset)?; - continue; } } - let sequence = seq_state.decode_sequence(remaining == 1); - remaining -= 1; - self.exec_sequence_maybe_split(sequence)?; } for sequence in &mut *ahead { *sequence = seq_state.decode_sequence(remaining == 1); remaining -= 1; } - if !seq_state.bit_d.is_at_end() { + let is_at_end = match left_at { + Some(_) => PaddedStart::is_reached(&seq_state.bit_d), + None => seq_state.bit_d.is_at_end(), + }; + if !is_at_end { return Err(ErrorCode::CorruptionDetected); } for &sequence in &*ahead { self.exec_sequence_maybe_split(sequence)?; } - Ok(()) + Ok(seq_state.prev_offset) } fn history(&mut self) -> History { diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index d352a4538f90..cd79c8e8e779 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -4,7 +4,6 @@ //! the way that is fastest here. The others are left to [`huf_exact`](super::huf_exact). use super::huf_exact::{self, TableType, select_decoder_x2}; -use crate::common::bitstream::{BitDStream, DStreamStatus}; use crate::common::entropy::huf_read_stats; use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; use crate::error::{ErrorCode, Result}; @@ -28,7 +27,7 @@ pub(super) struct Code { /// The loop that decodes four streams at once takes a table of this log, or one more. const FAST_LOG: u32 = 11; /// From how many literals on that loop makes up for the time that it takes to fill such a table. -const FAST_MIN_LITERALS: usize = 768; +const FAST_MIN_LITERALS: usize = 2048; /// From how many literals on the larger table is considered. const LARGE_MIN_LITERALS: usize = 1 << 14; @@ -88,17 +87,172 @@ impl Code { } } -/// What a lookup yields: up to 3 symbols, then a byte with how many bits they take in its low 6 bits and how many they -/// are above. -type Entry = [u8; 4]; +/// What a lookup yields. From the lowest byte on: up to 3 symbols, then how many bits they take in 6 bits and how many +/// they are above. +type Entry = u32; const COUNT_SHIFT: u32 = 6; -/// As a number, what a symbol adds to an entry that has `index` symbols. -const fn entry(symbol: u8, index: u32, nb_bits: u32) -> u32 { +/// What a symbol adds to an entry that has `index` symbols. +const fn entry(symbol: u8, index: u32, nb_bits: u32) -> Entry { ((symbol as u32) << (8 * index)) | ((nb_bits + (1 << COUNT_SHIFT)) << 24) } +/// For each of `symbols`, `1 << run_log` entries that are `base` and the symbol at `index`. +fn fill_runs(out: &mut [Entry], symbols: &[u8], run_log: u32, base: Entry, index: u32) { + #[inline(always)] + fn runs(out: &mut [Entry], symbols: &[u8], base: Entry, index: u32) { + let (out, _) = out.as_chunks_mut::(); + for (run, &symbol) in out.iter_mut().zip(symbols) { + *run = [base + (u32::from(symbol) << (8 * index)); RUN]; + } + } + match run_log { + 0 => runs::<1>(out, symbols, base, index), + 1 => runs::<2>(out, symbols, base, index), + 2 => runs::<4>(out, symbols, base, index), + 3 => runs::<8>(out, symbols, base, index), + _ => { + for (run, &symbol) in out.chunks_exact_mut(1 << run_log).zip(symbols) { + run.fill(base + (u32::from(symbol) << (8 * index))); + } + } + } +} + +/// The symbols of a code in the order of their codes, which is by increasing weight. +struct Sorted { + symbols: [u8; NB_SYMBOLS], + /// Where the symbols of each weight start in `symbols`. + class_start: [usize; NB_CLASSES], + /// Where their entries start in a table. + position: [usize; NB_CLASSES], + /// How many bits index that table. + log: u32, + table_log: u32, + /// The weight of the shortest code. + max_weight: usize, +} + +/// More than there are weights, and a power of 2. +const NB_CLASSES: usize = 16; + +impl Sorted { + fn new(code: &Code, log: u32) -> Result { + let mut class_start = [0usize; NB_CLASSES]; + let mut position = [0usize; NB_CLASSES]; + let mut nb_sorted = 0usize; + let mut filled = 0usize; + let mut max_weight = 1; + for (weight, &count) in code.rank_stats.iter().enumerate().skip(1) { + class_start.put(weight, nb_sorted)?; + position.put(weight, filled)?; + if count != 0 { + max_weight = weight; + } + nb_sorted += count as usize; + filled += ((count as usize) << (weight - 1)) << (log - code.table_log); + } + class_start.sub_mut(NB_RANKS..)?.fill(nb_sorted); + position.sub_mut(NB_RANKS..)?.fill(filled); + + // The symbols that have no code go after the others. + let mut symbols = [0u8; NB_SYMBOLS]; + let mut cursor = class_start; + cursor.put(0, nb_sorted)?; + let mut place = |symbol: usize, weight: u8| { + if let Some(at) = cursor.get_mut(usize::from(weight) % NB_CLASSES) { + if let Some(slot) = symbols.get_mut(*at) { + *slot = symbol as u8; + } + *at += 1; + } + }; + let (eights, others) = code + .weights + .sub(..code.nb_symbols as usize)? + .as_chunks::<8>(); + for (index, eight) in eights.iter().enumerate() { + if *eight != [0; 8] { + for (offset, &weight) in eight.iter().enumerate() { + place(8 * index + offset, weight); + } + } + } + for (offset, &weight) in others.iter().enumerate() { + place(8 * eights.len() + offset, weight); + } + Ok(Self { + symbols, + class_start, + position, + log, + table_log: code.table_log, + max_weight, + }) + } + + fn class(&self, weight: usize) -> Result<&[u8]> { + self.symbols + .sub(self.class_start.at(weight)?..self.class_start.at(weight + 1)?) + } + + fn nb_bits(&self, weight: usize) -> u32 { + self.table_log + 1 - weight as u32 + } + + fn shortest(&self) -> u32 { + self.nb_bits(self.max_weight) + } + + /// The lowest weight of a code of at most `room` bits. + fn min_weight(&self, room: u32) -> usize { + (self.table_log as usize + 1) + .saturating_sub(room as usize) + .max(1) + } + + /// Where the entries of the symbols of a weight start in a table that `room` bits index. + fn position(&self, weight: usize, room: u32) -> Result { + Ok(self.position.at(weight)? >> (self.log - room)) + } + + /// Fills a table that `room` bits index: each entry is `base`, which has `index` symbols, and the symbol that the + /// bits start with if they hold all of its code. `next` is what the bits after that add: the table for `n` bits is + /// there from `1 << n` on. + fn fill( + &self, + out: &mut [Entry], + room: u32, + base: Entry, + index: u32, + next: Option<&[Entry]>, + ) -> Result<()> { + let min_weight = self.min_weight(room); + out.sub_mut(..self.position(min_weight, room)?)?.fill(base); + for weight in min_weight..=self.max_weight { + let symbols = self.class(weight)?; + let room_after = room - self.nb_bits(weight); + let out = out.span_mut(self.position(weight, room)?, symbols.len() << room_after)?; + let base = base + entry(0, index, self.nb_bits(weight)); + match next { + Some(next) if room_after >= self.shortest() => { + let size = 1usize << room_after; + let next = next.span(size, size)?; + for (out, &symbol) in out.chunks_exact_mut(size).zip(symbols) { + let base = base + (u32::from(symbol) << (8 * index)); + for (entry, next) in out.iter_mut().zip(next) { + *entry = base + next; + } + } + } + _ => fill_runs(out, symbols, room_after, base, index), + } + } + Ok(()) + } +} + /// `HUF_DTable` #[derive(Clone)] pub(crate) struct HufDTable { @@ -121,7 +275,7 @@ impl HufDTable { }, table_type: TableType::X1, log: 0, - table: [[0; 4]; TABLE_SIZE], + table: [0; TABLE_SIZE], } } @@ -146,150 +300,63 @@ impl HufDTable { /// Fills `table` for lookups of `log` bits, which is at least as many as the longest code has. fn build(&mut self, log: u32) -> Result<()> { - let Code { - weights, - rank_stats, - nb_symbols, - table_log, - } = &self.code; - let max_weight = *table_log as usize; self.log = log; - - // The symbols by increasing weight, which is the order of their codes. `class_start[w]` is where those of - // weight `w` start, and `position[w]` where their entries start in the table. - let mut sorted = [0u8; NB_SYMBOLS]; - let mut class_start = [0usize; NB_RANKS + 1]; - let mut position = [0usize; NB_RANKS + 1]; - let mut nb_sorted = 0usize; - let mut filled = 0usize; - for weight in 1..=max_weight + 1 { - class_start.put(weight, nb_sorted)?; - position.put(weight, filled)?; - let count = rank_stats.get(weight).copied().unwrap_or(0) as usize; - nb_sorted += count; - filled += count << (weight as u32 - 1 + log - table_log); - } - let mut cursor = class_start; - for (symbol, &weight) in weights.iter().enumerate().take(*nb_symbols as usize) { - if weight != 0 { - let at = cursor.at_mut(usize::from(weight))?; - sorted.put(*at, symbol as u8)?; - *at += 1; - } + let sorted = Sorted::new(&self.code, log)?; + // The tables of what a third symbol adds end with the one for this many bits. + let room = log.saturating_sub(2 * sorted.shortest()); + if room < 7 { + self.fill(&sorted, &mut [0; 1 << 7]) + } else { + self.fill(&sorted, &mut [0; TABLE_SIZE / 2]) } - let class = - |weight: usize| sorted.sub(class_start.at(weight)?..class_start.at(weight + 1)?); - let nb_bits = |weight: usize| table_log + 1 - weight as u32; - // The lowest weight of a code of at most `room` bits. - let min_weight = |room: u32| (max_weight + 1).saturating_sub(room as usize).max(1); + } - // What follows a symbol in an index is an index into a smaller table of the same code, where the codes that are - // too long for it come first. For each size, from `size` on: what a third symbol adds to an entry. - let mut thirds = [0u32; TABLE_SIZE / 2]; - let shortest = nb_bits(max_weight); + fn fill(&mut self, sorted: &Sorted, thirds: &mut [Entry]) -> Result<()> { + let log = sorted.log; + let shortest = sorted.shortest(); for room in shortest..=log.saturating_sub(2 * shortest) { - let thirds = thirds.span_mut(1 << room, 1 << room)?; - for weight in min_weight(room)..=max_weight { - let run = 1usize << (room - nb_bits(weight)); - let symbols = class(weight)?; - let thirds = - thirds.span_mut(position.at(weight)? >> (log - room), run * symbols.len())?; - for (thirds, &symbol) in thirds.chunks_exact_mut(run).zip(symbols) { - thirds.fill(entry(symbol, 2, nb_bits(weight))); - } - } + sorted.fill(thirds.span_mut(1 << room, 1 << room)?, room, 0, 2, None)?; } - - for weight in 1..=max_weight { - let Some((&first_symbol, other_symbols)) = class(weight)?.split_first() else { + for weight in 1..=sorted.max_weight { + let symbols = sorted.class(weight)?; + let nb_bits = sorted.nb_bits(weight); + let room = log - nb_bits; + let entries = self + .table + .span_mut(sorted.position(weight, log)?, symbols.len() << room)?; + if room < shortest { + fill_runs(entries, symbols, room, entry(0, 0, nb_bits), 0); + continue; + } + let Some((&first_symbol, other_symbols)) = symbols.split_first() else { continue; }; - let first_bits = nb_bits(weight); - let room = log - first_bits; let size = 1usize << room; - let entries = self - .table - .span_mut(position.at(weight)?, size * (1 + other_symbols.len()))?; let (entries, other_entries) = entries.split_at_mut_checked(size).ok_or(BUG)?; - - let first = entry(first_symbol, 0, first_bits); - entries - .sub_mut(..position.at(min_weight(room))? >> first_bits)? - .fill(first.to_le_bytes()); - for second_weight in min_weight(room)..=max_weight { - let room = room - nb_bits(second_weight); - let size = 1usize << room; - let symbols = class(second_weight)?; - let thirds = thirds.span(size, size)?; - let entries = entries.span_mut( - position.at(second_weight)? >> first_bits, - size * symbols.len(), - )?; - for (entries, &symbol) in entries.chunks_exact_mut(size).zip(symbols) { - let two = first + entry(symbol, 1, nb_bits(second_weight)); - for (entry, third) in entries.iter_mut().zip(thirds) { - *entry = (two + third).to_le_bytes(); - } - } - } - + sorted.fill( + entries, + room, + entry(first_symbol, 0, nb_bits), + 1, + Some(thirds), + )?; for (copy, &symbol) in other_entries.chunks_exact_mut(size).zip(other_symbols) { + let change = u32::from(symbol).wrapping_sub(u32::from(first_symbol)); for (to, from) in copy.iter_mut().zip(&*entries) { - let [_, second, third, meta] = *from; - *to = [symbol, second, third, meta]; + *to = from.wrapping_add(change); } } } Ok(()) } - #[inline(always)] - fn entry(&self, index: u64) -> Entry { - self.table - .get(index as usize & (TABLE_SIZE - 1)) - .copied() - .unwrap_or_default() - } - - /// Fills `out` from a stream. Whether that takes exactly what is left of the stream. - fn decode_stream(&self, out: &mut [u8], bit_d: &mut BitDStream<'_>) -> bool { - let log = self.log; - let mut rest = &mut *out; - // Four lookups take at most 48 bits. - while rest.len() >= 16 && bit_d.reload() == DStreamStatus::Unfinished { - let mut done = 0usize; - for _ in 0..4 { - let entry = self.entry(bit_d.look_bits_fast(log)); - let [.., meta] = entry; - bit_d.skip_bits(u32::from(meta & 63)); - if let Some(to) = rest.get_mut(done..done + 4) { - to.copy_from_slice(&entry); - } - done += usize::from(meta >> COUNT_SHIFT); - } - rest = rest.get_mut(done..).unwrap_or_default(); - } - for byte in rest { - bit_d.reload(); - let [symbol, ..] = self.entry(bit_d.look_bits_fast(log)); - let weight = self - .code - .weights - .get(usize::from(symbol)) - .copied() - .unwrap_or(0); - bit_d.skip_bits(self.code.table_log + 1 - u32::from(weight)); - *byte = symbol; - } - bit_d.is_at_end() - } - /// One stream. Whether it is exactly `dst.len()` symbols. fn decode_1x(&self, dst: &mut [u8], src: &[u8]) -> bool { - match BitDStream::new(src) { - Ok(mut bit_d) => self.decode_stream(dst, &mut bit_d), - Err(_) => false, - } + let Some(mut stream) = Stream::new(Part { src, out: dst }) else { + return false; + }; + stream.decode_most(&self.table, self.log); + self.decode_tails([stream]) } /// Four streams after their sizes. Whether each is exactly its share of `dst.len()` symbols. @@ -300,25 +367,30 @@ impl HufDTable { let Some(mut streams) = Streams::new(parts) else { return false; }; - let mut no_output = [[0u8; 512]; 4]; - if self.log >= FAST_LOG { - self.decode_most(&mut streams, &mut no_output); + if streams.0.iter().all(|stream| stream.bits != 0) { + self.decode_most(&mut streams); } - let mut exact = true; - for stream in streams.0 { - let Ok(mut bit_d) = stream.bit_d() else { - return false; - }; - let out = stream.out.get_mut(stream.op as usize..).unwrap_or_default(); - exact &= self.decode_stream(out, &mut bit_d); + if !streams.0.iter().all(Stream::fits_tail) { + return self.decode_stragglers(streams); + } + self.decode_tails(streams.0) + } + + fn decode_most(&self, streams: &mut Streams<'_>) { + match self.log { + 11 => streams.decode_most::<11, 5>(&self.table, 0), + 12 => streams.decode_most::<12, 4>(&self.table, 0), + log => streams.decode_most::<0, 5>(&self.table, 64 - log), } - exact } - /// Decodes each stream until it is close to its start, or its output to its end. - fn decode_most<'a>(&self, streams: &mut Streams<'a>, no_output: &'a mut [[u8; 512]; 4]) { + /// The same for streams of which some have much left. + #[inline(never)] + fn decode_stragglers(&self, streams: Streams<'_>) -> bool { // Some streams get there before others, which go on four at a time all the same: with these. static NO_STREAM: [u8; 256] = [0x55; 256]; + let mut no_output = [[0u8; 512]; 4]; + let mut streams: Streams<'_> = streams; let mut decoding = Streams(no_output.each_mut().map(|out| Stream { src: &NO_STREAM, ip: 0, @@ -348,11 +420,159 @@ impl HufDTable { if left == [false; 4] { break; } - match self.log { - 11 => decoding.decode_most::<11, 5>(&self.table), - _ => decoding.decode_most::<12, 4>(&self.table), + self.decode_most(&mut decoding); + } + self.decode_tails(streams.0) + } + + /// Decodes what is left of streams. Whether each turns out to be exactly as many symbols as its output takes. + fn decode_tails(&self, mut streams: [Stream<'_>; N]) -> bool { + let mut tails = [const { Tail::new() }; N]; + for (tail, stream) in tails.iter_mut().zip(&streams) { + if !tail.set(stream) { + return false; + } + } + let shift = 64 - self.log; + loop { + let mut rounds = tails.each_ref().map(Tail::start_round); + for _ in 0..Tail::LOOKUPS { + for (tail, round) in tails.iter_mut().zip(&mut rounds) { + let index = (round.bits >> shift) as usize; + let entry = self.table.get(index % TABLE_SIZE).copied().unwrap_or(0); + round.bits <<= (entry >> 24) & 63; + if let Some(to) = tail.out.get_mut(round.op..round.op + 4) { + to.copy_from_slice(&entry.to_le_bytes()); + } + round.op += (entry >> (24 + COUNT_SHIFT)) as usize; + } + } + let mut more = false; + for (tail, round) in tails.iter_mut().zip(&rounds) { + more |= tail.end_round(round); + } + if !more { + break; } } + let mut exact = true; + for (tail, stream) in tails.iter().zip(&mut streams) { + exact &= tail.get(stream, self.code.table_log); + } + exact + } +} + +/// What is left of a stream that is close to its start or whose output is close to its end, where decoding can go too +/// far: that is made up for afterwards. +struct Tail { + /// Zeroes, then the start of the stream. + src: [u8; Self::PADDING + Self::MAX_SRC], + /// How many bits are left to read of `src`. + position: u32, + out: [u8; Self::MAX_SYMBOLS + Self::ROUND_SYMBOLS], + /// How much of `out` is written. + op: u32, +} + +/// A few lookups between two reads of a stream. +struct Round { + /// What is left of 64 bits, from the highest bit on, then a bit set. + bits: u64, + /// How many of the 64 bits were read before. + skipped: u32, + /// How much of the output is written. + op: usize, +} + +impl Tail { + const LOOKUPS: usize = 4; + /// How many bytes a round can write. + const ROUND_SYMBOLS: usize = 3 * Self::LOOKUPS + 1; + /// No less than a round can read, and 8 bytes. + const PADDING: usize = 16; + const MAX_SRC: usize = 64; + const MAX_SYMBOLS: usize = 128; + + const fn new() -> Self { + Self { + src: [0; _], + position: 0, + out: [0; _], + op: 0, + } + } + + /// Takes what is left of a stream, if it is not too much and the stream has an end mark. + fn set(&mut self, stream: &Stream<'_>) -> bool { + let Some(&last) = stream.src.last() else { + return false; + }; + if last == 0 || !stream.fits_tail() { + return false; + } + let end = stream.end(); + let src = self.src.get_mut(Self::PADDING..).unwrap_or_default(); + if let (Some(to), Some(from)) = ( + src.first_chunk_mut::<{ Self::MAX_SRC }>(), + stream.src.first_chunk::<{ Self::MAX_SRC }>(), + ) { + *to = *from; + } else if let (Some(to), Some(from)) = (src.get_mut(..end), stream.src.get(..end)) { + to.copy_from_slice(from); + } + let consumed = if stream.bits == 0 { + last.leading_zeros() + 1 + } else { + stream.bits.trailing_zeros() + }; + self.position = 8 * (Self::PADDING + end) as u32 - consumed; + true + } + + /// Whether neither the start of the stream nor the end of `out` is reached. + fn has_more(&self) -> bool { + self.position > 8 * Self::PADDING as u32 && self.op as usize <= Self::MAX_SYMBOLS + } + + #[inline(always)] + fn start_round(&self) -> Round { + let end = (self.position as usize).div_ceil(8); + let container = self + .src + .get(end.wrapping_sub(8)..end) + .and_then(|bytes| bytes.try_into().ok()) + .map_or(0, u64::from_le_bytes); + let skipped = self.position.wrapping_neg() & 7; + Round { + bits: (container | 1) << skipped, + skipped, + op: self.op as usize, + } + } + + /// Returns whether there is to be another round. + #[inline(always)] + fn end_round(&mut self, round: &Round) -> bool { + if !self.has_more() { + return false; + } + self.op = round.op as u32; + self.position -= round.bits.trailing_zeros() - round.skipped; + self.has_more() + } + + /// Completes the output of the stream. Whether the stream is exactly as many symbols as its output takes. + fn get(&self, stream: &mut Stream<'_>, table_log: u32) -> bool { + let out = stream.out.get_mut(stream.op as usize..).unwrap_or_default(); + let Some(symbols) = self.out.get(..out.len()) else { + return false; + }; + out.copy_from_slice(symbols); + // Zeroes are the longest code. + let nb_bits = (8 * Self::PADDING as u32).wrapping_sub(self.position); + let nb_symbols = self.op.wrapping_sub(out.len() as u32); + !self.has_more() && u64::from(nb_bits) == u64::from(nb_symbols) * u64::from(table_log) } } @@ -442,11 +662,48 @@ impl<'a> Stream<'a> { && reload(self.src, &mut self.ip, &mut self.bits) } - fn bit_d(&self) -> Result> { + /// Where what is left of `src` ends. + fn end(&self) -> usize { if self.bits == 0 { - return BitDStream::new(self.src); + self.src.len() + } else { + self.ip as usize + 8 } - BitDStream::resume(self.src, self.ip as usize, self.bits.trailing_zeros()) + } + + fn fits_tail(&self) -> bool { + self.end() <= Tail::MAX_SRC && self.out.len() - self.op as usize <= Tail::MAX_SYMBOLS + } + + /// Decodes until the stream is close to its start, or its output to its end. + fn decode_most(&mut self, table: &[Entry; TABLE_SIZE], log: u32) { + const LOOKUPS: usize = 4; + if self.bits == 0 { + return; + } + let shift = 64 - log; + let (mut ip, mut bits, mut op) = (self.ip, self.bits, self.op); + while let Some(window) = self + .out + .get_mut(op as usize..op as usize + 3 * LOOKUPS + 1) + .and_then(|window| window.as_mut_array::<{ 3 * LOOKUPS + 1 }>()) + && reload(self.src, &mut ip, &mut bits) + { + let mut done = 0; + for _ in 0..LOOKUPS { + let entry = table + .get((bits >> shift) as usize % TABLE_SIZE) + .copied() + .unwrap_or(0); + bits <<= (entry >> 24) & 63; + if let Some(to) = window.get_mut(done..done + 4) { + to.copy_from_slice(&entry.to_le_bytes()); + } + done += (entry >> (24 + COUNT_SHIFT)) as usize; + } + op += done as u32; + } + (self.ip, self.bits, self.op) = (ip, bits, op); } } @@ -462,23 +719,32 @@ impl<'a> Streams<'a> { ])) } - /// Decodes for as long as no stream is close to its start and no output to its end. `LOOKUPS` of `LOG` bits are at - /// most 57 bits. - fn decode_most(&mut self, table: &[Entry; TABLE_SIZE]) { + /// Decodes for as long as no stream is close to its start and no output to its end. A lookup is of `LOG` bits, or of + /// `64 - shift` if that is 0, and `LOOKUPS` of them are at most 55 bits. + fn decode_most( + &mut self, + table: &[Entry; TABLE_SIZE], + shift: u32, + ) { #[inline(always)] fn lookup( table: &[Entry; TABLE_SIZE], + shift: u32, bits: &mut u64, window: &mut [u8; WINDOW], done: &mut usize, ) { - if let Some(entry) = table.get((*bits >> (64 - LOG)) as usize) { - let [.., meta] = *entry; - *bits <<= meta & 63; + let index = if LOG == 0 { + (*bits >> shift) as usize % TABLE_SIZE + } else { + (*bits >> (64 - LOG)) as usize + }; + if let Some(entry) = table.get(index) { + *bits <<= (entry >> 24) & 63; if let Some(to) = window.get_mut(*done..*done + 4) { - to.copy_from_slice(entry); + to.copy_from_slice(&entry.to_le_bytes()); } - *done += usize::from(meta >> COUNT_SHIFT); + *done += (entry >> (24 + COUNT_SHIFT)) as usize; } } @@ -503,10 +769,10 @@ impl<'a> Streams<'a> { }; let (mut done0, mut done1, mut done2, mut done3) = (0, 0, 0, 0); for _ in 0..LOOKUPS { - lookup::(table, &mut bits0, window0, &mut done0); - lookup::(table, &mut bits1, window1, &mut done1); - lookup::(table, &mut bits2, window2, &mut done2); - lookup::(table, &mut bits3, window3, &mut done3); + lookup::(table, shift, &mut bits0, window0, &mut done0); + lookup::(table, shift, &mut bits1, window1, &mut done1); + lookup::(table, shift, &mut bits2, window2, &mut done2); + lookup::(table, shift, &mut bits3, window3, &mut done3); } let mut more = reload(s0.src, &mut ip0, &mut bits0) && reload(s1.src, &mut ip1, &mut bits1) @@ -514,10 +780,10 @@ impl<'a> Streams<'a> { && reload(s3.src, &mut ip3, &mut bits3); if more { for _ in 0..LOOKUPS { - lookup::(table, &mut bits0, window0, &mut done0); - lookup::(table, &mut bits1, window1, &mut done1); - lookup::(table, &mut bits2, window2, &mut done2); - lookup::(table, &mut bits3, window3, &mut done3); + lookup::(table, shift, &mut bits0, window0, &mut done0); + lookup::(table, shift, &mut bits1, window1, &mut done1); + lookup::(table, shift, &mut bits2, window2, &mut done2); + lookup::(table, shift, &mut bits3, window3, &mut done3); } more = reload(s0.src, &mut ip0, &mut bits0) && reload(s1.src, &mut ip1, &mut bits1) diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index dc432f2f8c15..cb1ebeb041f6 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -10,18 +10,18 @@ use std::borrow::Cow; use block::{ BlockDecoder, BlockOutput, EntropyDTables, ExtDict, OutputBuffer, Streaming, get_c_block_size, }; -use seq_table::{LL_BASE, ML_BASE, OF_BASE, OF_BITS, SeqTableSources, TableSource}; +use seq_table::{LL_CODES, ML_CODES, OF_CODES, SeqTableSources, TableSource}; use crate::common::entropy::read_ncount; use crate::common::xxhash::Xxh64; use crate::common::{ - BlockType, LL_BITS, LL_FSE_LOG, MAX_LL, MAX_ML, MAX_OFF, ML_BITS, ML_FSE_LOG, OFF_FSE_LOG, - REP_START_VALUE, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_BLOCKSIZE_MAX, - ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_DID_FIELD_SIZE, ZSTD_FCS_FIELD_SIZE, - ZSTD_FRAMEHEADERSIZE_MAX, ZSTD_FRAMEIDSIZE, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGIC_SKIPPABLE_MASK, - ZSTD_MAGIC_SKIPPABLE_START, ZSTD_MAGICNUMBER, ZSTD_SKIPPABLEHEADERSIZE, - ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WINDOWLOG_LIMIT_DEFAULT, ZSTD_WINDOWLOG_MAX, - ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, + BlockType, LL_FSE_LOG, MAX_LL, MAX_ML, MAX_OFF, ML_FSE_LOG, OFF_FSE_LOG, REP_START_VALUE, + WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_ERROR, + ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_DID_FIELD_SIZE, ZSTD_FCS_FIELD_SIZE, ZSTD_FRAMEHEADERSIZE_MAX, + ZSTD_FRAMEIDSIZE, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGIC_SKIPPABLE_MASK, ZSTD_MAGIC_SKIPPABLE_START, + ZSTD_MAGICNUMBER, ZSTD_SKIPPABLEHEADERSIZE, ZSTD_WINDOWLOG_ABSOLUTEMIN, + ZSTD_WINDOWLOG_LIMIT_DEFAULT, ZSTD_WINDOWLOG_MAX, ZSTD_WORKSPACETOOLARGE_FACTOR, + ZSTD_WORKSPACETOOLARGE_MAXDURATION, }; use crate::error::{ErrorCode, Result}; use crate::util::{SliceExt, read_le16, read_le32, read_le64, try_copy}; @@ -343,9 +343,9 @@ fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result { } entropy.seq.of.build( ncount.sub(..header.max_symbol_value as usize + 1)?, - &OF_BASE, - &OF_BITS, + &OF_CODES, header.table_log, + &mut entropy.workspace, )?; pos += header.size; } @@ -357,9 +357,9 @@ fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result { } entropy.seq.ml.build( ncount.sub(..header.max_symbol_value as usize + 1)?, - &ML_BASE, - &ML_BITS, + &ML_CODES, header.table_log, + &mut entropy.workspace, )?; pos += header.size; } @@ -371,9 +371,9 @@ fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result { } entropy.seq.ll.build( ncount.sub(..header.max_symbol_value as usize + 1)?, - &LL_BASE, - &LL_BITS, + &LL_CODES, header.table_log, + &mut entropy.workspace, )?; pos += header.size; } diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs index 3ca90fbcbc1b..c30bed4bcb39 100644 --- a/src/zstd/decompress/seq_table.rs +++ b/src/zstd/decompress/seq_table.rs @@ -1,14 +1,14 @@ //! The FSE tables that sequences are decoded with: `ZSTD_seqSymbol` and `ZSTD_buildFSETable`. use crate::common::{ - LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, ML_DEFAULT_NORM_LOG, + LL_BITS, LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, ML_BITS, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM_LOG, fse_table_step, }; use crate::error::Result; -use crate::util::{SliceExt, highbit32, write_le64}; +use crate::util::{SliceExt, write_le64}; #[rustfmt::skip] -pub(crate) const LL_BASE: [u32; MAX_LL as usize + 1] = [ +const LL_BASE: [u32; MAX_LL as usize + 1] = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 22, 24, 28, 32, 40, @@ -16,25 +16,9 @@ pub(crate) const LL_BASE: [u32; MAX_LL as usize + 1] = [ 0x2000, 0x4000, 0x8000, 0x10000, ]; -#[rustfmt::skip] -pub(crate) const OF_BASE: [u32; MAX_OFF as usize + 1] = [ - 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, - 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, - 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, - 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD, -]; - -#[rustfmt::skip] -pub(crate) const OF_BITS: [u8; MAX_OFF as usize + 1] = [ - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31, -]; - /// What a match length code starts at, above `MINMATCH`. #[rustfmt::skip] -pub(crate) const ML_BASE: [u32; MAX_ML as usize + 1] = [ +const ML_BASE: [u32; MAX_ML as usize + 1] = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, @@ -57,7 +41,13 @@ pub(crate) struct SeqSymbol { pub(crate) trait Cell: Copy + 'static { const EMPTY: Self; const EMPTY_REF: &'static Self; - fn pack(symbol: SeqSymbol) -> Self; + /// Where `next_state` is in `to_bits`. + const NEXT_STATE_SHIFT: u32; + /// For each `nb_bits`, `to_bits` of a cell where nothing else is set but what follows from it. + const NB_BITS: &'static [u64; 16]; + /// The fields one after the other, from the lowest bits on. + fn from_bits(bits: u64) -> Self; + fn has_additional_bits(self) -> bool; fn unpack(self) -> SeqSymbol; } @@ -84,6 +74,14 @@ const fn l(next_state: u16, nb_additional_bits: u8, nb_bits: u8, base_value: u32 } impl LengthCell { + const fn to_bits(self) -> u64 { + self.base_value as u64 + | (self.next_state as u64) << 16 + | (self.state_mask as u64) << 32 + | (self.nb_additional_bits as u64) << 48 + | (self.nb_bits as u64) << 56 + } + #[inline(always)] pub(crate) fn base_value(self) -> u32 { u32::from(self.base_value) + (u32::from(self.nb_additional_bits >> 4) << 16) @@ -94,17 +92,35 @@ impl Cell for LengthCell { const EMPTY: Self = l(0, 0, 0, 0); const EMPTY_REF: &'static Self = &Self::EMPTY; + const NEXT_STATE_SHIFT: u32 = 16; + const NB_BITS: &'static [u64; 16] = &{ + let mut all = [0; 16]; + let mut left: &mut [u64] = &mut all; + let mut nb_bits = 0; + while let [bits, others @ ..] = left { + *bits = l(0, 0, nb_bits, 0).to_bits(); + nb_bits += 1; + left = others; + } + all + }; + #[inline(always)] - fn pack(symbol: SeqSymbol) -> Self { + fn from_bits(bits: u64) -> Self { Self { - base_value: symbol.base_value as u16, - next_state: symbol.next_state, - state_mask: (1u16.wrapping_shl(u32::from(symbol.nb_bits))).wrapping_sub(1), - nb_additional_bits: symbol.nb_additional_bits, - nb_bits: symbol.nb_bits, + base_value: bits as u16, + next_state: (bits >> 16) as u16, + state_mask: (bits >> 32) as u16, + nb_additional_bits: (bits >> 48) as u8, + nb_bits: (bits >> 56) as u8, } } + #[inline(always)] + fn has_additional_bits(self) -> bool { + self.nb_additional_bits != 0 + } + #[inline(always)] fn unpack(self) -> SeqSymbol { SeqSymbol { @@ -139,28 +155,55 @@ const fn o(next_state: u8, nb_additional_bits: u8, nb_bits: u8) -> OffsetCell { } } +impl OffsetCell { + const fn to_bits(self) -> u64 { + self.first_value as u64 + | (self.next_state as u64) << 32 + | (self.state_mask as u64) << 40 + | (self.nb_additional_bits as u64) << 48 + | (self.nb_bits as u64) << 56 + } +} + impl Cell for OffsetCell { const EMPTY: Self = o(0, 0, 0); const EMPTY_REF: &'static Self = &Self::EMPTY; + const NEXT_STATE_SHIFT: u32 = 32; + const NB_BITS: &'static [u64; 16] = &{ + let mut all = [0; 16]; + let mut left: &mut [u64] = &mut all; + let mut nb_bits = 0; + while let [bits, others @ ..] = left { + *bits = o(0, 0, nb_bits).to_bits() - o(0, 0, 0).to_bits(); + nb_bits += 1; + left = others; + } + all + }; + #[inline(always)] - fn pack(symbol: SeqSymbol) -> Self { + fn from_bits(bits: u64) -> Self { Self { - first_value: 1u32.wrapping_shl(u32::from(symbol.nb_additional_bits)), - next_state: symbol.next_state as u8, - state_mask: (1u16.wrapping_shl(u32::from(symbol.nb_bits))).wrapping_sub(1) as u8, - nb_additional_bits: symbol.nb_additional_bits, - nb_bits: symbol.nb_bits, + first_value: bits as u32, + next_state: (bits >> 32) as u8, + state_mask: (bits >> 40) as u8, + nb_additional_bits: (bits >> 48) as u8, + nb_bits: (bits >> 56) as u8, } } + #[inline(always)] + fn has_additional_bits(self) -> bool { + self.nb_additional_bits != 0 + } + #[inline(always)] fn unpack(self) -> SeqSymbol { SeqSymbol { next_state: u16::from(self.next_state), nb_additional_bits: self.nb_additional_bits, nb_bits: self.nb_bits, - // `OF_BASE` base_value: if self.nb_additional_bits < 2 { u32::from(self.nb_additional_bits) } else { @@ -170,69 +213,124 @@ impl Cell for OffsetCell { } } +/// What each code of a kind stands for: `to_bits` of a cell that leads to no state. There are fewer codes than that. +pub(crate) type Codes = [u64; 64]; + +const fn length_codes(mut base_values: &[u32], mut nb_additional_bits: &[u8]) -> Codes { + let mut codes = [0; 64]; + let mut left: &mut [u64] = &mut codes; + while let ( + [code, other_codes @ ..], + [base_value, other_values @ ..], + [nb_bits, other_nb_bits @ ..], + ) = (left, base_values, nb_additional_bits) + { + *code = l(0, *nb_bits, 0, *base_value).to_bits(); + (left, base_values, nb_additional_bits) = (other_codes, other_values, other_nb_bits); + } + codes +} + +const fn offset_codes() -> Codes { + let mut codes = [0; 64]; + let (mut left, _) = codes.split_at_mut(MAX_OFF as usize + 1); + let mut nb_additional_bits = 0; + while let [code, others @ ..] = left { + *code = o(0, nb_additional_bits, 0).to_bits(); + nb_additional_bits += 1; + left = others; + } + codes +} + +pub(crate) static LL_CODES: Codes = length_codes(&LL_BASE, &LL_BITS); +pub(crate) static OF_CODES: Codes = offset_codes(); +pub(crate) static ML_CODES: Codes = length_codes(&ML_BASE, &ML_BITS); + const MAX_TABLE_SIZE: usize = 1 << 9; -/// The first two passes of `ZSTD_buildFSETable`: which symbol each state decodes. -struct Spread { - symbols: [u8; MAX_TABLE_SIZE + 8], - symbol_next: [u16; MAX_SEQ as usize + 1], +/// The position of the highest bit set. +static HIGH_BIT: [u8; 2 * MAX_TABLE_SIZE] = { + let mut table = [0; 2 * MAX_TABLE_SIZE]; + let (_, mut left) = table.split_at_mut(1); + let mut value = 1u32; + while let [high_bit, others @ ..] = left { + *high_bit = value.ilog2() as u8; + value += 1; + left = others; + } + table +}; + +/// What tables are built with. +#[derive(Clone)] +pub(crate) struct Workspace { + /// The symbols, each as many times as it has states. + in_order: [u8; MAX_TABLE_SIZE + 8], + /// Which symbol each state decodes. + symbols: [u8; MAX_TABLE_SIZE], + symbol_next: [u16; 64], + /// The symbols of low probability. + rare: [u8; 64], } -impl Spread { - fn new(normalized_counter: &[i16], table_log: u32) -> Result { - let table_size = 1usize << table_log; - let mut symbols = [0u8; MAX_TABLE_SIZE + 8]; - let mut symbol_next = [0u16; MAX_SEQ as usize + 1]; - let mut high_threshold = table_size - 1; +impl Workspace { + pub(crate) const fn new() -> Self { + Self { + in_order: [0; _], + symbols: [0; _], + symbol_next: [0; _], + rare: [0; _], + } + } - for (symbol, (&count, next)) in normalized_counter.iter().zip(&mut symbol_next).enumerate() + /// The first two passes of `ZSTD_buildFSETable`, with few branches to mispredict. + fn spread(&mut self, normalized_counter: &[i16], table_log: u32) -> Result<()> { + let table_size = 1usize << table_log; + let mut nb_rare = 0usize; + let mut nb_in_order = 0usize; + let mut eight_times = 0u64; + for ((symbol, &count), next) in normalized_counter + .iter() + .enumerate() + .zip(&mut self.symbol_next) { - if count == -1 { - symbols.put(high_threshold, symbol as u8)?; - high_threshold = high_threshold.wrapping_sub(1); - *next = 1; - } else { - *next = count as u16; + let is_rare = count < 0; + if let Some(rare) = self.rare.get_mut(nb_rare % 64) { + *rare = symbol as u8; + } + nb_rare += usize::from(is_rare); + *next = if is_rare { 1 } else { count as u16 }; + let count = if is_rare { 0 } else { count as usize }; + write_le64(&mut self.in_order, nb_in_order, eight_times)?; + if count > 8 { + for done in (8..count).step_by(8) { + write_le64(&mut self.in_order, nb_in_order + done, eight_times)?; + } } + nb_in_order += count; + eight_times += 0x0101_0101_0101_0101; } + // The states from here on are those of the rare symbols. Others that would go there go to the next place. + let first_rare = table_size - nb_rare.min(table_size); let table_mask = table_size - 1; let step = fse_table_step(table_size); - if high_threshold == table_size - 1 { - // No symbol of low probability: lay the symbols down in order, then spread them in a second pass that - // has no branch to mispredict. - let mut in_order = [0u8; MAX_TABLE_SIZE + 8]; - let mut pos = 0usize; - let mut sv = 0u64; - for &count in normalized_counter { - let n = count as usize; - for i in (0..n.max(1)).step_by(8) { - write_le64(&mut in_order, pos + i, sv)?; - } - pos += n; - sv += 0x0101_0101_0101_0101; - } - let mut position = 0usize; - for &symbol in in_order.sub(..table_size)? { - symbols.put(position, symbol)?; - position = (position + step) & table_mask; - } - } else { - let mut position = 0usize; - for (symbol, &count) in normalized_counter.iter().enumerate() { - for _ in 0..count { - symbols.put(position, symbol as u8)?; - position = (position + step) & table_mask; - while position > high_threshold { - position = (position + step) & table_mask; - } - } + let symbols = self.symbols.sub_mut(..table_size)?; + let mut position = 0usize; + let mut from = 0usize; + for _ in 0..table_size / 2 { + for position in [position, (position + step) & table_mask] { + let symbol = self.in_order.get(from % MAX_TABLE_SIZE).copied(); + symbols.put(position, symbol.unwrap_or(0))?; + from += usize::from(position < first_rare); } + position = (position + 2 * step) & table_mask; + } + for (symbol, &rare) in symbols.iter_mut().rev().zip(self.rare.sub(..nb_rare)?) { + *symbol = rare; } - Ok(Self { - symbols, - symbol_next, - }) + Ok(()) } } @@ -302,16 +400,12 @@ impl SeqTable { } /// `ZSTD_buildSeqTable_rle` - pub(crate) fn build_rle(&mut self, base_value: u32, nb_additional_bits: u8) { + pub(crate) fn build_rle(&mut self, code: u64) { + let code = C::from_bits(code); self.table_log = 0; - self.with_more_bits = u32::from(nb_additional_bits != 0) << 9; + self.with_more_bits = u32::from(code.has_additional_bits()) << 9; if let Some(cell) = self.table.first_mut() { - *cell = C::pack(SeqSymbol { - next_state: 0, - nb_additional_bits, - nb_bits: 0, - base_value, - }); + *cell = code; } } @@ -319,38 +413,36 @@ impl SeqTable { pub(crate) fn build( &mut self, normalized_counter: &[i16], - base_value: &[u32], - nb_additional_bits: &[u8], + codes: &Codes, table_log: u32, + workspace: &mut Workspace, ) -> Result<()> { let table_size = 1usize << table_log; let table_decode = self.table.sub_mut(..table_size)?; self.table_log = table_log; self.with_more_bits = normalized_counter .iter() - .zip(nb_additional_bits) - .filter(|&(_, &nb_bits)| nb_bits != 0) + .zip(codes) + .filter(|&(_, &code)| C::from_bits(code).has_additional_bits()) .map(|(&count, _)| u32::from(count.unsigned_abs())) .sum::() << (9 - table_log.min(9)); - let Spread { - symbols, - mut symbol_next, - } = Spread::new(normalized_counter, table_log)?; - - for (cell, &symbol) in table_decode.iter_mut().zip(&symbols) { - let symbol = usize::from(symbol); - let next = symbol_next.at_mut(symbol)?; - let next_state = u32::from(*next); + workspace.spread(normalized_counter, table_log)?; + + for (cell, &symbol) in table_decode.iter_mut().zip(&workspace.symbols) { + let symbol = usize::from(symbol) % 64; + let (Some(next), Some(code)) = + (workspace.symbol_next.get_mut(symbol), codes.get(symbol)) + else { + continue; + }; + let next_state = usize::from(*next); *next = next.wrapping_add(1); - let nb_bits = table_log.wrapping_sub(highbit32(next_state)); - *cell = C::pack(SeqSymbol { - next_state: (next_state.wrapping_shl(nb_bits) as usize).wrapping_sub(table_size) - as u16, - nb_additional_bits: nb_additional_bits.at(symbol)?, - nb_bits: nb_bits as u8, - base_value: base_value.at(symbol)?, - }); + let high_bit = HIGH_BIT.get(next_state % (2 * MAX_TABLE_SIZE)).copied(); + let nb_bits = table_log.wrapping_sub(u32::from(high_bit.unwrap_or(0))); + let next_state = next_state.wrapping_shl(nb_bits).wrapping_sub(table_size) as u64; + let nb_bits = C::NB_BITS.get(nb_bits as usize % 16).copied().unwrap_or(0); + *cell = C::from_bits(code | nb_bits | next_state << C::NEXT_STATE_SHIFT); } Ok(()) } From 0b8022ce1a549146845a27b270a83cb361c1f919 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:43:25 +0000 Subject: [PATCH 066/199] zstd: matches in a dictionary are of the common kind, and a thread's next decoder takes over the tables of its last --- src/zstd/decompress/block.rs | 191 ++++++++++++++++++++++++++++------- src/zstd/decompress/mod.rs | 23 +++-- 2 files changed, 166 insertions(+), 48 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index cbda3c5fa2b4..08d5fdf71774 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -5,7 +5,7 @@ use super::seq_table::{ Cell, Codes, LL_CODES, ML_CODES, OF_CODES, OffsetTable, SeqTable, SeqTableSources, SeqTables, TableSource, Workspace, }; -use core::cell::OnceCell; +use core::cell::{self, OnceCell}; use core::hint::{Locality, prefetch_read}; use crate::common::bitstream::BitDStream; @@ -16,7 +16,7 @@ use crate::common::{ REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, read_le16, read_le24, read_le32}; +use crate::util::{BUG, SliceExt, read_le16, read_le24, read_le32}; /// `blockProperties_t` pub(crate) struct BlockProperties { @@ -55,7 +55,6 @@ pub(crate) fn get_c_block_size(src: &[u8]) -> Result<(usize, BlockProperties)> { pub(crate) struct EntropyDTables { pub(crate) seq: SeqTables, pub(crate) huf_table: HufDTable, - pub(crate) rep: [u32; ZSTD_REP_NUM], pub(crate) workspace: Workspace, } @@ -64,7 +63,6 @@ impl EntropyDTables { Self { seq: SeqTables::new(), huf_table: HufDTable::new(), - rep: REP_START_VALUE, workspace: Workspace::new(), } } @@ -197,7 +195,9 @@ struct Literals { /// The state of a `ZSTD_DCtx` that lasts from block to block. pub(crate) struct BlockDecoder { - pub(crate) entropy: Box, + /// There once a block takes it. + scratch: Option>, + pub(crate) rep: [u32; ZSTD_REP_NUM], pub(crate) seq_sources: SeqTableSources, pub(crate) huf_source: TableSource, /// There is a Huffman table to repeat. @@ -205,22 +205,24 @@ pub(crate) struct BlockDecoder { /// There are sequence tables to repeat. pub(crate) fse_entropy: bool, pub(crate) ddict_is_cold: bool, +} + +/// What it takes to decode blocks that are compressed. +struct Scratch { + entropy: EntropyDTables, /// Grows to the literals of the largest block so far, and `WILDCOPY_OVERLENGTH`. lit_buffer: Vec, } -impl BlockDecoder { - pub(crate) fn new() -> Self { - Self { - entropy: Box::new(EntropyDTables::new()), - seq_sources: SeqTableSources::all(TableSource::Local), - huf_source: TableSource::Local, - lit_entropy: false, - fse_entropy: false, - ddict_is_cold: false, - lit_buffer: Vec::new(), - } - } +thread_local! { + /// That of the last decoder that was dropped, for the next one. Nothing depends on what is in it, and it takes + /// longer to clear a new one than to decode a small frame. + static SPARE_SCRATCH: cell::Cell>> = const { cell::Cell::new(None) }; +} + +impl Scratch { + /// A thread keeps no more than that for its next decoder. + const MAX_SPARE_LIT_BUFFER: usize = 1 << 14; fn lit_buffer(&mut self, lit_size: usize) -> Result<&mut [u8]> { let needed = lit_size + WILDCOPY_OVERLENGTH; @@ -232,6 +234,44 @@ impl BlockDecoder { } self.lit_buffer.sub_mut(..lit_size) } +} + +impl Drop for BlockDecoder { + fn drop(&mut self) { + if let Some(mut scratch) = self.scratch.take() { + if scratch.lit_buffer.capacity() > Scratch::MAX_SPARE_LIT_BUFFER { + scratch.lit_buffer = Vec::new(); + } + // Not if the thread is at its end. + let _ = SPARE_SCRATCH.try_with(|spare| spare.set(Some(scratch))); + } + } +} + +impl BlockDecoder { + pub(crate) const fn new() -> Self { + Self { + scratch: None, + rep: REP_START_VALUE, + seq_sources: SeqTableSources::all(TableSource::Local), + huf_source: TableSource::Local, + lit_entropy: false, + fse_entropy: false, + ddict_is_cold: false, + } + } + + fn scratch(&mut self) -> &mut Scratch { + self.scratch.get_or_insert_with(|| { + let spare = SPARE_SCRATCH.try_with(cell::Cell::take).ok().flatten(); + spare.unwrap_or_else(|| { + Box::new(Scratch { + entropy: EntropyDTables::new(), + lit_buffer: Vec::new(), + }) + }) + }) + } /// `ZSTD_decodeLiteralsBlock` fn decode_literals_block( @@ -285,12 +325,12 @@ impl BlockDecoder { let compressed = src.span(lh_size, lit_c_size)?; let huf_source = self.huf_source; - self.lit_buffer(lit_size)?; - let Self { + let scratch = self.scratch(); + scratch.lit_buffer(lit_size)?; + let Scratch { entropy, lit_buffer, - .. - } = self; + } = scratch; let dst = lit_buffer.sub_mut(..lit_size)?; let huf_result = if lit_enc_type == SymbolEncodingType::Repeat { let table = match (huf_source, dict_entropy) { @@ -347,7 +387,8 @@ impl BlockDecoder { if lit_size + lh_size > src.len() { return Err(ErrorCode::CorruptionDetected); } - self.lit_buffer(lit_size)? + self.scratch() + .lit_buffer(lit_size)? .copy_from_slice(src.span(lh_size, lit_size)?); return Ok(Literals { c_size: lh_size + lit_size, @@ -386,7 +427,7 @@ impl BlockDecoder { if expected_write_size < lit_size { return Err(ErrorCode::DstSizeTooSmall); } - self.lit_buffer(lit_size)?.fill(src.at(lh_size)?); + self.scratch().lit_buffer(lit_size)?.fill(src.at(lh_size)?); Ok(Literals { c_size: lh_size + 1, size: lit_size, @@ -438,39 +479,44 @@ impl BlockDecoder { ip += 1; let fse_entropy = self.fse_entropy; + self.scratch(); + let Some(scratch) = &mut self.scratch else { + return Err(BUG); + }; + let EntropyDTables { seq, workspace, .. } = &mut scratch.entropy; ip += build_seq_table( - &mut self.entropy.seq.ll, + &mut seq.ll, &mut self.seq_sources.ll, SymbolEncodingType::from_bits(modes >> 6), MAX_LL, LL_FSE_LOG, src.sub(ip..)?, &LL_CODES, - &mut self.entropy.workspace, + workspace, fse_entropy, ) .map_err(|_| ErrorCode::CorruptionDetected)?; ip += build_seq_table( - &mut self.entropy.seq.of, + &mut seq.of, &mut self.seq_sources.of, SymbolEncodingType::from_bits(modes >> 4), MAX_OFF, OFF_FSE_LOG, src.sub(ip..)?, &OF_CODES, - &mut self.entropy.workspace, + workspace, fse_entropy, ) .map_err(|_| ErrorCode::CorruptionDetected)?; ip += build_seq_table( - &mut self.entropy.seq.ml, + &mut seq.ml, &mut self.seq_sources.ml, SymbolEncodingType::from_bits(modes >> 2), MAX_ML, ML_FSE_LOG, src.sub(ip..)?, &ML_CODES, - &mut self.entropy.workspace, + workspace, fse_entropy, ) .map_err(|_| ErrorCode::CorruptionDetected)?; @@ -512,13 +558,21 @@ impl BlockDecoder { return Err(ErrorCode::DstSizeTooSmall); } + self.scratch(); let Self { + scratch: Some(scratch), + rep, + .. + } = self + else { + return Err(BUG); + }; + let Scratch { entropy, lit_buffer, - .. - } = self; - let EntropyDTables { seq, rep, .. } = &mut **entropy; - let tables = seq.gather(&mut self.seq_sources, dict_entropy.map(|dict| &dict.seq)); + } = &mut **scratch; + let dict_tables = dict_entropy.map(|dict| &dict.seq); + let tables = entropy.seq.gather(&mut self.seq_sources, dict_tables); // With matches far back in a long history, it pays to know the sequences ahead of time. Nothing is prefetched // here, but which of two problems in a damaged block is reported depends on the order. @@ -1250,10 +1304,36 @@ impl<'a> Executor<'_, 'a> { /// /// The last offsets are those of the 3 sequences before, which are rewritten for that where it takes it. #[inline(never)] - fn exec_batch(&mut self, sequences: &mut [RawSequence], mut index: usize) -> usize { + fn exec_batch(&mut self, sequences: &mut [RawSequence], index: usize) -> usize { + self.exec_batch_with::(sequences, index, &[]) + } + + /// The same where matches in `dict`, which comes before the output, are of the common kind too. + #[inline(never)] + fn exec_batch_after_dict( + &mut self, + sequences: &mut [RawSequence], + index: usize, + dict: &[u8], + ) -> usize { + self.exec_batch_with::(sequences, index, dict) + } + + #[inline(always)] + fn exec_batch_with( + &mut self, + sequences: &mut [RawSequence], + mut index: usize, + dict: &[u8], + ) -> usize { // Positions fit in 32 bits, so that the compiler sees that nothing added to them overflows. const MAX_BACK: usize = 3 << 30; let start = self.prefix_start.max(self.op.saturating_sub(MAX_BACK)); + let dict = if start == self.prefix_start { + dict + } else { + &[] + }; let buf = self.out.bytes(); let end = buf.len().min(start + u32::MAX as usize); let buf = buf.get_mut(start..end).unwrap_or_default(); @@ -1281,18 +1361,45 @@ impl<'a> Executor<'_, 'a> { || lit_end + WILDCOPY_OVERLENGTH > lits.len() || lit_length > 32 || match_length > 64 - || offset_value as usize > o_lit_end + 3 + || !DICT && offset_value as usize > o_lit_end + 3 { break; } - let match_pos = o_lit_end + 3 - offset_value as usize; let (Some(to), Some(from)) = ( buf.get_mut(op as usize..op as usize + 32), lits.get(lit_pos as usize..lit_pos as usize + 32), ) else { break; }; + if DICT && offset_value as usize > o_lit_end + 3 { + let back = offset_value as usize - 3 - o_lit_end; + let Some(match_pos) = dict.len().checked_sub(back) else { + break; + }; + let Some(([first, second], _)) = dict + .get(match_pos..match_pos + 64) + .map(<[u8]>::as_chunks::<32>) + else { + break; + }; + to.copy_from_slice(from); + let Some(to) = buf.get_mut(o_lit_end..o_lit_end + 32) else { + break; + }; + to.copy_from_slice(first); + if match_length > 32 { + let Some(to) = buf.get_mut(o_lit_end + 32..o_lit_end + 64) else { + break; + }; + to.copy_from_slice(second); + } + op = o_match_end as u32; + lit_pos = lit_end as u32; + index += 1; + continue; + } to.copy_from_slice(from); + let match_pos = o_lit_end + 3 - offset_value as usize; if !copy_16(buf, match_pos, o_lit_end) || !copy_16(buf, match_pos + 16, o_lit_end + 16) { break; @@ -1377,8 +1484,9 @@ impl<'a> Executor<'_, 'a> { let more_bits = tables.ll.with_more_bits + tables.ml.with_more_bits > 512 / 10; let mut remaining = nb_seq; - // Long copies go on while the next sequences are decoded, if that is what comes next. - if tables.ml.with_more_bits > 512 / 2 && !self.split { + // Long copies go on while the next sequences are decoded, if that is what comes next. And a batch of a few + // sequences is not worth it. + if (tables.ml.with_more_bits > 512 / 2 || nb_seq <= 8) && !self.split { let mut exec = self.fork(); while remaining > ahead.len() { let sequence = seq_state.decode_sequence(remaining == 1); @@ -1478,7 +1586,12 @@ impl<'a> Executor<'_, 'a> { } let mut index = 3; loop { - index = self.exec_batch(sequences, index); + index = match self.ext_dict { + ExtDict::Dict(dict) if !dict.is_empty() => { + self.exec_batch_after_dict(sequences, index, dict) + } + _ => self.exec_batch(sequences, index), + }; if index >= sequences.len() { break; } diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index cb1ebeb041f6..f44fbbd7274c 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -19,7 +19,7 @@ use crate::common::{ WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_DID_FIELD_SIZE, ZSTD_FCS_FIELD_SIZE, ZSTD_FRAMEHEADERSIZE_MAX, ZSTD_FRAMEIDSIZE, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGIC_SKIPPABLE_MASK, ZSTD_MAGIC_SKIPPABLE_START, - ZSTD_MAGICNUMBER, ZSTD_SKIPPABLEHEADERSIZE, ZSTD_WINDOWLOG_ABSOLUTEMIN, + ZSTD_MAGICNUMBER, ZSTD_REP_NUM, ZSTD_SKIPPABLEHEADERSIZE, ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WINDOWLOG_LIMIT_DEFAULT, ZSTD_WINDOWLOG_MAX, ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, }; @@ -324,8 +324,8 @@ pub(crate) fn find_decompressed_size(mut src: &[u8]) -> u64 { // ── dictionaries ────────────────────────────────────────────────────────── -/// `ZSTD_loadDEntropy`: reads the tables at the start of a dictionary, and returns where its content starts. -fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result { +/// `ZSTD_loadDEntropy`: reads the tables at the start of a dictionary, and returns the last offsets that it sets. +fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result<[u32; ZSTD_REP_NUM]> { const CORRUPTED: ErrorCode = ErrorCode::DictionaryCorrupted; if dict.len() <= 8 { return Err(CORRUPTED); @@ -382,14 +382,15 @@ fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result { return Err(CORRUPTED); } let dict_content_size = dict.len() - (pos + 12); - for rep in &mut entropy.rep { + let mut reps = [0; ZSTD_REP_NUM]; + for rep in &mut reps { *rep = read_le32(dict, pos)?; pos += 4; if *rep == 0 || *rep as usize > dict_content_size { return Err(CORRUPTED); } } - Ok(pos) + Ok(reps) } /// `ZSTD_dictContentType_e` @@ -406,6 +407,8 @@ pub struct DDict<'a> { content: Cow<'a, [u8]>, /// The tables, if the dictionary has some. entropy: Option>, + /// The last offsets that a frame starts with. + rep: [u32; ZSTD_REP_NUM], dict_id: u32, } @@ -415,6 +418,7 @@ impl<'a> DDict<'a> { let mut ddict = Self { content, entropy: None, + rep: REP_START_VALUE, dict_id: 0, }; let dict = &*ddict.content; @@ -426,7 +430,8 @@ impl<'a> DDict<'a> { } ddict.dict_id = read_le32(dict, ZSTD_FRAMEIDSIZE)?; let mut entropy = Box::new(EntropyDTables::new()); - load_d_entropy(&mut entropy, dict).map_err(|_| ErrorCode::DictionaryCorrupted)?; + ddict.rep = + load_d_entropy(&mut entropy, dict).map_err(|_| ErrorCode::DictionaryCorrupted)?; ddict.entropy = Some(entropy); Ok(ddict) } @@ -725,7 +730,7 @@ impl DState { self.block.fse_entropy = false; self.dict_id = 0; self.b_type = BlockType::Reserved; - self.block.entropy.rep = REP_START_VALUE; + self.block.rep = REP_START_VALUE; self.block.seq_sources = SeqTableSources::all(TableSource::Local); self.block.huf_source = TableSource::Local; @@ -733,12 +738,12 @@ impl DState { if let Some(ddict) = ddict { self.dict_id = ddict.dict_id; self.dict_end_address = ddict.end_address(); - if let Some(entropy) = &ddict.entropy { + self.block.rep = ddict.rep; + if ddict.entropy.is_some() { self.block.lit_entropy = true; self.block.fse_entropy = true; self.block.seq_sources = SeqTableSources::all(TableSource::Dict); self.block.huf_source = TableSource::Dict; - self.block.entropy.rep = entropy.rep; } } } From f4b98cd265fc649c5fadd8e78a33385b60ef015d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:53:27 +0000 Subject: [PATCH 067/199] zstd: the weights of a Huffman table, a lone stream of literals and the tables of sequences wait less for what they have just computed --- src/zstd/common/entropy.rs | 54 +++- src/zstd/common/fse_decompress.rs | 394 ++++++++++++++++++++---------- src/zstd/decompress/block.rs | 32 ++- src/zstd/decompress/huf.rs | 83 +++++-- src/zstd/decompress/mod.rs | 25 +- src/zstd/decompress/seq_table.rs | 149 +++-------- 6 files changed, 444 insertions(+), 293 deletions(-) diff --git a/src/zstd/common/entropy.rs b/src/zstd/common/entropy.rs index 83502cfbec66..5b7fb05bd5cb 100644 --- a/src/zstd/common/entropy.rs +++ b/src/zstd/common/entropy.rs @@ -1,6 +1,6 @@ //! `entropy_common.c`: the headers of FSE and Huffman tables. -use super::fse_decompress::fse_decompress_weights; +use super::fse_decompress::{Spread, fse_decompress_weights}; use super::{FSE_MIN_TABLELOG, FSE_TABLELOG_ABSOLUTE_MAX, HUF_TABLELOG_MAX}; use crate::error::{ErrorCode, Result}; use crate::util::{SliceExt, highbit32, read_le32}; @@ -161,11 +161,30 @@ pub(crate) struct HufStats { pub(crate) table_log: u32, } +/// `weights`, which are fewer than 4, then zeroes. +fn pad(weights: &[u8]) -> [u8; 4] { + let mut padded = [0; 4]; + for (to, &weight) in padded.iter_mut().zip(weights) { + *to = weight; + } + padded +} + /// `HUF_readStats` pub(crate) fn huf_read_stats( huff_weight: &mut [u8; 256], rank_stats: &mut [u32; HUF_TABLELOG_MAX as usize + 1], src: &[u8], +) -> Result { + huf_read_stats_wksp(huff_weight, rank_stats, src, &mut Spread::new()) +} + +/// `HUF_readStats_wksp` +pub(crate) fn huf_read_stats_wksp( + huff_weight: &mut [u8; 256], + rank_stats: &mut [u32; HUF_TABLELOG_MAX as usize + 1], + src: &[u8], + spread: &mut Spread, ) -> Result { let Some((&first, rest)) = src.split_first() else { return Err(ErrorCode::SrcSizeWrong); @@ -191,17 +210,34 @@ pub(crate) fn huf_read_stats( if i_size + 1 > src.len() { return Err(ErrorCode::SrcSizeWrong); } - o_size = fse_decompress_weights(huff_weight.sub_mut(..255)?, rest.sub(..i_size)?)?; + o_size = fse_decompress_weights(huff_weight.sub_mut(..255)?, rest.sub(..i_size)?, spread)?; } - rank_stats.fill(0); + // Four counts of each weight: one that is incremented again and again has to be waited for. + let mut counts = [[0u32; 16]; 4]; let mut weight_total: u32 = 0; - for &weight in huff_weight.sub(..o_size)? { - let Some(rank) = rank_stats.get_mut(usize::from(weight)) else { - return Err(ErrorCode::CorruptionDetected); - }; - *rank += 1; - weight_total += (1 << weight) >> 1; + let mut max_weight = 0; + let (fours, others) = huff_weight.sub(..o_size)?.as_chunks::<4>(); + for weights in fours.iter().chain(&[pad(others)]) { + for (&weight, counts) in weights.iter().zip(&mut counts) { + if let Some(count) = counts.get_mut(usize::from(weight) % 16) { + *count += 1; + } + weight_total += (1 << (weight % 16)) >> 1; + max_weight = max_weight.max(weight); + } + } + if u32::from(max_weight) > HUF_TABLELOG_MAX { + return Err(ErrorCode::CorruptionDetected); + } + let [counts0, counts1, counts2, counts3] = &counts; + for (weight, rank) in rank_stats.iter_mut().enumerate() { + *rank = + counts0.at(weight)? + counts1.at(weight)? + counts2.at(weight)? + counts3.at(weight)?; + } + // Those that complete the last four. + if let Some(rank) = rank_stats.first_mut() { + *rank -= (4 - others.len()) as u32; } if weight_total == 0 { return Err(ErrorCode::CorruptionDetected); diff --git a/src/zstd/common/fse_decompress.rs b/src/zstd/common/fse_decompress.rs index a6fd62b9c6e1..9d83c5b935b1 100644 --- a/src/zstd/common/fse_decompress.rs +++ b/src/zstd/common/fse_decompress.rs @@ -1,30 +1,165 @@ //! `fse_decompress.c`. zstd only decodes the weights of a Huffman table this way: sequences have their own decoder. -use super::bitstream::{BitDStream, DStreamStatus}; +use super::bitstream::BitDStream; use super::entropy::read_ncount; use super::{FSE_MAX_SYMBOL_VALUE, FSE_MAX_TABLELOG, fse_table_step}; use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, highbit32}; +use crate::util::{SliceExt, write_le64}; /// The largest table log of the weights of a Huffman table. const MAX_LOG: u32 = 6; /// `FSE_decode_t` -#[derive(Clone, Copy, Default)] +#[derive(Clone, Copy)] struct FseDecode { - new_state: u16, + new_state: u8, symbol: u8, nb_bits: u8, + /// As many bits set as `nb_bits`. + mask: u8, +} + +impl FseDecode { + const EMPTY: Self = Self { + new_state: 0, + symbol: 0, + nb_bits: 0, + mask: 0, + }; } /// `FSE_DTable` struct FseDTable { table_log: u32, - /// Every state reads at least one bit. - fast_mode: bool, table: [FseDecode; 1 << MAX_LOG], } +/// Where a state leads: to `state` and what the next `nb_bits` are. +pub(crate) struct NextState { + pub(crate) state: u16, + pub(crate) nb_bits: u8, +} + +impl NextState { + /// That of the `n`th state of a symbol that has `count` of them, in a table of `size`, for `2 * size + count + n`. + pub(crate) const fn at(index: u32) -> Self { + let mut next = Self { + state: 0, + nb_bits: 0, + }; + if index >= 4 { + let table_log = index.ilog2() - 1; + let next_state = index - (2 << table_log); + if next_state != 0 { + let nb_bits = table_log - next_state.ilog2(); + next.state = ((next_state << nb_bits) - (1 << table_log)) as u16; + next.nb_bits = nb_bits as u8; + } + } + next + } +} + +/// [`NextState::at`] as an [`FseDecode`] of symbol 0. +static NEXT_STATES: [FseDecode; 4 << MAX_LOG] = { + let mut all = [FseDecode::EMPTY; 4 << MAX_LOG]; + let mut left: &mut [FseDecode] = &mut all; + let mut index = 0; + while let [decode, others @ ..] = left { + let NextState { state, nb_bits } = NextState::at(index); + *decode = FseDecode { + new_state: state as u8, + symbol: 0, + nb_bits, + mask: ((1u32 << nb_bits) - 1) as u8, + }; + index += 1; + left = others; + } + all +}; + +/// The largest table that [`Spread`] is for. +pub(crate) const MAX_SPREAD_SIZE: usize = 1 << 9; + +/// Which symbol each state of a table decodes: what tables are built from. +#[derive(Clone)] +pub(crate) struct Spread { + /// The symbols, each as many times as it has states. + in_order: [u8; MAX_SPREAD_SIZE + 8], + pub(crate) symbols: [u8; MAX_SPREAD_SIZE], + /// For each symbol, what to give [`NextState::at`] for its first state. + pub(crate) symbol_next: [u16; FSE_MAX_SYMBOL_VALUE as usize + 1], + /// The symbols of low probability. + rare: [u8; FSE_MAX_SYMBOL_VALUE as usize + 1], + /// Room for the counts that all that is made from. + pub(crate) counts: [i16; FSE_MAX_SYMBOL_VALUE as usize + 1], +} + +impl Spread { + pub(crate) const fn new() -> Self { + Self { + in_order: [0; _], + symbols: [0; _], + symbol_next: [0; _], + rare: [0; _], + counts: [0; _], + } + } + + /// The first two passes of `FSE_buildDTable` and `ZSTD_buildFSETable`, with few branches to mispredict, for the + /// first `nb_symbols` of `counts`. They add up to the size of the table, which is at most [`MAX_SPREAD_SIZE`]. + pub(crate) fn set(&mut self, nb_symbols: usize, table_log: u32) -> Result<()> { + let table_size = 1usize << table_log; + let mut nb_rare = 0usize; + let mut nb_in_order = 0usize; + let mut eight_times = 0u64; + for ((symbol, &count), next) in self + .counts + .sub(..nb_symbols)? + .iter() + .enumerate() + .zip(&mut self.symbol_next) + { + let is_rare = count < 0; + if let Some(rare) = self.rare.get_mut(nb_rare) { + *rare = symbol as u8; + } + nb_rare += usize::from(is_rare); + *next = (2 * table_size) as u16 + if is_rare { 1 } else { count as u16 }; + let count = if is_rare { 0 } else { count as usize }; + write_le64(&mut self.in_order, nb_in_order, eight_times)?; + if count > 8 { + for done in (8..count).step_by(8) { + write_le64(&mut self.in_order, nb_in_order + done, eight_times)?; + } + } + nb_in_order += count; + eight_times += 0x0101_0101_0101_0101; + } + + // The states from here on are those of the rare symbols. Others that would go there go to the next place. + let first_rare = table_size - nb_rare.min(table_size); + let table_mask = table_size - 1; + let step = fse_table_step(table_size); + let symbols = self.symbols.sub_mut(..table_size)?; + let mut position = 0usize; + let mut from = 0usize; + for _ in 0..table_size / 2 { + for position in [position, (position + step) & table_mask] { + let symbol = self.in_order.get(from % MAX_SPREAD_SIZE).copied(); + symbols.put(position, symbol.unwrap_or(0))?; + from += usize::from(position < first_rare); + } + position = (position + 2 * step) & table_mask; + } + for (symbol, &rare) in symbols.iter_mut().rev().zip(self.rare.sub(..nb_rare)?) { + *symbol = rare; + } + Ok(()) + } +} + // The C code carves its tables out of a workspace of `HUF_READ_STATS_WORKSPACE_SIZE` bytes, and refuses the headers // whose tables do not fit. const fn dtable_size_u32(table_log: u32) -> usize { @@ -44,19 +179,14 @@ const fn decompress_wksp_size(table_log: u32, max_symbol_value: u32) -> usize { const WKSP_SIZE: usize = decompress_wksp_size(MAX_LOG, super::HUF_TABLELOG_MAX - 1); const NCOUNT_SIZE: usize = size_of::() * (FSE_MAX_SYMBOL_VALUE as usize + 1); -/// `FSE_buildDTable_internal` +/// `FSE_buildDTable_internal`, from `spread.counts`. fn build_dtable( dt: &mut FseDTable, - normalized_counter: &[i16], + spread: &mut Spread, max_symbol_value: u32, table_log: u32, wksp_size: usize, ) -> Result<()> { - let mut symbol_next = [0u16; FSE_MAX_SYMBOL_VALUE as usize + 1]; - let max_sv1 = max_symbol_value as usize + 1; - let table_size = 1usize << table_log; - let mut high_threshold = table_size - 1; - if build_dtable_wksp_size(table_log, max_symbol_value) > wksp_size { return Err(ErrorCode::MaxSymbolValueTooLarge); } @@ -66,139 +196,161 @@ fn build_dtable( if table_log > FSE_MAX_TABLELOG { return Err(ErrorCode::TableLogTooLarge); } - let counts = normalized_counter.sub(..max_sv1)?; - dt.table_log = table_log; - dt.fast_mode = true; - let large_limit = 1i16 << (table_log - 1); - for (s, (&count, next)) in counts.iter().zip(&mut symbol_next).enumerate() { - if count == -1 { - dt.table.at_mut(high_threshold)?.symbol = s as u8; - high_threshold = high_threshold.wrapping_sub(1); - *next = 1; - } else { - if count >= large_limit { - dt.fast_mode = false; - } - *next = count as u16; - } - } - - let table_mask = table_size - 1; - let step = fse_table_step(table_size); - let mut position = 0usize; - for (s, &count) in counts.iter().enumerate() { - for _ in 0..count { - dt.table.at_mut(position)?.symbol = s as u8; - position = (position + step) & table_mask; - while position > high_threshold { - position = (position + step) & table_mask; - } - } - } - if position != 0 { - return Err(ErrorCode::Generic); - } - - for entry in dt.table.sub_mut(..table_size)? { - let next = symbol_next.at_mut(usize::from(entry.symbol))?; - let next_state = u32::from(*next); + spread.set(max_symbol_value as usize + 1, table_log)?; + for (decode, &symbol) in dt + .table + .sub_mut(..1 << table_log)? + .iter_mut() + .zip(&spread.symbols) + { + let Some(next) = spread.symbol_next.get_mut(usize::from(symbol)) else { + continue; + }; + let next_state = NEXT_STATES.get(usize::from(*next) % NEXT_STATES.len()); *next = next.wrapping_add(1); - entry.nb_bits = (table_log - highbit32(next_state)) as u8; - entry.new_state = ((next_state << entry.nb_bits) as usize).wrapping_sub(table_size) as u16; + *decode = FseDecode { + symbol, + ..*next_state.unwrap_or(&FseDecode::EMPTY) + }; } Ok(()) } -/// `FSE_DState_t` -struct FseDState { - state: usize, +/// A stream that two states take turns to read. +struct Decoder<'a> { + table: &'a [FseDecode; 1 << MAX_LOG], + /// The state that decodes the next symbol, then the other one. + states: [u32; 2], + /// Zeroes, then the stream. + src: &'a [u8; PADDING + MAX_SRC_SIZE], + /// How many bits are left to read of `src`. + position: u32, } -impl FseDState { +/// No less than is read without a look at what is left, and 8 bytes. +const PADDING: usize = 16; +/// `Decoder::position` at the start of the stream. +const START: u32 = 8 * PADDING as u32; +/// What the size of a stream of weights is below. +const MAX_SRC_SIZE: usize = 128; + +impl Decoder<'_> { + /// The next 57 bits or more, from the highest on. + #[inline(always)] + fn bits(&self) -> u64 { + let end = (self.position as usize).div_ceil(8); + let container = (self.src.get(end.wrapping_sub(8)..end)) + .and_then(|bytes| bytes.try_into().ok()) + .map_or(0, u64::from_le_bytes); + container.rotate_left(self.position.wrapping_neg() & 7) + } + /// `FSE_initDState` - #[inline] - fn new(bit_d: &mut BitDStream<'_>, dt: &FseDTable) -> Self { - let state = bit_d.read_bits(dt.table_log) as usize; - bit_d.reload(); - Self { state } + fn read_state(&mut self, table_log: u32) -> u32 { + let bits = self.bits().rotate_left(table_log); + self.position -= table_log; + bits as u32 & ((1 << table_log) - 1) } - /// `FSE_decodeSymbol` and `FSE_decodeSymbolFast` + /// Decodes as many symbols as `out` takes, which are few enough to take no more than 57 bits. `positions` gets + /// `position` after each. #[inline(always)] - fn decode_symbol( - &mut self, - bit_d: &mut BitDStream<'_>, - dt: &FseDTable, - ) -> Result { - let info = dt.table.at(self.state)?; - let nb_bits = u32::from(info.nb_bits); - let low_bits = if FAST { - bit_d.read_bits_fast(nb_bits) - } else { - bit_d.read_bits(nb_bits) - }; - self.state = usize::from(info.new_state) + low_bits as usize; - Ok(info.symbol) + fn decode(&mut self, out: &mut [u8; N], positions: &mut [u32; N]) { + // The bits that are read go to the other end. + let mut bits = self.bits(); + let [mut state, mut other_state] = self.states; + for (symbol, position) in out.iter_mut().zip(positions) { + let decode = + (self.table.get(state as usize % (1 << MAX_LOG))).unwrap_or(&FseDecode::EMPTY); + *symbol = decode.symbol; + bits = bits.rotate_left(u32::from(decode.nb_bits)); + self.position = self.position.wrapping_sub(u32::from(decode.nb_bits)); + *position = self.position; + state = other_state; + other_state = u32::from(decode.new_state) + (bits as u32 & u32::from(decode.mask)); + } + self.states = [state, other_state]; + } + + fn decode_one(&mut self) -> u8 { + let mut symbol = [0]; + self.decode(&mut symbol, &mut [0]); + let [symbol] = symbol; + symbol + } + + /// Whether more was read than the stream has. + fn is_past_start(&self) -> bool { + self.position < START } } /// `FSE_decompress_usingDTable_generic` -fn decompress_using_dtable( - dst: &mut [u8], - src: &[u8], - dt: &FseDTable, -) -> Result { - let omax = dst.len(); - let mut op = 0usize; +fn decompress_using_dtable(dst: &mut [u8], src: &[u8], dt: &FseDTable) -> Result { + /// 8 symbols take at most 48 bits. + const GROUP: usize = 8; - let mut bit_d = BitDStream::new(src)?; - let mut state1 = FseDState::new(&mut bit_d, dt); - let mut state2 = FseDState::new(&mut bit_d, dt); - if bit_d.reload() == DStreamStatus::Overflow { + let end_mark = + BitDStream::new(src)?.bits_consumed - 8 * 8usize.saturating_sub(src.len()) as u32; + let mut padded = [0; PADDING + MAX_SRC_SIZE]; + padded.write_at(PADDING, src)?; + let mut decoder = Decoder { + table: &dt.table, + states: [0; 2], + src: &padded, + position: 8 * (PADDING + src.len()) as u32 - end_mark, + }; + decoder.states = [ + decoder.read_state(dt.table_log), + decoder.read_state(dt.table_log), + ]; + if decoder.is_past_start() { return Err(ErrorCode::CorruptionDetected); } - while (bit_d.reload() == DStreamStatus::Unfinished) & (op + 3 < omax) { - *dst.array_at_mut(op)? = [ - state1.decode_symbol::(&mut bit_d, dt)?, - state2.decode_symbol::(&mut bit_d, dt)?, - state1.decode_symbol::(&mut bit_d, dt)?, - state2.decode_symbol::(&mut bit_d, dt)?, - ]; - op += 4; + // All but the last symbol leave room for one more. + let omax = dst.len(); + let mut op = 0usize; + while let Some(group) = dst.get_mut(op..op + GROUP) + && let Some(group) = group.as_mut_array::() + && op + GROUP < omax + { + let mut positions = [0; GROUP]; + decoder.decode(group, &mut positions); + op += GROUP; + // The symbol after the one that takes more bits than there are is the last. + if let Some(extra) = (positions.iter().rev()).position(|&position| position >= START) { + if extra == 0 { + continue; + } + if extra == 1 { + dst.put(op, decoder.decode_one())?; + } + return Ok(op + 2 - extra); + } + return Ok(op + 2 - GROUP); } - loop { if op + 2 > omax { return Err(ErrorCode::DstSizeTooSmall); } - dst.put(op, state1.decode_symbol::(&mut bit_d, dt)?)?; + dst.put(op, decoder.decode_one())?; op += 1; - if bit_d.reload() == DStreamStatus::Overflow { - dst.put(op, state2.decode_symbol::(&mut bit_d, dt)?)?; - op += 1; - break; - } - - if op + 2 > omax { - return Err(ErrorCode::DstSizeTooSmall); - } - dst.put(op, state2.decode_symbol::(&mut bit_d, dt)?)?; - op += 1; - if bit_d.reload() == DStreamStatus::Overflow { - dst.put(op, state1.decode_symbol::(&mut bit_d, dt)?)?; - op += 1; - break; + if decoder.is_past_start() { + dst.put(op, decoder.decode_one())?; + return Ok(op + 1); } } - Ok(op) } /// `FSE_decompress_wksp_bmi2`, as `HUF_readStats` calls it. Returns how many weights there are. -pub(crate) fn fse_decompress_weights(dst: &mut [u8], src: &[u8]) -> Result { - let mut ncount = [0i16; FSE_MAX_SYMBOL_VALUE as usize + 1]; - let header = read_ncount(&mut ncount, FSE_MAX_SYMBOL_VALUE, src)?; +pub(crate) fn fse_decompress_weights( + dst: &mut [u8], + src: &[u8], + spread: &mut Spread, +) -> Result { + let header = read_ncount(&mut spread.counts, FSE_MAX_SYMBOL_VALUE, src)?; if header.table_log > MAX_LOG { return Err(ErrorCode::TableLogTooLarge); } @@ -211,20 +363,14 @@ pub(crate) fn fse_decompress_weights(dst: &mut [u8], src: &[u8]) -> Result(dst, body, &dtable) - } else { - decompress_using_dtable::(dst, body, &dtable) - } + decompress_using_dtable(dst, body, &dtable) } diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 08d5fdf71774..734c94d7dcef 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -3,15 +3,16 @@ use super::huf::{self, HufDTable}; use super::seq_table::{ Cell, Codes, LL_CODES, ML_CODES, OF_CODES, OffsetTable, SeqTable, SeqTableSources, SeqTables, - TableSource, Workspace, + TableSource, }; use core::cell::{self, OnceCell}; use core::hint::{Locality, prefetch_read}; use crate::common::bitstream::BitDStream; use crate::common::entropy::read_ncount; +use crate::common::fse_decompress::Spread; use crate::common::{ - BlockType, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MIN_CBLOCK_SIZE, + BlockType, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MAX_OFF, MIN_CBLOCK_SIZE, MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, MINMATCH, ML_FSE_LOG, OFF_FSE_LOG, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; @@ -55,7 +56,7 @@ pub(crate) fn get_c_block_size(src: &[u8]) -> Result<(usize, BlockProperties)> { pub(crate) struct EntropyDTables { pub(crate) seq: SeqTables, pub(crate) huf_table: HufDTable, - pub(crate) workspace: Workspace, + pub(crate) workspace: Spread, } impl EntropyDTables { @@ -63,7 +64,7 @@ impl EntropyDTables { Self { seq: SeqTables::new(), huf_table: HufDTable::new(), - workspace: Workspace::new(), + workspace: Spread::new(), } } } @@ -346,9 +347,19 @@ impl BlockDecoder { huf::decompress_4x_using_dtable(dst, compressed, table) } } else if single_stream { - huf::decompress_1x1(&mut entropy.huf_table, dst, compressed) + huf::decompress_1x1( + &mut entropy.huf_table, + dst, + compressed, + &mut entropy.workspace, + ) } else { - huf::decompress_4x_huf_only(&mut entropy.huf_table, dst, compressed) + huf::decompress_4x_huf_only( + &mut entropy.huf_table, + dst, + compressed, + &mut entropy.workspace, + ) }; if huf_result.is_err() { return Err(ErrorCode::CorruptionDetected); @@ -641,7 +652,7 @@ fn build_seq_table( max_log: u32, src: &[u8], codes: &Codes, - workspace: &mut Workspace, + workspace: &mut Spread, flag_repeat_table: bool, ) -> Result { match encoding_type { @@ -667,14 +678,13 @@ fn build_seq_table( Ok(0) } SymbolEncodingType::Compressed => { - let mut norm = [0i16; MAX_SEQ as usize + 1]; - let header = - read_ncount(&mut norm, max, src).map_err(|_| ErrorCode::CorruptionDetected)?; + let header = read_ncount(&mut workspace.counts, max, src) + .map_err(|_| ErrorCode::CorruptionDetected)?; if header.table_log > max_log { return Err(ErrorCode::CorruptionDetected); } table.build( - norm.sub(..header.max_symbol_value as usize + 1)?, + header.max_symbol_value as usize + 1, codes, header.table_log, workspace, diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index cd79c8e8e779..bc9e03ca2ec2 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -4,7 +4,8 @@ //! the way that is fastest here. The others are left to [`huf_exact`](super::huf_exact). use super::huf_exact::{self, TableType, select_decoder_x2}; -use crate::common::entropy::huf_read_stats; +use crate::common::entropy::huf_read_stats_wksp; +use crate::common::fse_decompress::Spread; use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; use crate::error::{ErrorCode, Result}; use crate::util::{BUG, SliceExt, read_le16}; @@ -281,8 +282,19 @@ impl HufDTable { /// Reads the description of a table at the start of `src`, and returns its size. The table is to decode /// `nb_literals` with. - fn read(&mut self, src: &[u8], table_type: TableType, nb_literals: usize) -> Result { - let stats = huf_read_stats(&mut self.code.weights, &mut self.code.rank_stats, src)?; + fn read( + &mut self, + src: &[u8], + table_type: TableType, + nb_literals: usize, + spread: &mut Spread, + ) -> Result { + let Code { + weights, + rank_stats, + .. + } = &mut self.code; + let stats = huf_read_stats_wksp(weights, rank_stats, src, spread)?; self.code.nb_symbols = stats.nb_symbols; self.code.table_log = stats.table_log; self.table_type = table_type; @@ -682,28 +694,49 @@ impl<'a> Stream<'a> { return; } let shift = 64 - log; - let (mut ip, mut bits, mut op) = (self.ip, self.bits, self.op); - while let Some(window) = self - .out - .get_mut(op as usize..op as usize + 3 * LOOKUPS + 1) + // With nothing else to do in the meantime, what counts is how soon the next lookup can start. So more bits + // come from where the last ones came from, which is known ahead of time. + let consumed = self.bits.trailing_zeros(); + // Where the bytes after those that `bits` was read from end. + let mut next = self.ip as usize; + let Some(Ok(container)) = self.src.get(next..next + 8).map(<[u8; 8]>::try_from) else { + return; + }; + // The next bits, from the highest on: `valid` of them, or somewhat more. + let mut bits = u64::from_le_bytes(container) << consumed; + let mut valid = 64 - consumed; + let mut op = self.op as usize; + while let Some(window) = (self.out.get_mut(op..op + 3 * LOOKUPS + 1)) .and_then(|window| window.as_mut_array::<{ 3 * LOOKUPS + 1 }>()) - && reload(self.src, &mut ip, &mut bits) + && next >= 16 + && let Some(Ok(more)) = self.src.get(next - 8..next).map(<[u8; 8]>::try_from) { + bits |= u64::from_le_bytes(more) >> (valid & 63); + let nb_bytes = (63 - (valid & 63)) >> 3; + next -= nb_bytes as usize; + valid += 8 * nb_bytes; let mut done = 0; for _ in 0..LOOKUPS { - let entry = table - .get((bits >> shift) as usize % TABLE_SIZE) - .copied() - .unwrap_or(0); - bits <<= (entry >> 24) & 63; + let entry = table.get((bits >> shift) as usize).copied().unwrap_or(0); + let nb_bits = (entry >> 24) & 63; + bits <<= nb_bits; + valid = valid.wrapping_sub(nb_bits); if let Some(to) = window.get_mut(done..done + 4) { to.copy_from_slice(&entry.to_le_bytes()); } done += (entry >> (24 + COUNT_SHIFT)) as usize; } - op += done as u32; + op += done; + } + let position = 8 * next + valid as usize; + let end = position.div_ceil(8); + if let Some(Ok(container)) = + (self.src.get(end.wrapping_sub(8)..end)).map(<[u8; 8]>::try_from) + { + self.ip = (end - 8) as u32; + self.bits = (u64::from_le_bytes(container) | 1) << ((8 * end - position) & 63); + self.op = op as u32; } - (self.ip, self.bits, self.op) = (ip, bits, op); } } @@ -805,8 +838,12 @@ impl<'a> Streams<'a> { } /// `HUF_readDTableX2_wksp`. Returns the size of the table's description in `src`. -pub(crate) fn read_dtable_x2(dtable: &mut HufDTable, src: &[u8]) -> Result { - dtable.read(src, TableType::X2, usize::MAX) +pub(crate) fn read_dtable_x2( + dtable: &mut HufDTable, + src: &[u8], + spread: &mut Spread, +) -> Result { + dtable.read(src, TableType::X2, usize::MAX, spread) } /// `HUF_decompress1X_usingDTable` @@ -822,8 +859,13 @@ pub(crate) fn decompress_1x_using_dtable( } /// `HUF_decompress1X1_DCtx_wksp` -pub(crate) fn decompress_1x1(dtable: &mut HufDTable, dst: &mut [u8], src: &[u8]) -> Result { - let h_size = dtable.read(src, TableType::X1, dst.len())?; +pub(crate) fn decompress_1x1( + dtable: &mut HufDTable, + dst: &mut [u8], + src: &[u8], + spread: &mut Spread, +) -> Result { + let h_size = dtable.read(src, TableType::X1, dst.len(), spread)?; if h_size >= src.len() { return Err(ErrorCode::SrcSizeWrong); } @@ -847,6 +889,7 @@ pub(crate) fn decompress_4x_huf_only( dtable: &mut HufDTable, dst: &mut [u8], src: &[u8], + spread: &mut Spread, ) -> Result { if dst.is_empty() { return Err(ErrorCode::DstSizeTooSmall); @@ -859,7 +902,7 @@ pub(crate) fn decompress_4x_huf_only( } else { TableType::X1 }; - let h_size = dtable.read(src, table_type, dst.len())?; + let h_size = dtable.read(src, table_type, dst.len(), spread)?; if h_size >= src.len() { return Err(ErrorCode::SrcSizeWrong); } diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index f44fbbd7274c..b66eeb12de48 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -333,16 +333,21 @@ fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result<[u32; ZST // The magic number and the ID. let mut pos = 8usize; - pos += huf::read_dtable_x2(&mut entropy.huf_table, dict.sub(pos..)?).map_err(|_| CORRUPTED)?; + pos += huf::read_dtable_x2( + &mut entropy.huf_table, + dict.sub(pos..)?, + &mut entropy.workspace, + ) + .map_err(|_| CORRUPTED)?; { - let mut ncount = [0i16; MAX_OFF as usize + 1]; - let header = read_ncount(&mut ncount, MAX_OFF, dict.sub(pos..)?).map_err(|_| CORRUPTED)?; + let header = read_ncount(&mut entropy.workspace.counts, MAX_OFF, dict.sub(pos..)?) + .map_err(|_| CORRUPTED)?; if header.max_symbol_value > MAX_OFF || header.table_log > OFF_FSE_LOG { return Err(CORRUPTED); } entropy.seq.of.build( - ncount.sub(..header.max_symbol_value as usize + 1)?, + header.max_symbol_value as usize + 1, &OF_CODES, header.table_log, &mut entropy.workspace, @@ -350,13 +355,13 @@ fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result<[u32; ZST pos += header.size; } { - let mut ncount = [0i16; MAX_ML as usize + 1]; - let header = read_ncount(&mut ncount, MAX_ML, dict.sub(pos..)?).map_err(|_| CORRUPTED)?; + let header = read_ncount(&mut entropy.workspace.counts, MAX_ML, dict.sub(pos..)?) + .map_err(|_| CORRUPTED)?; if header.max_symbol_value > MAX_ML || header.table_log > ML_FSE_LOG { return Err(CORRUPTED); } entropy.seq.ml.build( - ncount.sub(..header.max_symbol_value as usize + 1)?, + header.max_symbol_value as usize + 1, &ML_CODES, header.table_log, &mut entropy.workspace, @@ -364,13 +369,13 @@ fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result<[u32; ZST pos += header.size; } { - let mut ncount = [0i16; MAX_LL as usize + 1]; - let header = read_ncount(&mut ncount, MAX_LL, dict.sub(pos..)?).map_err(|_| CORRUPTED)?; + let header = read_ncount(&mut entropy.workspace.counts, MAX_LL, dict.sub(pos..)?) + .map_err(|_| CORRUPTED)?; if header.max_symbol_value > MAX_LL || header.table_log > LL_FSE_LOG { return Err(CORRUPTED); } entropy.seq.ll.build( - ncount.sub(..header.max_symbol_value as usize + 1)?, + header.max_symbol_value as usize + 1, &LL_CODES, header.table_log, &mut entropy.workspace, diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs index c30bed4bcb39..d24fcaf86a75 100644 --- a/src/zstd/decompress/seq_table.rs +++ b/src/zstd/decompress/seq_table.rs @@ -1,11 +1,12 @@ //! The FSE tables that sequences are decoded with: `ZSTD_seqSymbol` and `ZSTD_buildFSETable`. +use crate::common::fse_decompress::{MAX_SPREAD_SIZE, NextState, Spread}; use crate::common::{ LL_BITS, LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, ML_BITS, ML_DEFAULT_NORM_LOG, - OF_DEFAULT_NORM_LOG, fse_table_step, + OF_DEFAULT_NORM_LOG, }; use crate::error::Result; -use crate::util::{SliceExt, write_le64}; +use crate::util::SliceExt; #[rustfmt::skip] const LL_BASE: [u32; MAX_LL as usize + 1] = [ @@ -41,16 +42,16 @@ pub(crate) struct SeqSymbol { pub(crate) trait Cell: Copy + 'static { const EMPTY: Self; const EMPTY_REF: &'static Self; - /// Where `next_state` is in `to_bits`. - const NEXT_STATE_SHIFT: u32; - /// For each `nb_bits`, `to_bits` of a cell where nothing else is set but what follows from it. - const NB_BITS: &'static [u64; 16]; + /// [`NextState::at`] as `to_bits` of a cell where nothing else is set. + const NEXT_STATES: &'static [u64; 4 * MAX_TABLE_SIZE]; /// The fields one after the other, from the lowest bits on. fn from_bits(bits: u64) -> Self; fn has_additional_bits(self) -> bool; fn unpack(self) -> SeqSymbol; } +const MAX_TABLE_SIZE: usize = MAX_SPREAD_SIZE; + /// A state of the table of literal lengths or of match lengths. #[derive(Clone, Copy)] pub(crate) struct LengthCell { @@ -92,14 +93,14 @@ impl Cell for LengthCell { const EMPTY: Self = l(0, 0, 0, 0); const EMPTY_REF: &'static Self = &Self::EMPTY; - const NEXT_STATE_SHIFT: u32 = 16; - const NB_BITS: &'static [u64; 16] = &{ - let mut all = [0; 16]; + const NEXT_STATES: &'static [u64; 4 * MAX_TABLE_SIZE] = &{ + let mut all = [0; 4 * MAX_TABLE_SIZE]; let mut left: &mut [u64] = &mut all; - let mut nb_bits = 0; + let mut index = 0; while let [bits, others @ ..] = left { - *bits = l(0, 0, nb_bits, 0).to_bits(); - nb_bits += 1; + let NextState { state, nb_bits } = NextState::at(index); + *bits = l(state, 0, nb_bits, 0).to_bits(); + index += 1; left = others; } all @@ -169,14 +170,14 @@ impl Cell for OffsetCell { const EMPTY: Self = o(0, 0, 0); const EMPTY_REF: &'static Self = &Self::EMPTY; - const NEXT_STATE_SHIFT: u32 = 32; - const NB_BITS: &'static [u64; 16] = &{ - let mut all = [0; 16]; + const NEXT_STATES: &'static [u64; 4 * MAX_TABLE_SIZE] = &{ + let mut all = [0; 4 * MAX_TABLE_SIZE]; let mut left: &mut [u64] = &mut all; - let mut nb_bits = 0; + let mut index = 0; while let [bits, others @ ..] = left { - *bits = o(0, 0, nb_bits).to_bits() - o(0, 0, 0).to_bits(); - nb_bits += 1; + let NextState { state, nb_bits } = NextState::at(index); + *bits = o(state as u8, 0, nb_bits).to_bits() - o(0, 0, 0).to_bits(); + index += 1; left = others; } all @@ -247,93 +248,6 @@ pub(crate) static LL_CODES: Codes = length_codes(&LL_BASE, &LL_BITS); pub(crate) static OF_CODES: Codes = offset_codes(); pub(crate) static ML_CODES: Codes = length_codes(&ML_BASE, &ML_BITS); -const MAX_TABLE_SIZE: usize = 1 << 9; - -/// The position of the highest bit set. -static HIGH_BIT: [u8; 2 * MAX_TABLE_SIZE] = { - let mut table = [0; 2 * MAX_TABLE_SIZE]; - let (_, mut left) = table.split_at_mut(1); - let mut value = 1u32; - while let [high_bit, others @ ..] = left { - *high_bit = value.ilog2() as u8; - value += 1; - left = others; - } - table -}; - -/// What tables are built with. -#[derive(Clone)] -pub(crate) struct Workspace { - /// The symbols, each as many times as it has states. - in_order: [u8; MAX_TABLE_SIZE + 8], - /// Which symbol each state decodes. - symbols: [u8; MAX_TABLE_SIZE], - symbol_next: [u16; 64], - /// The symbols of low probability. - rare: [u8; 64], -} - -impl Workspace { - pub(crate) const fn new() -> Self { - Self { - in_order: [0; _], - symbols: [0; _], - symbol_next: [0; _], - rare: [0; _], - } - } - - /// The first two passes of `ZSTD_buildFSETable`, with few branches to mispredict. - fn spread(&mut self, normalized_counter: &[i16], table_log: u32) -> Result<()> { - let table_size = 1usize << table_log; - let mut nb_rare = 0usize; - let mut nb_in_order = 0usize; - let mut eight_times = 0u64; - for ((symbol, &count), next) in normalized_counter - .iter() - .enumerate() - .zip(&mut self.symbol_next) - { - let is_rare = count < 0; - if let Some(rare) = self.rare.get_mut(nb_rare % 64) { - *rare = symbol as u8; - } - nb_rare += usize::from(is_rare); - *next = if is_rare { 1 } else { count as u16 }; - let count = if is_rare { 0 } else { count as usize }; - write_le64(&mut self.in_order, nb_in_order, eight_times)?; - if count > 8 { - for done in (8..count).step_by(8) { - write_le64(&mut self.in_order, nb_in_order + done, eight_times)?; - } - } - nb_in_order += count; - eight_times += 0x0101_0101_0101_0101; - } - - // The states from here on are those of the rare symbols. Others that would go there go to the next place. - let first_rare = table_size - nb_rare.min(table_size); - let table_mask = table_size - 1; - let step = fse_table_step(table_size); - let symbols = self.symbols.sub_mut(..table_size)?; - let mut position = 0usize; - let mut from = 0usize; - for _ in 0..table_size / 2 { - for position in [position, (position + step) & table_mask] { - let symbol = self.in_order.get(from % MAX_TABLE_SIZE).copied(); - symbols.put(position, symbol.unwrap_or(0))?; - from += usize::from(position < first_rare); - } - position = (position + 2 * step) & table_mask; - } - for (symbol, &rare) in symbols.iter_mut().rev().zip(self.rare.sub(..nb_rare)?) { - *symbol = rare; - } - Ok(()) - } -} - /// A table of at most `N` states, which is a power of 2. #[derive(Clone)] pub(crate) struct SeqTable { @@ -409,40 +323,37 @@ impl SeqTable { } } - /// `ZSTD_buildFSETable`: `table_log` is such that the table fits, and `normalized_counter` adds up to its size. + /// `ZSTD_buildFSETable`: `table_log` is such that the table fits, and the first `nb_symbols` of `spread.counts` add + /// up to its size. pub(crate) fn build( &mut self, - normalized_counter: &[i16], + nb_symbols: usize, codes: &Codes, table_log: u32, - workspace: &mut Workspace, + spread: &mut Spread, ) -> Result<()> { let table_size = 1usize << table_log; let table_decode = self.table.sub_mut(..table_size)?; self.table_log = table_log; - self.with_more_bits = normalized_counter + self.with_more_bits = (spread.counts.sub(..nb_symbols)?) .iter() .zip(codes) .filter(|&(_, &code)| C::from_bits(code).has_additional_bits()) .map(|(&count, _)| u32::from(count.unsigned_abs())) .sum::() << (9 - table_log.min(9)); - workspace.spread(normalized_counter, table_log)?; + spread.set(nb_symbols, table_log)?; - for (cell, &symbol) in table_decode.iter_mut().zip(&workspace.symbols) { - let symbol = usize::from(symbol) % 64; + for (cell, &symbol) in table_decode.iter_mut().zip(&spread.symbols) { + let symbol = usize::from(symbol); let (Some(next), Some(code)) = - (workspace.symbol_next.get_mut(symbol), codes.get(symbol)) + (spread.symbol_next.get_mut(symbol), codes.get(symbol % 64)) else { continue; }; - let next_state = usize::from(*next); + let next_state = C::NEXT_STATES.get(usize::from(*next) % (4 * MAX_TABLE_SIZE)); *next = next.wrapping_add(1); - let high_bit = HIGH_BIT.get(next_state % (2 * MAX_TABLE_SIZE)).copied(); - let nb_bits = table_log.wrapping_sub(u32::from(high_bit.unwrap_or(0))); - let next_state = next_state.wrapping_shl(nb_bits).wrapping_sub(table_size) as u64; - let nb_bits = C::NB_BITS.get(nb_bits as usize % 16).copied().unwrap_or(0); - *cell = C::from_bits(code | nb_bits | next_state << C::NEXT_STATE_SHIFT); + *cell = C::from_bits(code | next_state.copied().unwrap_or(0)); } Ok(()) } From bf490c653ed31b3199ef70030a17970ec0e8cbd2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:58:02 +0000 Subject: [PATCH 068/199] zstd: a frame header is read into place, and the counts of a table without a look at the end of the input for each --- src/zstd/common/entropy.rs | 102 ++++++++++++++++++++++++++++++++++- src/zstd/decompress/block.rs | 32 +++++------ src/zstd/decompress/mod.rs | 43 ++++++++------- 3 files changed, 141 insertions(+), 36 deletions(-) diff --git a/src/zstd/common/entropy.rs b/src/zstd/common/entropy.rs index 5b7fb05bd5cb..50d2e0a188fa 100644 --- a/src/zstd/common/entropy.rs +++ b/src/zstd/common/entropy.rs @@ -18,11 +18,111 @@ pub(crate) fn read_ncount( normalized_counter: &mut [i16], max_symbol_value: u32, header: &[u8], +) -> Result { + match read_valid_ncount(normalized_counter, max_symbol_value, header) { + Some(ncount) => Ok(ncount), + None => read_any_ncount(normalized_counter, max_symbol_value, header), + } +} + +/// The same for a header that has nothing wrong with it and is not at the very end of `header`. +fn read_valid_ncount( + normalized_counter: &mut [i16], + max_symbol_value: u32, + header: &[u8], +) -> Option { + /// The bits from `position` on, from the lowest on: 57 of them or more, then zeroes. + #[inline(always)] + fn load(header: &[u8], position: usize) -> Option { + let bytes = header.get(position / 8..position / 8 + 8)?; + Some(u64::from_le_bytes(bytes.try_into().ok()?) >> (position % 8)) + } + /// After how many bits more are loaded. The 17 that are left are enough for a count. + const MAX_USED: u32 = 40; + + // As few as there usually are, and as many as can be cleared without a call. + let counts = normalized_counter.first_chunk_mut::<64>()?; + *counts = [0; 64]; + let counts = counts.get_mut(..(max_symbol_value as usize + 1).min(64))?; + // `bits` is what comes `used` bits after `position`. + let mut position = 0usize; + let mut bits = load(header, 0)?; + let table_log = (bits & 0xF) as u32 + FSE_MIN_TABLELOG; + if table_log > FSE_TABLELOG_ABSOLUTE_MAX { + return None; + } + bits >>= 4; + let mut used = 4; + let mut remaining = (1i32 << table_log) + 1; + let mut threshold = 1i32 << table_log; + let mut nb_bits = table_log + 1; + let mut symbol = 0usize; + let mut previous0 = false; + + while remaining > 1 { + if previous0 { + // Each pair of set bits stands for three more symbols of count 0. + loop { + position += used as usize; + used = 0; + bits = load(header, position)?; + let repeats = ((!bits).trailing_zeros() / 2).min(24); + symbol += 3 * repeats as usize; + bits >>= 2 * repeats; + used += 2 * repeats; + if repeats < 24 { + break; + } + } + symbol += (bits & 3) as usize; + bits >>= 2; + used += 2; + } + if used > MAX_USED { + position += used as usize; + used = 0; + bits = load(header, position)?; + } + + let max = 2 * threshold - 1 - remaining; + let low = bits as i32 & (threshold - 1); + let all = bits as i32 & (2 * threshold - 1); + let is_short = low < max; + let long = all - if all >= threshold { max } else { 0 }; + // The stored value is the count plus one, and a count of -1 takes one slot. + let count = if is_short { low } else { long } - 1; + let count_bits = nb_bits - u32::from(is_short); + bits >>= count_bits; + used += count_bits; + remaining -= count.abs(); + *counts.get_mut(symbol)? = count as i16; + symbol += 1; + previous0 = count == 0; + if remaining < threshold && remaining > 1 { + nb_bits = remaining.ilog2() + 1; + threshold = 1 << (nb_bits - 1); + } + } + if remaining != 1 { + return None; + } + Some(NCount { + size: (position + used as usize).div_ceil(8), + max_symbol_value: symbol as u32 - 1, + table_log, + }) +} + +/// The same as the C code does it. +fn read_any_ncount( + normalized_counter: &mut [i16], + max_symbol_value: u32, + header: &[u8], ) -> Result { if header.len() < 8 { let mut buffer = [0u8; 8]; buffer.write_at(0, header)?; - let ncount = read_ncount(normalized_counter, max_symbol_value, &buffer)?; + let ncount = read_any_ncount(normalized_counter, max_symbol_value, &buffer)?; if ncount.size > header.len() { return Err(ErrorCode::CorruptionDetected); } diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 734c94d7dcef..77c8f7fab00a 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -213,6 +213,8 @@ struct Scratch { entropy: EntropyDTables, /// Grows to the literals of the largest block so far, and `WILDCOPY_OVERLENGTH`. lit_buffer: Vec, + /// Sequences are decoded that many at a time, after 3 that only stand for the last offsets. + batch: [RawSequence; 3 + 128], } thread_local! { @@ -269,6 +271,7 @@ impl BlockDecoder { Box::new(Scratch { entropy: EntropyDTables::new(), lit_buffer: Vec::new(), + batch: [RawSequence::EMPTY; _], }) }) }) @@ -331,6 +334,7 @@ impl BlockDecoder { let Scratch { entropy, lit_buffer, + .. } = scratch; let dst = lit_buffer.sub_mut(..lit_size)?; let huf_result = if lit_enc_type == SymbolEncodingType::Repeat { @@ -581,6 +585,7 @@ impl BlockDecoder { let Scratch { entropy, lit_buffer, + batch, } = &mut **scratch; let dict_tables = dict_entropy.map(|dict| &dict.seq); let tables = entropy.seq.gather(&mut self.seq_sources, dict_tables); @@ -632,9 +637,9 @@ impl BlockDecoder { if nb_seq > 0 { let seq_state = SeqState::new(sequences, tables, rep)?; let prev_offset = if use_prefetch_decoder { - exec.run::(&seq_state, nb_seq)? + exec.run::(&seq_state, nb_seq, batch)? } else { - exec.run::(&seq_state, nb_seq)? + exec.run::(&seq_state, nb_seq, batch)? }; *rep = prev_offset.map(|offset| offset as u32); } @@ -714,7 +719,7 @@ struct Sequence { } /// A sequence whose offset is as it is coded: 1 to 3 for one of the last offsets, or 3 more than the offset. -#[derive(Clone, Copy, Default)] +#[derive(Clone, Copy)] struct RawSequence { offset_value: u32, lit_length: u32, @@ -722,6 +727,14 @@ struct RawSequence { match_length: u32, } +impl RawSequence { + const EMPTY: Self = Self { + offset_value: 0, + lit_length: 0, + match_length: 0, + }; +} + /// Where matches are, for `ZSTD_prefetchMatch`. struct History { prefix_start: *const u8, @@ -1467,19 +1480,6 @@ impl<'a> Executor<'_, 'a> { &mut self, seq_state: &SeqState<'_>, nb_seq: usize, - ) -> Result<[usize; ZSTD_REP_NUM]> { - // Sequences are decoded that many at a time, after 3 that only stand for the last offsets. - if nb_seq < 32 { - self.run_by_batch::(seq_state, nb_seq, &mut [RawSequence::default(); 3 + 32]) - } else { - self.run_by_batch::(seq_state, nb_seq, &mut [RawSequence::default(); 3 + 128]) - } - } - - fn run_by_batch( - &mut self, - seq_state: &SeqState<'_>, - nb_seq: usize, batch: &mut [RawSequence], ) -> Result<[usize; ZSTD_REP_NUM]> { // The C code then decodes 8 sequences ahead, so it sees whether the stream ends where it should before it diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index b66eeb12de48..51577c32bf52 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -71,7 +71,7 @@ impl FrameHeader { } enum HeaderStatus { - Complete(FrameHeader), + Complete, /// The header cannot be read from fewer bytes than this. Needs(usize), } @@ -97,8 +97,8 @@ fn frame_header_size(src: &[u8]) -> Result { + usize::from(single_segment && fcs_id == 0)) } -/// `ZSTD_getFrameHeader` -fn get_frame_header(src: &[u8]) -> Result { +/// `ZSTD_getFrameHeader`: sets `header` if the one at the start of `src` is complete. +fn get_frame_header(header: &mut FrameHeader, src: &[u8]) -> Result { const MIN_INPUT_SIZE: usize = FRAMEHEADERSIZE_PREFIX; if src.len() < MIN_INPUT_SIZE { @@ -124,13 +124,14 @@ fn get_frame_header(src: &[u8]) -> Result { if src.len() < ZSTD_SKIPPABLEHEADERSIZE { return Ok(HeaderStatus::Needs(ZSTD_SKIPPABLEHEADERSIZE)); } - return Ok(HeaderStatus::Complete(FrameHeader { + *header = FrameHeader { frame_type: FrameType::Skippable, dict_id: magic - ZSTD_MAGIC_SKIPPABLE_START, header_size: ZSTD_SKIPPABLEHEADERSIZE, frame_content_size: u64::from(read_le32(src, ZSTD_FRAMEIDSIZE)?), ..FrameHeader::EMPTY - })); + }; + return Ok(HeaderStatus::Complete); } return Err(ErrorCode::PrefixUnknown); } @@ -192,7 +193,7 @@ fn get_frame_header(src: &[u8]) -> Result { window_size = frame_content_size; } - Ok(HeaderStatus::Complete(FrameHeader { + *header = FrameHeader { frame_type: FrameType::Frame, frame_content_size, window_size, @@ -200,14 +201,16 @@ fn get_frame_header(src: &[u8]) -> Result { header_size: fhsize, dict_id, checksum_flag, - })) + }; + Ok(HeaderStatus::Complete) } /// `ZSTD_getFrameContentSize`: [`ZSTD_CONTENTSIZE_UNKNOWN`] if the frame at the start of `src` does not say, and /// [`ZSTD_CONTENTSIZE_ERROR`] if there is no whole frame header. pub fn get_frame_content_size(src: &[u8]) -> u64 { - match get_frame_header(src) { - Ok(HeaderStatus::Complete(header)) => match header.frame_type { + let mut header = FrameHeader::EMPTY; + match get_frame_header(&mut header, src) { + Ok(HeaderStatus::Complete) => match header.frame_type { FrameType::Skippable => 0, FrameType::Frame => header.frame_content_size, }, @@ -257,10 +260,10 @@ pub fn frame_extent(src: &[u8]) -> Result { FrameExtent::AtLeast(size) }); } - let header = match get_frame_header(src)? { - HeaderStatus::Complete(header) => header, - HeaderStatus::Needs(size) => return Ok(FrameExtent::AtLeast(size)), - }; + let mut header = FrameHeader::EMPTY; + if let HeaderStatus::Needs(size) = get_frame_header(&mut header, src)? { + return Ok(FrameExtent::AtLeast(size)); + } let mut ip = header.header_size; loop { let Some(remaining) = src @@ -770,9 +773,8 @@ impl DState { /// `ZSTD_decodeFrameHeader`: `src` is all of a frame header. fn decode_frame_header(&mut self, src: &[u8]) -> Result<()> { - match get_frame_header(src)? { - HeaderStatus::Complete(header) => self.f_params = header, - HeaderStatus::Needs(_) => return Err(ErrorCode::SrcSizeWrong), + if let HeaderStatus::Needs(_) = get_frame_header(&mut self.f_params, src)? { + return Err(ErrorCode::SrcSizeWrong); } if self.f_params.dict_id != 0 && self.dict_id != self.f_params.dict_id { return Err(ErrorCode::DictionaryWrong); @@ -1155,8 +1157,8 @@ impl DState { } if let DStreamStage::LoadHeader { lh_size } = self.stream_stage { - match get_frame_header(self.header_buffer.sub(..lh_size)?)? { - HeaderStatus::Complete(header) => self.f_params = header, + match get_frame_header(&mut self.f_params, self.header_buffer.sub(..lh_size)?)? { + HeaderStatus::Complete => {} HeaderStatus::Needs(h_size) => { let to_load = h_size - lh_size; let remaining_input = iend - ip; @@ -1166,7 +1168,10 @@ impl DState { self.stream_stage = DStreamStage::LoadHeader { lh_size }; input.pos = iend; // Refuse what cannot become a frame header. - get_frame_header(self.header_buffer.sub(..lh_size)?)?; + get_frame_header( + &mut self.f_params, + self.header_buffer.sub(..lh_size)?, + )?; return Ok( FRAMEHEADERSIZE_MIN.max(h_size) - lh_size + ZSTD_BLOCKHEADERSIZE ); From e62cc5c285a2d9d302d76b41b9ecd2fdff4a05d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:00:51 +0000 Subject: [PATCH 069/199] zstd: no shift by more than there are bits where a lookup is longer than the longest code --- src/zstd/decompress/huf.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index bc9e03ca2ec2..e243de081713 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -41,12 +41,11 @@ impl Code { let mut share = [0u64; NB_RANKS + 1]; let mut total = 0; for nb_bits in 1..=log as usize { - let weight = (table_log + 1).wrapping_sub(nb_bits); - let count = self.rank_stats.get(weight).copied().unwrap_or(0); - let of_these = if weight == 0 { - 0 - } else { - u64::from(count) << (weight - 1) + let of_these = match (table_log + 1).saturating_sub(nb_bits) { + 0 => 0, + weight => { + u64::from(self.rank_stats.get(weight).copied().unwrap_or(0)) << (weight - 1) + } }; total += of_these; if let (Some(share), Some(one)) = (share.get_mut(nb_bits), one.get_mut(nb_bits)) { From 4bd2cf761259ce65d558d01edda10e09d2517b04 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:04:04 +0000 Subject: [PATCH 070/199] zstd: the loops of the decoder that come in several kinds are inlined in functions that do not --- src/zstd/decompress/block.rs | 28 ++++++++++++++++------- src/zstd/decompress/huf.rs | 10 +++++++- src/zstd/decompress/seq_table.rs | 39 +++++++++++++++----------------- 3 files changed, 47 insertions(+), 30 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 77c8f7fab00a..b04189e99769 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -844,11 +844,27 @@ impl<'a> SeqState<'a> { } /// Decodes sequences for as long as the start of the stream is far. Returns how many that is. - /// With `PREFETCH`, `history` follows the output, and what matches refer to is on its way to the cache. + /// With `prefetch`, `history` follows the output, and what matches refer to is on its way to the cache. /// - /// `MORE_BITS` is for when it cannot be predicted which lengths come with additional bits. + /// `more_bits` is for when it cannot be predicted which lengths come with additional bits. #[inline(never)] - fn decode_batch( + fn decode_batch( + &mut self, + prefetch: bool, + more_bits: bool, + out: &mut [RawSequence], + history: &mut History, + ) -> Result { + match (prefetch, more_bits) { + (false, false) => self.decode_batch_as::(out, history), + (false, true) => self.decode_batch_as::(out, history), + (true, false) => self.decode_batch_as::(out, history), + (true, true) => self.decode_batch_as::(out, history), + } + } + + #[inline(always)] + fn decode_batch_as( &mut self, out: &mut [RawSequence], history: &mut History, @@ -1515,11 +1531,7 @@ impl<'a> Executor<'_, 'a> { let room = (batch.len() - 3).min(remaining - ahead.len().max(1)); let room = batch.span_mut(3, room)?; let history = &mut self.history(); - count = if more_bits { - seq_state.decode_batch::(room, history)? - } else { - seq_state.decode_batch::(room, history)? - }; + count = seq_state.decode_batch(LONG, more_bits, room, history)?; if count == 0 && can_leave && let Some(bit_d) = start.set(&seq_state.bit_d) diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index e243de081713..14ce20b3f82d 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -384,9 +384,10 @@ impl HufDTable { if !streams.0.iter().all(Stream::fits_tail) { return self.decode_stragglers(streams); } - self.decode_tails(streams.0) + self.decode_4_tails(streams) } + #[inline(never)] fn decode_most(&self, streams: &mut Streams<'_>) { match self.log { 11 => streams.decode_most::<11, 5>(&self.table, 0), @@ -433,10 +434,16 @@ impl HufDTable { } self.decode_most(&mut decoding); } + self.decode_4_tails(streams) + } + + #[inline(never)] + fn decode_4_tails(&self, streams: Streams<'_>) -> bool { self.decode_tails(streams.0) } /// Decodes what is left of streams. Whether each turns out to be exactly as many symbols as its output takes. + #[inline(always)] fn decode_tails(&self, mut streams: [Stream<'_>; N]) -> bool { let mut tails = [const { Tail::new() }; N]; for (tail, stream) in tails.iter_mut().zip(&streams) { @@ -753,6 +760,7 @@ impl<'a> Streams<'a> { /// Decodes for as long as no stream is close to its start and no output to its end. A lookup is of `LOG` bits, or of /// `64 - shift` if that is 0, and `LOOKUPS` of them are at most 55 bits. + #[inline(always)] fn decode_most( &mut self, table: &[Entry; TABLE_SIZE], diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs index d24fcaf86a75..c0047e29ffa9 100644 --- a/src/zstd/decompress/seq_table.rs +++ b/src/zstd/decompress/seq_table.rs @@ -46,7 +46,6 @@ pub(crate) trait Cell: Copy + 'static { const NEXT_STATES: &'static [u64; 4 * MAX_TABLE_SIZE]; /// The fields one after the other, from the lowest bits on. fn from_bits(bits: u64) -> Self; - fn has_additional_bits(self) -> bool; fn unpack(self) -> SeqSymbol; } @@ -117,11 +116,6 @@ impl Cell for LengthCell { } } - #[inline(always)] - fn has_additional_bits(self) -> bool { - self.nb_additional_bits != 0 - } - #[inline(always)] fn unpack(self) -> SeqSymbol { SeqSymbol { @@ -194,11 +188,6 @@ impl Cell for OffsetCell { } } - #[inline(always)] - fn has_additional_bits(self) -> bool { - self.nb_additional_bits != 0 - } - #[inline(always)] fn unpack(self) -> SeqSymbol { SeqSymbol { @@ -244,6 +233,20 @@ const fn offset_codes() -> Codes { codes } +/// `nb_additional_bits` is at the same place in cells of both kinds. +const fn has_additional_bits(code: u64) -> bool { + (code >> 48) as u8 != 0 +} + +/// How many of 512 symbols are expected to come with additional bits in a table made from `counts`. +fn share_with_more_bits(counts: &[i16], codes: &Codes, table_log: u32) -> u32 { + (counts.iter().zip(codes)) + .filter(|&(_, &code)| has_additional_bits(code)) + .map(|(&count, _)| u32::from(count.unsigned_abs())) + .sum::() + << (9 - table_log.min(9)) +} + pub(crate) static LL_CODES: Codes = length_codes(&LL_BASE, &LL_BITS); pub(crate) static OF_CODES: Codes = offset_codes(); pub(crate) static ML_CODES: Codes = length_codes(&ML_BASE, &ML_BITS); @@ -315,11 +318,10 @@ impl SeqTable { /// `ZSTD_buildSeqTable_rle` pub(crate) fn build_rle(&mut self, code: u64) { - let code = C::from_bits(code); self.table_log = 0; - self.with_more_bits = u32::from(code.has_additional_bits()) << 9; + self.with_more_bits = u32::from(has_additional_bits(code)) << 9; if let Some(cell) = self.table.first_mut() { - *cell = code; + *cell = C::from_bits(code); } } @@ -335,13 +337,8 @@ impl SeqTable { let table_size = 1usize << table_log; let table_decode = self.table.sub_mut(..table_size)?; self.table_log = table_log; - self.with_more_bits = (spread.counts.sub(..nb_symbols)?) - .iter() - .zip(codes) - .filter(|&(_, &code)| C::from_bits(code).has_additional_bits()) - .map(|(&count, _)| u32::from(count.unsigned_abs())) - .sum::() - << (9 - table_log.min(9)); + self.with_more_bits = + share_with_more_bits(spread.counts.sub(..nb_symbols)?, codes, table_log); spread.set(nb_symbols, table_log)?; for (cell, &symbol) in table_decode.iter_mut().zip(&spread.symbols) { From eb486dea3d94e7bec684bced297fb5ec8612ab08 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:06:16 +0000 Subject: [PATCH 071/199] zstd: matches in what a streaming decoder's buffer holds from before it wrapped around are of the common kind --- src/zstd/decompress/block.rs | 47 +++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index b04189e99769..939381d52fd0 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -7,6 +7,7 @@ use super::seq_table::{ }; use core::cell::{self, OnceCell}; use core::hint::{Locality, prefetch_read}; +use core::ops::Range; use crate::common::bitstream::BitDStream; use crate::common::entropy::read_ncount; @@ -1036,6 +1037,10 @@ impl<'a> SeqState<'a> { } } +const NO_EXT: u8 = 0; +const EXT_DICT: u8 = 1; +const EXT_WITHIN: u8 = 2; + /// Copies 16 bytes within `buf`, between positions that do not overflow. Whether they are in `buf`. #[inline(always)] fn copy_16(buf: &mut [u8], src: usize, dst: usize) -> bool { @@ -1344,7 +1349,7 @@ impl<'a> Executor<'_, 'a> { /// The last offsets are those of the 3 sequences before, which are rewritten for that where it takes it. #[inline(never)] fn exec_batch(&mut self, sequences: &mut [RawSequence], index: usize) -> usize { - self.exec_batch_with::(sequences, index, &[]) + self.exec_batch_with::(sequences, index, &[], 0..0) } /// The same where matches in `dict`, which comes before the output, are of the common kind too. @@ -1355,23 +1360,36 @@ impl<'a> Executor<'_, 'a> { index: usize, dict: &[u8], ) -> usize { - self.exec_batch_with::(sequences, index, dict) + self.exec_batch_with::(sequences, index, dict, 0..0) } + /// The same for `within` of the buffer that the output goes to. + #[inline(never)] + fn exec_batch_after_wrap( + &mut self, + sequences: &mut [RawSequence], + index: usize, + within: Range, + ) -> usize { + self.exec_batch_with::(sequences, index, &[], within) + } + + /// What comes before the output is nothing of the common kind, `dict` or `within`, as `EXT` tells. #[inline(always)] - fn exec_batch_with( + fn exec_batch_with( &mut self, sequences: &mut [RawSequence], mut index: usize, dict: &[u8], + within: Range, ) -> usize { // Positions fit in 32 bits, so that the compiler sees that nothing added to them overflows. const MAX_BACK: usize = 3 << 30; let start = self.prefix_start.max(self.op.saturating_sub(MAX_BACK)); - let dict = if start == self.prefix_start { - dict + let (dict, within) = if start == self.prefix_start && within.start >= start { + (dict, within.start - start..within.end - start) } else { - &[] + (&[][..], 0..0) }; let buf = self.out.bytes(); let end = buf.len().min(start + u32::MAX as usize); @@ -1400,7 +1418,7 @@ impl<'a> Executor<'_, 'a> { || lit_end + WILDCOPY_OVERLENGTH > lits.len() || lit_length > 32 || match_length > 64 - || !DICT && offset_value as usize > o_lit_end + 3 + || EXT == NO_EXT && offset_value as usize > o_lit_end + 3 { break; } @@ -1410,7 +1428,7 @@ impl<'a> Executor<'_, 'a> { ) else { break; }; - if DICT && offset_value as usize > o_lit_end + 3 { + if EXT == EXT_DICT && offset_value as usize > o_lit_end + 3 { let back = offset_value as usize - 3 - o_lit_end; let Some(match_pos) = dict.len().checked_sub(back) else { break; @@ -1437,8 +1455,16 @@ impl<'a> Executor<'_, 'a> { index += 1; continue; } + let mut match_end = o_lit_end + 3; + if EXT == EXT_WITHIN && offset_value as usize > o_lit_end + 3 { + let back = offset_value as usize - 3 - o_lit_end; + if back > within.len() || match_length > back { + break; + } + match_end += within.end; + } to.copy_from_slice(from); - let match_pos = o_lit_end + 3 - offset_value as usize; + let match_pos = match_end - offset_value as usize; if !copy_16(buf, match_pos, o_lit_end) || !copy_16(buf, match_pos + 16, o_lit_end + 16) { break; @@ -1612,6 +1638,9 @@ impl<'a> Executor<'_, 'a> { ExtDict::Dict(dict) if !dict.is_empty() => { self.exec_batch_after_dict(sequences, index, dict) } + ExtDict::Within { end, len } if len != 0 => { + self.exec_batch_after_wrap(sequences, index, end - len..end) + } _ => self.exec_batch(sequences, index), }; if index >= sequences.len() { From 5f7d028f8db76c47adfa2a907ea5e4ffb831fe5d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:09:18 +0000 Subject: [PATCH 072/199] zstd: the tables of a few sequences are used where they are --- src/zstd/decompress/block.rs | 25 +++++++++++++------- src/zstd/decompress/seq_table.rs | 39 ++++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 939381d52fd0..7c5961985359 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -2,8 +2,8 @@ use super::huf::{self, HufDTable}; use super::seq_table::{ - Cell, Codes, LL_CODES, ML_CODES, OF_CODES, OffsetTable, SeqTable, SeqTableSources, SeqTables, - TableSource, + Cell, Codes, LL_CODES, ML_CODES, OF_CODES, OffsetTable, SeqTable, SeqTableRefs, + SeqTableSources, SeqTables, TableSource, }; use core::cell::{self, OnceCell}; use core::hint::{Locality, prefetch_read}; @@ -589,13 +589,17 @@ impl BlockDecoder { batch, } = &mut **scratch; let dict_tables = dict_entropy.map(|dict| &dict.seq); - let tables = entropy.seq.gather(&mut self.seq_sources, dict_tables); + let tables = if nb_seq <= MAX_ONE_BY_ONE { + entropy.seq.pick(self.seq_sources, dict_tables) + } else { + entropy.seq.gather(&mut self.seq_sources, dict_tables) + }; // With matches far back in a long history, it pays to know the sequences ahead of time. Nothing is prefetched // here, but which of two problems in a damaged block is reported depends on the order. if !use_prefetch_decoder && total_history_size > (1 << 24) && nb_seq > 8 { const MIN_SHARE: u32 = 7; - use_prefetch_decoder = long_offset_share(&tables.of) >= MIN_SHARE; + use_prefetch_decoder = long_offset_share(tables.of) >= MIN_SHARE; } self.ddict_is_cold = false; if nb_seq > 0 { @@ -811,6 +815,9 @@ impl PaddedStart { } } +/// Up to how many sequences it is not worth it to decode them by the batch. +const MAX_ONE_BY_ONE: usize = 8; + /// `seqState_t` #[derive(Clone, Copy)] struct SeqState<'a> { @@ -818,12 +825,12 @@ struct SeqState<'a> { state_ll: usize, state_off: usize, state_ml: usize, - tables: &'a SeqTables, + tables: SeqTableRefs<'a>, prev_offset: [usize; ZSTD_REP_NUM], } impl<'a> SeqState<'a> { - fn new(src: &'a [u8], tables: &'a SeqTables, rep: &[u32; ZSTD_REP_NUM]) -> Result { + fn new(src: &'a [u8], tables: SeqTableRefs<'a>, rep: &[u32; ZSTD_REP_NUM]) -> Result { let mut bit_d = BitDStream::new(src).map_err(|_| ErrorCode::CorruptionDetected)?; // `ZSTD_initFseState` let mut init = |table_log: u32| { @@ -891,7 +898,9 @@ impl<'a> SeqState<'a> { // How many bits are left to read, and 7. let mut position = src.len() as u32 * 8 + 7 - self.bit_d.bits_consumed; let containers = src.get(FAR - 8..).unwrap_or_default(); - let tables = self.tables; + let Some(tables) = self.tables.side_by_side else { + return Ok(0); + }; let mut state_ll = self.state_ll as u32 & 511; let mut state_ml = self.state_ml as u32 & 511; let mut state_off = self.state_off as u32 & 255; @@ -1538,7 +1547,7 @@ impl<'a> Executor<'_, 'a> { let mut remaining = nb_seq; // Long copies go on while the next sequences are decoded, if that is what comes next. And a batch of a few // sequences is not worth it. - if (tables.ml.with_more_bits > 512 / 2 || nb_seq <= 8) && !self.split { + if (tables.ml.with_more_bits > 512 / 2 || nb_seq <= MAX_ONE_BY_ONE) && !self.split { let mut exec = self.fork(); while remaining > ahead.len() { let sequence = seq_state.decode_sequence(remaining == 1); diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs index c0047e29ffa9..208d519edc6b 100644 --- a/src/zstd/decompress/seq_table.rs +++ b/src/zstd/decompress/seq_table.rs @@ -460,20 +460,39 @@ impl SeqTables { } } - /// The three tables that `sources` designates, side by side. If they are not, those that are not in `self` are - /// copied there, which `sources` then says. + /// The three tables that `sources` designates, where they are. + pub(crate) fn pick<'a>( + &'a self, + sources: SeqTableSources, + dict: Option<&'a Self>, + ) -> SeqTableRefs<'a> { + let from = |source| match (source, dict) { + (TableSource::Predefined, _) => &DEFAULT_TABLES, + (TableSource::Dict, Some(dict)) => dict, + _ => self, + }; + SeqTableRefs { + ll: &from(sources.ll).ll, + of: &from(sources.of).of, + ml: &from(sources.ml).ml, + side_by_side: (sources == SeqTableSources::all(sources.ll)).then(|| from(sources.ll)), + } + } + + /// The same side by side. If they are not, those that are not in `self` are copied there, which `sources` then + /// says. pub(crate) fn gather<'a>( &'a mut self, sources: &mut SeqTableSources, dict: Option<&'a Self>, - ) -> &'a Self { + ) -> SeqTableRefs<'a> { let from = |source| match (source, dict) { (TableSource::Predefined, _) => Some(&DEFAULT_TABLES), (TableSource::Dict, Some(dict)) => Some(dict), _ => None, }; if *sources == SeqTableSources::all(sources.ll) { - return from(sources.ll).unwrap_or(self); + return self.pick(*sources, dict); } if let Some(other) = from(sources.ll) { self.ll.copy_from(&other.ll); @@ -485,6 +504,16 @@ impl SeqTables { self.ml.copy_from(&other.ml); } *sources = SeqTableSources::all(TableSource::Local); - self + self.pick(*sources, dict) } } + +/// Three tables to decode sequences with. +#[derive(Clone, Copy)] +pub(crate) struct SeqTableRefs<'a> { + pub(crate) ll: &'a LengthTable, + pub(crate) of: &'a OffsetTable, + pub(crate) ml: &'a LengthTable, + /// What the three are in, if that is one thing. + pub(crate) side_by_side: Option<&'a SeqTables>, +} From 964de3eca9ce1a8e603ae21563cb3285b75a728f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:11:08 +0000 Subject: [PATCH 073/199] zstd: what the tables of sequences tell about them is theirs to tell --- src/zstd/decompress/block.rs | 6 ++---- src/zstd/decompress/seq_table.rs | 11 +++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 7c5961985359..267df3d92fb3 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -1540,14 +1540,12 @@ impl<'a> Executor<'_, 'a> { let start = PaddedStart::new(); let mut seq_state: SeqState<'_> = *seq_state; - // A branch that goes wrong costs about 10 times what it saves. - let tables = seq_state.tables; - let more_bits = tables.ll.with_more_bits + tables.ml.with_more_bits > 512 / 10; + let more_bits = seq_state.tables.lengths_often_have_more_bits(); let mut remaining = nb_seq; // Long copies go on while the next sequences are decoded, if that is what comes next. And a batch of a few // sequences is not worth it. - if (tables.ml.with_more_bits > 512 / 2 || nb_seq <= MAX_ONE_BY_ONE) && !self.split { + if (seq_state.tables.matches_are_long() || nb_seq <= MAX_ONE_BY_ONE) && !self.split { let mut exec = self.fork(); while remaining > ahead.len() { let sequence = seq_state.decode_sequence(remaining == 1); diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs index 208d519edc6b..b84501d30b4d 100644 --- a/src/zstd/decompress/seq_table.rs +++ b/src/zstd/decompress/seq_table.rs @@ -517,3 +517,14 @@ pub(crate) struct SeqTableRefs<'a> { /// What the three are in, if that is one thing. pub(crate) side_by_side: Option<&'a SeqTables>, } + +impl SeqTableRefs<'_> { + /// Often enough for a branch on that to cost more than it saves: one that goes wrong costs about 10 times that. + pub(crate) fn lengths_often_have_more_bits(&self) -> bool { + self.ll.with_more_bits + self.ml.with_more_bits > 512 / 10 + } + + pub(crate) fn matches_are_long(&self) -> bool { + self.ml.with_more_bits > 512 / 2 + } +} From 675a92a0167f004f577e9a3b0e77152ca6877f54 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:26:02 +0000 Subject: [PATCH 074/199] zstd: the size of a decoder counts all that it allocates to decode blocks --- src/zstd/decompress/block.rs | 2 ++ src/zstd/decompress/mod.rs | 4 ++-- test/js/node/zlib/zlib-estimated-size-gc.test.ts | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 267df3d92fb3..521b70ebffdb 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -253,6 +253,8 @@ impl Drop for BlockDecoder { } impl BlockDecoder { + pub(crate) const SCRATCH_SIZE: usize = size_of::(); + pub(crate) const fn new() -> Self { Self { scratch: None, diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 51577c32bf52..2e1b66cacba5 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -584,8 +584,8 @@ impl Default for DCtx { } impl DCtx { - /// How much memory a new one takes. - pub const SIZE: usize = size_of::() + size_of::(); + /// How much memory one takes, apart from buffers whose size depends on the frames. + pub const SIZE: usize = size_of::() + BlockDecoder::SCRATCH_SIZE; /// `ZSTD_createDCtx` pub fn new() -> Self { diff --git a/test/js/node/zlib/zlib-estimated-size-gc.test.ts b/test/js/node/zlib/zlib-estimated-size-gc.test.ts index e10176b3849e..4d8db0a355f7 100644 --- a/test/js/node/zlib/zlib-estimated-size-gc.test.ts +++ b/test/js/node/zlib/zlib-estimated-size-gc.test.ts @@ -12,7 +12,7 @@ import zlib from "node:zlib"; // estimateShallowMemoryUsageOf(cell) == sizeof(cell) + estimated_size(). `min` // is a floor below the per-mode footprint the constructor caches (brotli encode -// 5143, brotli decode 855, zstd compress about 14000, zstd decompress about 35000). +// 5143, brotli decode 855, zstd compress about 14000, zstd decompress about 31000). function checkFootprintStableAcrossClose(create: () => any, min: number) { const engine = create(); engine.on("error", () => {}); From d99e43ebe927d97030ddebd2f5029d683d11fa0c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:30:17 +0000 Subject: [PATCH 075/199] zstd: btlazy2 sets up its searches once per block `TreeFinder` holds what the searches in the binary trees work on, which only depends on the block but for the lowest index a match can be at. It was worked out again for each position. --- src/zstd/compress/lazy.rs | 225 +++++++++++++++++--------------------- 1 file changed, 98 insertions(+), 127 deletions(-) diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index ec0ea5719d43..f077013f1ed9 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -37,35 +37,37 @@ const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { // ── binary tree search ──────────────────────────────────────────────────── -/// What a search in the binary trees works on. +/// What the searches in the binary trees work on for the time of a block: the tables of a [`MatchState`], and what does +/// not change of it. struct Dubt<'a> { hash_table: &'a mut [u32], bt: &'a mut Nodes, input: TreeInput<'a>, + /// The dictionary, if searches go on in it. + dms: Option<&'a DictMatchState<'a>>, hasher: Hasher, /// `window.lowLimit` low_limit: u32, max_distance: u32, - /// `ZSTD_getLowestMatchIndex` of `curr`. - window_low: u32, + /// How far a match can be, if that is not down to `low_limit`. + reach: u32, nb_compares: u32, - next_to_update: u32, - /// The position that is searched for, which is `ip` in `input.near`. - curr: u32, - ip: u32, + next_to_update: &'a mut u32, } impl<'a> Dubt<'a> { - fn new( - ms: &'a mut MatchState, - src: &BlockSrc<'a>, - ip: usize, - iend: usize, - mls: u32, - ) -> Result { - let c_params = ms.c_params; - if ms.hash_table.len() != 1usize.wrapping_shl(c_params.hash_log) - || ms.chain_table.len() != 1usize.wrapping_shl(c_params.chain_log) + fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, search_dms: bool) -> Result { + let MatchState { + hash_table, + chain_table, + next_to_update, + window, + c_params, + loaded_dict_end, + .. + } = ms; + if hash_table.len() != 1usize.wrapping_shl(c_params.hash_log) + || chain_table.len() != 1usize.wrapping_shl(c_params.chain_log) { return Err(BUG); } @@ -73,29 +75,33 @@ impl<'a> Dubt<'a> { buf: src.dict_buf, base: src.dict_base, }; - let curr = src.index_of(ip); - let dict_limit = ms.window.dict_limit; + let max_distance = 1u32.wrapping_shl(c_params.window_log); Ok(Self { - window_low: ms.get_lowest_match_index(curr, c_params.window_log), - hash_table: ms.hash_table.as_mut_slice(), - bt: bt_nodes(&mut ms.chain_table), - input: TreeInput::new(src.prefix(), dict, dict_limit, iend)?, - hasher: Hasher::new(c_params.hash_log, mls), - low_limit: ms.window.low_limit, - max_distance: 1u32.wrapping_shl(c_params.window_log), + hash_table: hash_table.as_mut_slice(), + bt: bt_nodes(chain_table), + input: TreeInput::new(src.prefix(), dict, window.dict_limit, src.end)?, + dms: src.dms.as_ref().filter(|_| search_dms), + hasher: Hasher::new(c_params.hash_log, c_params.min_match.clamp(4, 6)), + low_limit: window.low_limit, + max_distance, + // A dictionary is valid as a whole until it is invalidated as a whole. + reach: if *loaded_dict_end != 0 { + u32::MAX + } else { + max_distance + }, nb_compares: 1u32.wrapping_shl(c_params.search_log), - next_to_update: ms.next_to_update, - curr, - ip: curr.wrapping_sub(dict_limit), + next_to_update, }) } /// `ZSTD_updateDUBT`: adds the positions from `next_to_update` up to `curr` to the trees, as unsorted chains. #[inline(always)] - fn update(&mut self) -> Result<()> { + fn update(&mut self, curr: u32) -> Result<()> { let near = self.input.near; - if self.curr.saturating_sub(self.next_to_update) >= LONG_UPDATE { - let first = self.next_to_update..self.curr; + let next_to_update = *self.next_to_update; + if curr.saturating_sub(next_to_update) >= LONG_UPDATE { + let first = next_to_update..curr; let dict_limit = self.input.dict_limit; return update_many( self.hash_table, @@ -106,7 +112,7 @@ impl<'a> Dubt<'a> { dict_limit, ); } - for idx in self.next_to_update..self.curr { + for idx in next_to_update..curr { let bytes = near.read64(idx.wrapping_sub(self.input.dict_limit)); let h = self.hasher.hash(bytes.ok_or(BUG)?); let match_index = table_get(self.hash_table, h); @@ -144,18 +150,22 @@ impl<'a> Dubt<'a> { sort_dubt1(self.bt, &self.input, curr, nb_compares, bt_low, window_low) } - /// `ZSTD_DUBT_findBestMatch`, but for the dictionary. Returns the length of the best match, or 0, and how many more - /// positions could have been compared if the dictionary is to be searched. + /// `ZSTD_DUBT_findBestMatch` for the position `ip` of `input.near`, but for the dictionary. Returns the length of the + /// best match, or 0, and how many more positions could have been compared if the dictionary is to be searched. #[inline(always)] - fn find_best_match(&mut self, off_base_ptr: &mut u32) -> Result { - let curr = self.curr; - let h = self - .hasher - .hash(self.input.near.read64(self.ip).ok_or(BUG)?); + fn find_best_match(&mut self, ip: u32, off_base_ptr: &mut u32) -> Result { + let curr = ip.wrapping_add(self.input.dict_limit); + // `ZSTD_getLowestMatchIndex` + let window_low = if curr.wrapping_sub(self.low_limit) > self.reach { + curr.wrapping_sub(self.reach) + } else { + self.low_limit + }; + let h = self.hasher.hash(self.input.near.read64(ip).ok_or(BUG)?); let mut match_index = table_get(self.hash_table, h); - bt_look_ahead::<2, 1>(self.hash_table, self.bt, &self.input, self.hasher, self.ip); + bt_look_ahead::<2, 1>(self.hash_table, self.bt, &self.input, self.hasher, ip); - let unsort_limit = bt_low(self.bt, curr).max(self.window_low); + let unsort_limit = bt_low(self.bt, curr).max(window_low); let mut nb_candidates = self.nb_compares; let mut previous_candidate = 0u32; @@ -194,9 +204,9 @@ impl<'a> Dubt<'a> { let nb_compares = bt_walk( self.bt, &self.input, - self.ip, + ip, match_index, - self.window_low.saturating_add(1), + window_low.saturating_add(1), self.nb_compares, Descent::new(self.bt, curr), |match_index, common| { @@ -222,7 +232,7 @@ impl<'a> Dubt<'a> { )?; // Skip repetitive patterns. - self.next_to_update = match_end_idx.wrapping_sub(8); + *self.next_to_update = match_end_idx.wrapping_sub(8); Ok(DubtFound { best_length, nb_compares: if search_dict { nb_compares } else { 0 }, @@ -231,10 +241,45 @@ impl<'a> Dubt<'a> { } impl Dubt<'_> { - /// [`DictTree::root`] for the position that is searched for. + /// `ZSTD_BtFindBestMatch`: updates the trees up to the position `ip` of `input.near` and returns the best match there. + /// Its length is 0 if there is none. + #[inline(never)] + fn search(&mut self, ip: u32) -> Result { + let mut off_base = 999_999_999u32; + let curr = ip.wrapping_add(self.input.dict_limit); + let len = if curr < *self.next_to_update { + // In an area that is skipped. + 0 + } else { + let dict_root = match self.dms { + Some(dms) => self.dict_root(dms, ip)?, + None => 0, + }; + self.update(curr)?; + let found = self.find_best_match(ip, &mut off_base)?; + match self.dms { + Some(dms) if dict_root != 0 && found.nb_compares != 0 => find_better_dict_match( + dms, + self.input, + self.low_limit, + ip, + dict_root, + found, + &mut off_base, + )?, + _ => found.best_length, + } + }; + Ok(Found { + len, + off_base: NonZeroU32::new(off_base).ok_or(BUG)?, + }) + } + + /// [`DictTree::root`] for the position `ip` of `input.near`. #[inline(always)] - fn dict_root(&self, dms: &DictMatchState<'_>) -> Result { - let bytes = self.input.near.read64(self.ip).ok_or(BUG)?; + fn dict_root(&self, dms: &DictMatchState<'_>, ip: u32) -> Result { + let bytes = self.input.near.read64(ip).ok_or(BUG)?; let hasher = self.hasher.with_log(dms.ms.c_params.hash_log); DictTree::root(*dms, hasher.hash(bytes)) } @@ -359,59 +404,6 @@ struct DubtFound { nb_compares: u32, } -/// `ZSTD_BtFindBestMatch`, when `mls` and the mode are not constants. -#[inline(never)] -fn bt_search( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - off_base_ptr: &mut u32, - mls: u32, - search_dms: bool, -) -> Result { - if ip < src.pos_of(ms.next_to_update) { - // In an area that is skipped. - return Ok(0); - } - let mut dubt = Dubt::new(ms, src, ip, i_limit, mls)?; - let dms = src.dms.as_ref().filter(|_| search_dms); - let dict_root = match dms { - Some(dms) => dubt.dict_root(dms)?, - None => 0, - }; - dubt.update()?; - let found = dubt.find_best_match(off_base_ptr)?; - let best_length = match dms { - Some(dms) if dict_root != 0 && found.nb_compares != 0 => find_better_dict_match( - dms, - dubt.input, - dubt.low_limit, - dubt.ip, - dict_root, - found, - off_base_ptr, - )?, - _ => found.best_length, - }; - ms.next_to_update = dubt.next_to_update; - Ok(best_length as usize) -} - -/// `ZSTD_BtFindBestMatch`: updates the trees up to `ip` and returns the length of the best match there, or 0. -/// `off_base_ptr` is written to if there is one. -#[inline(always)] -fn bt_find_best_match( - ms: &mut MatchState, - src: &BlockSrc<'_>, - ip: usize, - i_limit: usize, - off_base_ptr: &mut u32, -) -> Result { - let search_dms = DICT_MODE == DICT_MATCH_STATE; - bt_search(ms, src, ip, i_limit, off_base_ptr, MLS, search_dms) -} - // ── what the match finders share ────────────────────────────────────────── /// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match @@ -724,25 +716,14 @@ trait Finder<'a>: Sized { // ── binary trees, for the parser ────────────────────────────────────────── -struct TreeFinder<'a, const DICT_MODE: u8> { - ms: &'a mut MatchState, - src: &'a BlockSrc<'a>, - /// Where the position 0 is in `src.buf`. - prefix_start: usize, - mls: u32, -} +struct TreeFinder<'a, const DICT_MODE: u8>(Dubt<'a>); impl<'a, const DICT_MODE: u8> Finder<'a> for TreeFinder<'a, DICT_MODE> { const MARGIN: u32 = 8; #[inline(always)] - fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result { - Ok(Self { - mls: ms.c_params.min_match.clamp(4, 6), - prefix_start: src.pos_of(block.base_index), - ms, - src, - }) + fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, _block: LazyBlock<'a>) -> Result { + Ok(Self(Dubt::new(ms, src, DICT_MODE == DICT_MATCH_STATE)?)) } #[inline(always)] @@ -752,18 +733,8 @@ impl<'a, const DICT_MODE: u8> Finder<'a> for TreeFinder<'a, DICT_MODE> { wanted: Wanted, _skipping: bool, ) -> Result> { - let ip = self.prefix_start + ip as usize; - let iend = self.src.end; - let mut off_base = 999_999_999u32; - let len = match self.mls { - 4 => bt_find_best_match::<4, DICT_MODE>(self.ms, self.src, ip, iend, &mut off_base), - 5 => bt_find_best_match::<5, DICT_MODE>(self.ms, self.src, ip, iend, &mut off_base), - _ => bt_find_best_match::<6, DICT_MODE>(self.ms, self.src, ip, iend, &mut off_base), - }? as u32; - if len <= wanted.longer_than { - return Ok(None); - } - Ok(NonZeroU32::new(off_base).map(|off_base| Found { len, off_base })) + let found = self.0.search(ip)?; + Ok(Some(found).filter(|found| found.len > wanted.longer_than)) } #[inline(always)] From 09f9383d50b72d096e0132e2aebd6f24dd6d924c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:39:02 +0000 Subject: [PATCH 076/199] zstd: the next sequence waits less for the states that it is decoded from --- src/zstd/decompress/block.rs | 50 ++++++++++++++++++-------------- src/zstd/decompress/seq_table.rs | 46 ++++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 30 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 521b70ebffdb..850e716e3cb5 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -915,30 +915,41 @@ impl<'a> SeqState<'a> { unused = 1; break; }; - let ll_cell = tables.ll.cell(state_ll as usize); - let ml_cell = tables.ml.cell(state_ml as usize); - let of_cell = tables.of.cell(state_off as usize); + // Nothing waits for a check, as it would for a mask. + let (Some(ll_cell), Some(ml_cell), Some(of_cell)) = ( + tables.ll.table.get(state_ll as usize), + tables.ml.table.get(state_ml as usize), + tables.of.table.get(state_off as usize), + ) else { + unused = 1; + break; + }; let of_bits = u32::from(of_cell.nb_additional_bits); let ml_bits = u32::from(ml_cell.nb_additional_bits); let ll_bits = u32::from(ll_cell.nb_additional_bits); + let ll_nb_bits = u32::from(ll_cell.nb_bits); + let ml_nb_bits = u32::from(ml_cell.nb_bits); + let of_nb_bits = u32::from(of_cell.nb_bits); + let nb_all_bits = + (ml_bits + ll_bits + ll_nb_bits + ml_nb_bits) + (of_bits + of_nb_bits); let mut match_length = u32::from(ml_cell.base_value); let mut lit_length = u32::from(ll_cell.base_value); let first_value = of_cell.first_value; + let mut loaded = bits; bits = bits.rotate_left(of_bits); - position -= of_bits; let offset_value = first_value + (bits as u32 & (first_value - 1)); if MORE_BITS || ml_bits + ll_bits != 0 { bits = bits.rotate_left(ml_bits); - position -= ml_bits; - match_length = ml_cell.base_value() + (bits as u32 & ((1 << ml_bits) - 1)); + match_length = tables.length(ml_cell.base_value, ml_cell.nb_additional_bits, bits); // All has to fit in 57 bits with the 26 of the next states. if of_bits + ml_bits + ll_bits >= 31 { + let position = position - of_bits - ml_bits; bits = load(src, position + (FAR as u32 - 8) * 8).unwrap_or(0); + loaded = bits.rotate_right(of_bits + ml_bits); } bits = bits.rotate_left(ll_bits); - position -= ll_bits; - lit_length = ll_cell.base_value() + (bits as u32 & ((1 << ll_bits) - 1)); + lit_length = tables.length(ll_cell.base_value, ll_cell.nb_additional_bits, bits); } *slot = RawSequence { offset_value, @@ -949,22 +960,17 @@ impl<'a> SeqState<'a> { history.prefetch(lit_length, match_length, offset_value); } - let ll_nb_bits = u32::from(ll_cell.nb_bits); - let ml_nb_bits = u32::from(ml_cell.nb_bits); - let of_nb_bits = u32::from(of_cell.nb_bits); - position -= ll_nb_bits + ml_nb_bits + of_nb_bits; + position -= nb_all_bits; bits = bits.rotate_left(ll_nb_bits); - state_ll = (u32::from(ll_cell.next_state) - + (bits as u32 & u32::from(ll_cell.state_mask))) - & 511; + state_ll = + u32::from(ll_cell.next_state) + (bits as u32 & u32::from(ll_cell.state_mask)); bits = bits.rotate_left(ml_nb_bits); - state_ml = (u32::from(ml_cell.next_state) - + (bits as u32 & u32::from(ml_cell.state_mask))) - & 511; - bits = bits.rotate_left(of_nb_bits); - state_off = (u32::from(of_cell.next_state) - + (bits as u32 & u32::from(of_cell.state_mask))) - & 255; + state_ml = + u32::from(ml_cell.next_state) + (bits as u32 & u32::from(ml_cell.state_mask)); + // This is read last, and it is what the first bits of the next sequence depend on. So it does not wait + // for the rest to be read. + state_off = u32::from(of_cell.next_state) + + (loaded.rotate_left(nb_all_bits) as u32 & u32::from(of_cell.state_mask)); } let count = room - slots.len() - unused; diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs index b84501d30b4d..fde720f01b63 100644 --- a/src/zstd/decompress/seq_table.rs +++ b/src/zstd/decompress/seq_table.rs @@ -41,7 +41,6 @@ pub(crate) struct SeqSymbol { /// How a table keeps a [`SeqSymbol`]. pub(crate) trait Cell: Copy + 'static { const EMPTY: Self; - const EMPTY_REF: &'static Self; /// [`NextState::at`] as `to_bits` of a cell where nothing else is set. const NEXT_STATES: &'static [u64; 4 * MAX_TABLE_SIZE]; /// The fields one after the other, from the lowest bits on. @@ -90,7 +89,6 @@ impl LengthCell { impl Cell for LengthCell { const EMPTY: Self = l(0, 0, 0, 0); - const EMPTY_REF: &'static Self = &Self::EMPTY; const NEXT_STATES: &'static [u64; 4 * MAX_TABLE_SIZE] = &{ let mut all = [0; 4 * MAX_TABLE_SIZE]; @@ -162,7 +160,6 @@ impl OffsetCell { impl Cell for OffsetCell { const EMPTY: Self = o(0, 0, 0); - const EMPTY_REF: &'static Self = &Self::EMPTY; const NEXT_STATES: &'static [u64; 4 * MAX_TABLE_SIZE] = &{ let mut all = [0; 4 * MAX_TABLE_SIZE]; @@ -297,11 +294,6 @@ impl SeqTable { .unpack() } - #[inline(always)] - pub(crate) fn cell(&self, state: usize) -> &C { - self.table.get(state & (N - 1)).unwrap_or(C::EMPTY_REF) - } - /// The states in use. pub(crate) fn symbols(&self) -> &[C] { self.table.get(..1 << self.table_log).unwrap_or_default() @@ -425,12 +417,38 @@ pub(crate) struct SeqTables { pub(crate) ll: LengthTable, pub(crate) of: OffsetTable, pub(crate) ml: LengthTable, + /// [`ADDITIONAL`], where it takes no more to refer to it than to the tables. + additional: [Additional; 32], } +/// What to make of the additional bits of a length. +#[derive(Clone, Copy)] +struct Additional { + /// As many bits set as there are additional bits. + mask: u32, + /// The bit that [`LengthCell::base_value`] lacks. + bit_16: u32, +} + +/// For each number of additional bits. +const ADDITIONAL: [Additional; 32] = { + let mut all = [Additional { mask: 0, bit_16: 0 }; 32]; + let mut left: &mut [Additional] = &mut all; + let mut nb_bits = 0; + while let [additional, others @ ..] = left { + additional.mask = (1 << nb_bits) - 1; + additional.bit_16 = ((nb_bits == 16) as u32) << 16; + nb_bits += 1; + left = others; + } + all +}; + static DEFAULT_TABLES: SeqTables = SeqTables { ll: SeqTable::predefined(LL_DEFAULT_NORM_LOG, 32, LL_DEFAULT), of: SeqTable::predefined(OF_DEFAULT_NORM_LOG, 31, OF_DEFAULT), ml: SeqTable::predefined(ML_DEFAULT_NORM_LOG, 21, ML_DEFAULT), + additional: ADDITIONAL, }; /// `LLTptr`, `OFTptr` and `MLTptr` @@ -457,6 +475,18 @@ impl SeqTables { ll: SeqTable::new(), of: SeqTable::new(), ml: SeqTable::new(), + additional: ADDITIONAL, + } + } + + /// What additional bits that are the lowest of `bits` make of the `base_value` of a [`LengthCell`]. + #[inline(always)] + pub(crate) fn length(&self, base_value: u16, nb_additional_bits: u8, bits: u64) -> u32 { + match self.additional.get(usize::from(nb_additional_bits) % 32) { + Some(additional) => { + u32::from(base_value) + additional.bit_16 + (bits as u32 & additional.mask) + } + None => 0, } } From 5c967a6cc70495e83d60e783f4ec46abaa94aba7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:48:43 +0000 Subject: [PATCH 077/199] zstd: a name for additional bits that is not that of another parameter --- src/zstd/decompress/seq_table.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs index b84501d30b4d..f1a56d4e28af 100644 --- a/src/zstd/decompress/seq_table.rs +++ b/src/zstd/decompress/seq_table.rs @@ -212,10 +212,10 @@ const fn length_codes(mut base_values: &[u32], mut nb_additional_bits: &[u8]) -> while let ( [code, other_codes @ ..], [base_value, other_values @ ..], - [nb_bits, other_nb_bits @ ..], + [more_bits, other_nb_bits @ ..], ) = (left, base_values, nb_additional_bits) { - *code = l(0, *nb_bits, 0, *base_value).to_bits(); + *code = l(0, *more_bits, 0, *base_value).to_bits(); (left, base_values, nb_additional_bits) = (other_codes, other_values, other_nb_bits); } codes From ff8bd0ec6ce0cd5cf29529f1ad09638bb5e1d100 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:48:43 +0000 Subject: [PATCH 078/199] zstd: a hash rate above the window does not make for a table of 8 GiB As upstream after 1.5.7 (facebook/zstd cef5a5611). --- src/zstd/compress/ldm.rs | 3 ++- test/js/bun/util/zstd.test.ts | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/zstd/compress/ldm.rs b/src/zstd/compress/ldm.rs index b1b17a6b677d..f0b3e89c4f15 100644 --- a/src/zstd/compress/ldm.rs +++ b/src/zstd/compress/ldm.rs @@ -182,9 +182,10 @@ pub(crate) fn adjust_parameters(params: &mut LdmParams, c_params: &CompressionPa } } if params.hash_log == 0 { + // Not as in 1.5.7, where the difference wraps around and the table is then of the largest size there is. params.hash_log = params .window_log - .wrapping_sub(params.hash_rate_log) + .saturating_sub(params.hash_rate_log) .clamp(ZSTD_HASHLOG_MIN as u32, ZSTD_HASHLOG_MAX as u32); } if params.min_match_length == 0 { diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 7d88d9221aec..239d5de2a7b9 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1431,3 +1431,22 @@ describe("frames that no encoder makes decode as with zstd 1.5.7", () => { } }); }); + +describe("bugs that zstd fixed after 1.5.7", () => { + // facebook/zstd cef5a5611: `windowLog - ldmHashRateLog` wrapped around, which made for a table of 8 GiB. + it("a hash rate above the window does not make for the largest table of long distance matches", async () => { + const script = ` + const zlib = require("node:zlib"); + const c = zlib.constants; + const input = Buffer.alloc(1024, "long distance "); + const output = zlib.zstdCompressSync(input, { + params: { [c.ZSTD_c_enableLongDistanceMatching]: 1, [c.ZSTD_c_ldmHashRateLog]: 11 }, + }); + console.log(input.equals(zlib.zstdDecompressSync(output)), process.resourceUsage().maxRSS < 4 * 1024 * 1024); + `; + await using proc = Bun.spawn({ cmd: [bunExe(), "-e", script], env: bunEnv, stdout: "pipe", stderr: "pipe" }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout, stderr: stderr.slice(0, 2000) }).toEqual({ stdout: "true true\n", stderr: "" }); + expect(exitCode).toBe(0); + }); +}); From 34ccfabfce756522142712e07d64e8421d341720 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:53:48 +0000 Subject: [PATCH 079/199] zstd: a small output is decoded where there is room to copy in chunks up to its end, and a match in a dictionary is copied so too --- src/zstd/decompress/block.rs | 94 ++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 19 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 850e716e3cb5..fa95f654f602 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -216,6 +216,8 @@ struct Scratch { lit_buffer: Vec, /// Sequences are decoded that many at a time, after 3 that only stand for the last offsets. batch: [RawSequence; 3 + 128], + /// Where a block that is all of a small output is decoded: with room to copy in chunks up to its end. + small_output: [u8; Self::MAX_SMALL_OUTPUT + WILDCOPY_OVERLENGTH], } thread_local! { @@ -225,6 +227,7 @@ thread_local! { } impl Scratch { + const MAX_SMALL_OUTPUT: usize = 1 << 12; /// A thread keeps no more than that for its next decoder. const MAX_SPARE_LIT_BUFFER: usize = 1 << 14; @@ -275,6 +278,7 @@ impl BlockDecoder { entropy: EntropyDTables::new(), lit_buffer: Vec::new(), batch: [RawSequence::EMPTY; _], + small_output: [0; _], }) }) }) @@ -589,6 +593,7 @@ impl BlockDecoder { entropy, lit_buffer, batch, + small_output, } = &mut **scratch; let dict_tables = dict_entropy.map(|dict| &dict.seq); let tables = if nb_seq <= MAX_ONE_BY_ONE { @@ -624,11 +629,23 @@ impl BlockDecoder { } else { literals.size }; + let is_small = start == prefix_start + && capacity <= Scratch::MAX_SMALL_OUTPUT + && literals.location == LitLocation::NotInDst; + let (out, copy_to) = match buf { + OutputBuffer::Whole(bytes) if is_small => ( + OutputBuffer::Whole(small_output.sub_mut(..capacity + WILDCOPY_OVERLENGTH)?), + Some(bytes.span_mut(start, capacity)?), + ), + OutputBuffer::Whole(bytes) => (OutputBuffer::Whole(bytes.sub_mut(..oend)?), None), + used => (used, None), + }; + let (start, prefix_start, oend) = match copy_to { + Some(_) => (0, 0, capacity), + None => (start, prefix_start, oend), + }; let mut exec = Executor { - out: match buf { - OutputBuffer::Whole(bytes) => OutputBuffer::Whole(bytes.sub_mut(..oend)?), - used => used, - }, + out, oend, op: start, prefix_start, @@ -651,7 +668,11 @@ impl BlockDecoder { *rep = prev_offset.map(|offset| offset as u32); } exec.last_literals()?; - Ok(exec.op - start) + let size = exec.op - start; + if let Some(to) = copy_to { + to.write_at(0, exec.out.bytes().sub(..size)?)?; + } + Ok(size) } } @@ -818,7 +839,7 @@ impl PaddedStart { } /// Up to how many sequences it is not worth it to decode them by the batch. -const MAX_ONE_BY_ONE: usize = 8; +const MAX_ONE_BY_ONE: usize = 32; /// `seqState_t` #[derive(Clone, Copy)] @@ -983,6 +1004,25 @@ impl<'a> SeqState<'a> { Ok(count) } + /// Decodes the last sequence, far from the start of the stream. + fn decode_last(&mut self) -> Option { + let ll_cell = self.tables.ll.table.get(self.state_ll)?; + let ml_cell = self.tables.ml.table.get(self.state_ml)?; + let of_cell = self.tables.of.table.get(self.state_off)?; + let of_bits = u32::from(of_cell.nb_additional_bits); + let ml_bits = u32::from(ml_cell.nb_additional_bits); + let ll_bits = u32::from(ll_cell.nb_additional_bits); + let offset_value = of_cell.first_value + self.bit_d.read_bits(of_bits) as u32; + let match_length = ml_cell.base_value() + self.bit_d.read_bits(ml_bits) as u32; + self.bit_d.reload(); + let lit_length = ll_cell.base_value() + self.bit_d.read_bits(ll_bits) as u32; + Some(RawSequence { + offset_value, + lit_length, + match_length, + }) + } + /// `ZSTD_decodeSequence` #[inline(always)] fn decode_sequence(&mut self, is_last_seq: bool) -> Sequence { @@ -1252,11 +1292,21 @@ impl<'a> Executor<'_, 'a> { } let o_lit_end = op + lit_length; - if offset > o_lit_end - self.prefix_start { - return None; - } let (before, room) = self.out.bytes().split_at_mut_checked(o_lit_end)?; - let from = before.get(before.len().checked_sub(offset)?..)?; + let in_prefix = o_lit_end - self.prefix_start; + let from = if offset > in_prefix { + // All of it in a dictionary, and then some to copy in chunks. + let ExtDict::Dict(dict) = self.ext_dict else { + return None; + }; + let back = offset - in_prefix; + if back < match_length + 16 { + return None; + } + dict.get(dict.len().checked_sub(back)?..)? + } else { + before.get(before.len().checked_sub(offset)?..)? + }; if offset >= 16 { // `ZSTD_wildcopy(ZSTD_no_overlap)` *room.first_chunk_mut::<16>()? = *from.first_chunk()?; @@ -1569,18 +1619,24 @@ impl<'a> Executor<'_, 'a> { while remaining > ahead.len() { let mut count = 0; if !self.split { - let room = (batch.len() - 3).min(remaining - ahead.len().max(1)); - let room = batch.span_mut(3, room)?; - let history = &mut self.history(); - count = seq_state.decode_batch(LONG, more_bits, room, history)?; - if count == 0 - && can_leave - && let Some(bit_d) = start.set(&seq_state.bit_d) - { + if can_leave && let Some(bit_d) = start.set(&seq_state.bit_d) { can_leave = false; left_at = Some((seq_state, remaining)); seq_state.bit_d = bit_d; - continue; + } + // The last sequence is followed by no state. + let room = (batch.len() - 4).min(remaining - ahead.len().max(1)); + if room > 0 { + let history = &mut self.history(); + let room = batch.span_mut(3, room)?; + count = seq_state.decode_batch(LONG, more_bits, room, history)?; + } + if left_at.is_some() + && count + 1 == remaining + && let Some(last) = seq_state.decode_last() + { + batch.put(3 + count, last)?; + count += 1; } } let mut sequence = None; From 510435d1118e50f2282c5654e832832c134a058c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:55:39 +0000 Subject: [PATCH 080/199] zstd: the loop over four Huffman streams knows how many bits it looks up --- src/zstd/decompress/huf.rs | 50 +++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index 14ce20b3f82d..e6741a7450cc 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -25,9 +25,11 @@ pub(super) struct Code { pub(super) table_log: u32, } -/// The loop that decodes four streams at once takes a table of this log, or one more. +/// No fewer bits are looked up at once. +const MIN_LOG: u32 = 8; +/// How many are if there are enough literals to make up for the time that it takes to fill the table. const FAST_LOG: u32 = 11; -/// From how many literals on that loop makes up for the time that it takes to fill such a table. +/// How many literals are enough for that. const FAST_MIN_LITERALS: usize = 2048; /// From how many literals on the larger table is considered. const LARGE_MIN_LITERALS: usize = 1 << 14; @@ -74,7 +76,7 @@ impl Code { /// How many bits to look up at once to decode `nb_literals`. fn best_log(&self, nb_literals: usize) -> u32 { if nb_literals < FAST_MIN_LITERALS { - return self.table_log; + return self.table_log.max(MIN_LOG); } // The larger table takes twice the time to fill, half of the cache, and a refill for 4 lookups and not 5. if self.table_log > FAST_LOG @@ -390,9 +392,11 @@ impl HufDTable { #[inline(never)] fn decode_most(&self, streams: &mut Streams<'_>) { match self.log { - 11 => streams.decode_most::<11, 5>(&self.table, 0), - 12 => streams.decode_most::<12, 4>(&self.table, 0), - log => streams.decode_most::<0, 5>(&self.table, 64 - log), + 8 => streams.decode_most::<8, 5>(&self.table), + 9 => streams.decode_most::<9, 5>(&self.table), + 10 => streams.decode_most::<10, 5>(&self.table), + 11 => streams.decode_most::<11, 5>(&self.table), + _ => streams.decode_most::<12, 4>(&self.table), } } @@ -758,28 +762,18 @@ impl<'a> Streams<'a> { ])) } - /// Decodes for as long as no stream is close to its start and no output to its end. A lookup is of `LOG` bits, or of - /// `64 - shift` if that is 0, and `LOOKUPS` of them are at most 55 bits. + /// Decodes for as long as no stream is close to its start and no output to its end. `LOOKUPS` of `LOG` bits are at + /// most 55 bits. #[inline(always)] - fn decode_most( - &mut self, - table: &[Entry; TABLE_SIZE], - shift: u32, - ) { + fn decode_most(&mut self, table: &[Entry; TABLE_SIZE]) { #[inline(always)] fn lookup( table: &[Entry; TABLE_SIZE], - shift: u32, bits: &mut u64, window: &mut [u8; WINDOW], done: &mut usize, ) { - let index = if LOG == 0 { - (*bits >> shift) as usize % TABLE_SIZE - } else { - (*bits >> (64 - LOG)) as usize - }; - if let Some(entry) = table.get(index) { + if let Some(entry) = table.get((*bits >> (64 - LOG)) as usize) { *bits <<= (entry >> 24) & 63; if let Some(to) = window.get_mut(*done..*done + 4) { to.copy_from_slice(&entry.to_le_bytes()); @@ -809,10 +803,10 @@ impl<'a> Streams<'a> { }; let (mut done0, mut done1, mut done2, mut done3) = (0, 0, 0, 0); for _ in 0..LOOKUPS { - lookup::(table, shift, &mut bits0, window0, &mut done0); - lookup::(table, shift, &mut bits1, window1, &mut done1); - lookup::(table, shift, &mut bits2, window2, &mut done2); - lookup::(table, shift, &mut bits3, window3, &mut done3); + lookup::(table, &mut bits0, window0, &mut done0); + lookup::(table, &mut bits1, window1, &mut done1); + lookup::(table, &mut bits2, window2, &mut done2); + lookup::(table, &mut bits3, window3, &mut done3); } let mut more = reload(s0.src, &mut ip0, &mut bits0) && reload(s1.src, &mut ip1, &mut bits1) @@ -820,10 +814,10 @@ impl<'a> Streams<'a> { && reload(s3.src, &mut ip3, &mut bits3); if more { for _ in 0..LOOKUPS { - lookup::(table, shift, &mut bits0, window0, &mut done0); - lookup::(table, shift, &mut bits1, window1, &mut done1); - lookup::(table, shift, &mut bits2, window2, &mut done2); - lookup::(table, shift, &mut bits3, window3, &mut done3); + lookup::(table, &mut bits0, window0, &mut done0); + lookup::(table, &mut bits1, window1, &mut done1); + lookup::(table, &mut bits2, window2, &mut done2); + lookup::(table, &mut bits3, window3, &mut done3); } more = reload(s0.src, &mut ip0, &mut bits0) && reload(s1.src, &mut ip1, &mut bits1) From a91dd57096db400497934c2a66c877a24dbc7ac0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:55:40 +0000 Subject: [PATCH 081/199] zstd: the older segment on its own goes by that name --- src/zstd/compress/lazy.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 1ec7b481a14b..55dde0b96b1c 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -365,10 +365,10 @@ fn sort_dubt1_in_dict( bt_low: u32, window_low: u32, ) -> Result<()> { - let no_dict = Prefix { buf: &[], base: 0 }; - let dict_end = input.dict.pos_of(input.dict_limit); - let dict = TreeInput::new(input.dict, no_dict, low_limit, dict_end)?; - sort_dubt1(bt, &dict, curr, nb_compares, bt_low, window_low) + let older = input.dict; + let nothing = Prefix { buf: &[], base: 0 }; + let older = TreeInput::new(older, nothing, low_limit, older.pos_of(input.dict_limit))?; + sort_dubt1(bt, &older, curr, nb_compares, bt_low, window_low) } /// The body of `ZSTD_insertDUBT1`, for a position of `input.near`. From dbf8feabb4a3cd46386fb016ffcca4f634ba66e3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:56:54 +0000 Subject: [PATCH 082/199] zstd: a block of a few literals is left as it is without making tables for the next one --- src/zstd/compress/cctx.rs | 68 ++++++++++++++++++++++++++--------- src/zstd/compress/literals.rs | 2 +- 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index be56ea5f8d62..d4c2ad572b6c 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -18,6 +18,7 @@ use super::internal::{ min_gain, no_compress_block, offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, }; +use super::literals::min_literals_to_compress; use super::params::{ CCtxParams, CParamMode, Format, MatchStateSizes, adjust_c_params_internal, allocate_chain_table, cdict_indices_are_tagged, compress_bound, cycle_log, @@ -1180,6 +1181,22 @@ impl FrameCompressor { /// `ZSTD_buildSeqStore` fn build_seq_store(&mut self, input: &Input<'_>, src: Span) -> Result { + Ok(match self.find_sequences(input, src)? { + Some(rep) => { + block_state_mut(&mut self.next_c_block).rep = rep; + BuildSeqStore::Compress + } + None => BuildSeqStore::NoCompress, + }) + } + + /// `ZSTD_buildSeqStore`, but for the repeat offsets after the block, which it returns rather than store. `None` is + /// `ZSTDbss_noCompress`. + fn find_sequences( + &mut self, + input: &Input<'_>, + src: Span, + ) -> Result> { let params = &self.applied_params; let ms = &mut self.match_state; let src_size = src.len(); @@ -1193,7 +1210,7 @@ impl FrameCompressor { params.c_params.min_match, )?; } - return Ok(BuildSeqStore::NoCompress); + return Ok(None); } self.seq_store.start_block(src_size)?; ms.opt.literal_compression_mode = ParamSwitch::Auto; @@ -1224,8 +1241,8 @@ impl FrameCompressor { }; let dict_mode = ms.dict_mode(block.dms.as_ref()); let use_row_match_finder = params.use_row_match_finder == ParamSwitch::Enable; - let rep = &mut block_state_mut(&mut self.next_c_block).rep; - *rep = block_state(&self.prev_c_block).rep; + let mut rep_after = block_state(&self.prev_c_block).rep; + let rep = &mut rep_after; let last_ll_size = if self.extern_seq_store.pos < self.extern_seq_store.size { ldm::block_compress( @@ -1270,7 +1287,18 @@ impl FrameCompressor { }; let last_literals = block.buf.sub(src.end - last_ll_size..src.end)?; self.seq_store.store_last_literals(last_literals)?; - Ok(BuildSeqStore::Compress) + Ok(Some(rep_after)) + } + + /// Whether the block is nothing but literals, too few for `ZSTD_compressLiterals` to try, which is larger than the + /// block is, if it fits in `dst_size` bytes at all. The tables for the next block need not be made to find that out. + fn is_too_short_to_compress(&self, dst_size: usize) -> bool { + let prev_huf = &block_state(&self.prev_c_block).entropy.huf; + let strategy = self.applied_params.c_params.strat(); + let store = &self.seq_store; + store.sequences.is_empty() + && store.lit_len < min_literals_to_compress(strategy, prev_huf.repeat_mode) + && store.lit_len + 3 + 4 <= dst_size } /// `ZSTD_compressSeqStore_singleBlock` @@ -1479,19 +1507,25 @@ impl FrameCompressor { const RLE_MAX_LENGTH: usize = 25; let bytes = src.bytes(&input.bufs)?; - let c_size = match self.build_seq_store(input, src)? { - BuildSeqStore::NoCompress => Ok(0), - BuildSeqStore::Compress => { - let whole = self.seq_store.whole(); - let mut c_size = entropy_compress_seq_store( - &mut self.seq_store, - &whole, - &block_state(&self.prev_c_block).entropy, - &mut block_state_mut(&mut self.next_c_block).entropy, - &self.applied_params, - dst, - bytes.len(), - ); + let c_size = match self.find_sequences(input, src)? { + None => Ok(0), + Some(rep) => { + let mut c_size = if self.is_too_short_to_compress(dst.len()) { + Ok(0) + } else { + let next = block_state_mut(&mut self.next_c_block); + next.rep = rep; + let whole = self.seq_store.whole(); + entropy_compress_seq_store( + &mut self.seq_store, + &whole, + &block_state(&self.prev_c_block).entropy, + &mut next.entropy, + &self.applied_params, + dst, + bytes.len(), + ) + }; if !self.is_first_block && c_size.is_ok_and(|c_size| c_size < RLE_MAX_LENGTH) && is_rle(bytes) diff --git a/src/zstd/compress/literals.rs b/src/zstd/compress/literals.rs index f143f67df024..c37e8c0c26fe 100644 --- a/src/zstd/compress/literals.rs +++ b/src/zstd/compress/literals.rs @@ -38,7 +38,7 @@ pub(crate) fn compress_rle_literals_block(dst: &mut [u8], src: &[u8]) -> Result< } /// `ZSTD_minLiteralsToCompress`: fewer literals than this are not worth a try. -fn min_literals_to_compress(strategy: Strategy, huf_repeat: HufRepeat) -> usize { +pub(crate) fn min_literals_to_compress(strategy: Strategy, huf_repeat: HufRepeat) -> usize { let shift = (9 - strategy as i32).min(3); if huf_repeat == HufRepeat::Valid { 6 From 460ed0b49875f62823138c2b7d9e44a698c90fe9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:00:39 +0000 Subject: [PATCH 083/199] zstd: zeros follow the literals of a block, and a thread's next decoder takes over no literals --- src/zstd/decompress/block.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 521b70ebffdb..313c23e5c9fd 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -219,8 +219,8 @@ struct Scratch { } thread_local! { - /// That of the last decoder that was dropped, for the next one. Nothing depends on what is in it, and it takes - /// longer to clear a new one than to decode a small frame. + /// That of the last decoder that was dropped, without its literals, for the next one. Nothing depends on what is + /// in the tables, and it takes longer to clear new ones than to decode a small frame. static SPARE_SCRATCH: cell::Cell>> = const { cell::Cell::new(None) }; } @@ -236,6 +236,8 @@ impl Scratch { .map_err(|_| ErrorCode::MemoryAllocation)?; self.lit_buffer.resize(needed, 0); } + // Literals are copied by more than there are: this much can be read along with the last ones. + self.lit_buffer.sub_mut(lit_size..needed)?.fill(0); self.lit_buffer.sub_mut(..lit_size) } } @@ -243,6 +245,7 @@ impl Scratch { impl Drop for BlockDecoder { fn drop(&mut self) { if let Some(mut scratch) = self.scratch.take() { + scratch.lit_buffer.clear(); if scratch.lit_buffer.capacity() > Scratch::MAX_SPARE_LIT_BUFFER { scratch.lit_buffer = Vec::new(); } From 8ff6f3f7da69a5c8f4684134fe2855c5ced6097c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:08:36 +0000 Subject: [PATCH 084/199] DecompressionStream: empty chunks in the middle of a zstd frame are not an error The decoder gives up after 16 calls in a row that neither read nor write. --- src/runtime/webcore/CompressionStreamCoder.rs | 4 ++++ test/js/web/streams/compression.test.ts | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/runtime/webcore/CompressionStreamCoder.rs b/src/runtime/webcore/CompressionStreamCoder.rs index 65080f81894f..0ba38a586b1a 100644 --- a/src/runtime/webcore/CompressionStreamCoder.rs +++ b/src/runtime/webcore/CompressionStreamCoder.rs @@ -609,6 +609,10 @@ impl CompressionStreamCoder { Backend::ZstdDecode(dctx) => { // After a frame completes, whatever follows must be another frame // magic (see `zstd_head`); `step` has already re-attached a split one. + if input.is_empty() && !finish && !continuing { + // The decoder gives up after 16 calls in a row that get it nowhere. + return Ok(Progress::Done); + } let mut input_buf = zstd::InBuffer { src: input, pos: 0 }; let mut out = zstd::VecOutput::new(out); loop { diff --git a/test/js/web/streams/compression.test.ts b/test/js/web/streams/compression.test.ts index a6229848f860..0579156ae455 100644 --- a/test/js/web/streams/compression.test.ts +++ b/test/js/web/streams/compression.test.ts @@ -347,6 +347,18 @@ describe("CompressionStream and DecompressionStream", () => { expect(out).toBe("payload"); }); + test("decompresses a zstd stream with many empty chunks in the middle of a frame", async () => { + const frame = zlib.zstdCompressSync(Buffer.from("a frame in two parts")); + const ds = new DecompressionStream("zstd"); + const writer = ds.writable.getWriter(); + const read = new Response(ds.readable).text(); + await writer.write(frame.subarray(0, 10)); + for (let i = 0; i < 20; i++) await writer.write(new Uint8Array(0)); + await writer.write(frame.subarray(10)); + await writer.close(); + expect(await read).toBe("a frame in two parts"); + }); + test("rejects trailing garbage after a zstd frame", async () => { const frame = zlib.zstdCompressSync(Buffer.from("hello")); const withJunk = Buffer.concat([frame, Buffer.from([0xde, 0xad, 0xbe, 0xef])]); From 3fe9b40849b35df29be37a4681008e35f3db6ded Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:08:36 +0000 Subject: [PATCH 085/199] zstd: with workers, input that is not of the pledged size is an error In 1.5.7 only a frame of one job is checked. Any other has the wrong size in its header and cannot be decoded. --- src/zstd/compress/stream.rs | 9 ++++++++- test/js/bun/util/zstd.test.ts | 19 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs index 5cfc01d8240d..5a4f8f1788a7 100644 --- a/src/zstd/compress/stream.rs +++ b/src/zstd/compress/stream.rs @@ -409,9 +409,16 @@ impl CCtx { loop { let ipos = input.pos; let opos = output.pos; - let flush_min = mtctx.compress_stream_generic(output, input, end_op); + let mut flush_min = mtctx.compress_stream_generic(output, input, end_op); self.frame.consumed_src_size += (input.pos - ipos) as u64; self.frame.produced_c_size += (output.pos - opos) as u64; + // Not in 1.5.7, where only a frame of one job is checked, and any other is one that cannot be decoded. + let pledged = self.frame.pledged_src_size_plus_one; + let consumed = self.frame.consumed_src_size + 1; + let is_complete = end_op == EndDirective::End && flush_min == Ok(0); + if pledged != 0 && (consumed > pledged || (is_complete && consumed != pledged)) { + flush_min = flush_min.and(Err(ErrorCode::SrcSizeWrong)); + } if flush_min.is_err() || (end_op == EndDirective::End && flush_min == Ok(0)) { // `reset_session`, but `mtctx` is borrowed. self.stream_stage = StreamStage::Init; diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 239d5de2a7b9..f41846890dcb 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1432,8 +1432,8 @@ describe("frames that no encoder makes decode as with zstd 1.5.7", () => { }); }); -describe("bugs that zstd fixed after 1.5.7", () => { - // facebook/zstd cef5a5611: `windowLog - ldmHashRateLog` wrapped around, which made for a table of 8 GiB. +describe("bugs of zstd 1.5.7", () => { + // Fixed by facebook/zstd cef5a5611: `windowLog - ldmHashRateLog` wrapped around, which made for a table of 8 GiB. it("a hash rate above the window does not make for the largest table of long distance matches", async () => { const script = ` const zlib = require("node:zlib"); @@ -1449,4 +1449,19 @@ describe("bugs that zstd fixed after 1.5.7", () => { expect({ stdout, stderr: stderr.slice(0, 2000) }).toEqual({ stdout: "true true\n", stderr: "" }); expect(exitCode).toBe(0); }); + + // With workers, only a frame of one job was checked. Any other came out with the wrong size in its header. + it.each([-1, 1])("with workers, input that is %i off the pledged size is an error", async difference => { + const c = zlib.constants; + const input = Buffer.alloc(3 << 20, "several jobs "); + const stream = zlib.createZstdCompress({ + pledgedSrcSize: input.length - difference, + params: { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 1 << 20 }, + }); + const { promise, resolve } = Promise.withResolvers(); + stream.on("error", resolve).on("end", resolve).resume(); + stream.write(input.subarray(0, 1000)); + stream.end(input.subarray(1000)); + expect(await promise).toMatchObject({ code: "ZSTD_error_srcSize_wrong" }); + }); }); From 745d76fb73e0310546972e889fa15ec524efec88 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:08:37 +0000 Subject: [PATCH 086/199] zstd: a streaming decoder starts each frame with empty buffers, and copies no more of the older part than there is --- src/zstd/decompress/block.rs | 3 ++- src/zstd/decompress/mod.rs | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 313c23e5c9fd..32e7999e282f 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -1472,7 +1472,8 @@ impl<'a> Executor<'_, 'a> { let mut match_end = o_lit_end + 3; if EXT == EXT_WITHIN && offset_value as usize > o_lit_end + 3 { let back = offset_value as usize - 3 - o_lit_end; - if back > within.len() || match_length > back { + // All that gets copied is of the older part, which is more than the match. + if back > within.len() || back < 64 { break; } match_end += within.end; diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 2e1b66cacba5..1036679af55b 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -1123,6 +1123,9 @@ impl DState { self.in_buff = Buffer::with_size(needed_in_buff_size)?; self.out_buff = Buffer::with_size(needed_out_buff_size)?; } + // Nothing of the frame before is there to be read. + self.in_buff.bytes.clear(); + self.out_buff.bytes.clear(); Ok(()) } From 7d3ca60ef7b5bcbb931ddcf416675412075eaba5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:11:27 +0000 Subject: [PATCH 087/199] zstd: a small output is decoded elsewhere only if nothing before it is in the same buffer --- src/zstd/decompress/block.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index c2b5722e257b..b75979ea8ac9 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -634,7 +634,8 @@ impl BlockDecoder { }; let is_small = start == prefix_start && capacity <= Scratch::MAX_SMALL_OUTPUT - && literals.location == LitLocation::NotInDst; + && literals.location == LitLocation::NotInDst + && matches!(ext_dict, ExtDict::Dict(_)); let (out, copy_to) = match buf { OutputBuffer::Whole(bytes) if is_small => ( OutputBuffer::Whole(small_output.sub_mut(..capacity + WILDCOPY_OVERLENGTH)?), From 27cd08852c6f513b9343de3ae064a8a41699c572 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:13:42 +0000 Subject: [PATCH 088/199] zstd: a compressed block that makes more than the largest block is an error in one call too In 1.5.7 only a streaming decoder checks that. Decoded in one call, such a block can overwrite literals that it has yet to read. --- src/zstd/decompress/mod.rs | 9 +++++++-- test/js/bun/util/zstd.test.ts | 9 +++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 1036679af55b..f9c956ebd220 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -850,14 +850,19 @@ impl DState { let decoded_size = match properties.block_type { BlockType::Compressed => { let capacity = dst.len() - op; - self.decompress_block( + let decoded_size = self.decompress_block( OutputBuffer::Whole(dst), op, capacity, block, ddict, Streaming::No, - )? + )?; + // Only a streaming decoder checks that in 1.5.7, where such a block can overwrite its own literals. + if decoded_size > self.f_params.block_size_max { + return Err(ErrorCode::CorruptionDetected); + } + decoded_size } BlockType::Raw => copy_raw_block(dst.sub_mut(op..)?, block)?, BlockType::Rle => set_rle_block( diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index f41846890dcb..19ebdf64ef3e 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1450,6 +1450,15 @@ describe("bugs of zstd 1.5.7", () => { expect(exitCode).toBe(0); }); + // Only a streaming decoder checked that. Decoded in one call, such a block could overwrite literals it had yet to read. + it("a compressed block that makes more than 128 KB is an error", () => { + const frame = Buffer.from("28b52ffd8050c6ff030040000041424344454647487500008dfb1f7a02540103340000030058", "hex"); + expect(() => zlib.zstdDecompressSync(frame, { chunkSize: 262086 })).toThrow( + expect.objectContaining({ code: "ZSTD_error_corruption_detected" }), + ); + expect(() => Bun.zstdDecompressSync(frame)).toThrow(); + }); + // With workers, only a frame of one job was checked. Any other came out with the wrong size in its header. it.each([-1, 1])("with workers, input that is %i off the pledged size is an error", async difference => { const c = zlib.constants; From deafcc00d05efd6251c4b14f7f318c44e2581dd4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:17:29 +0000 Subject: [PATCH 089/199] zstd: a long match does not end a run of common sequences --- src/zstd/decompress/block.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index b75979ea8ac9..7acc8a293b8d 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -1117,6 +1117,8 @@ fn copy_16(buf: &mut [u8], src: usize, dst: usize) -> bool { /// `buf[dst + i] = buf[src + i]`, one after the other, so that a match repeats itself where it overlaps its copy. /// `src` is less than `dst`. +#[cold] +#[inline(never)] fn copy_match_exact(buf: &mut [u8], src: usize, dst: usize, len: usize) -> Result<()> { let offset = dst - src; if offset >= len { @@ -1488,8 +1490,7 @@ impl<'a> Executor<'_, 'a> { if o_match_end + WILDCOPY_OVERLENGTH > buf.len() || lit_end + WILDCOPY_OVERLENGTH > lits.len() || lit_length > 32 - || match_length > 64 - || EXT == NO_EXT && offset_value as usize > o_lit_end + 3 + || (EXT == NO_EXT || match_length > 64) && offset_value as usize > o_lit_end + 3 { break; } @@ -1537,14 +1538,16 @@ impl<'a> Executor<'_, 'a> { } to.copy_from_slice(from); let match_pos = match_end - offset_value as usize; - if !copy_16(buf, match_pos, o_lit_end) || !copy_16(buf, match_pos + 16, o_lit_end + 16) - { - break; - } - if match_length > 32 - && (!copy_16(buf, match_pos + 32, o_lit_end + 32) - || !copy_16(buf, match_pos + 48, o_lit_end + 48)) - { + let is_copied = if match_length > 64 { + copy_match_exact(buf, match_pos, o_lit_end, match_length).is_ok() + } else { + copy_16(buf, match_pos, o_lit_end) + && copy_16(buf, match_pos + 16, o_lit_end + 16) + && (match_length <= 32 + || copy_16(buf, match_pos + 32, o_lit_end + 32) + && copy_16(buf, match_pos + 48, o_lit_end + 48)) + }; + if !is_copied { break; } @@ -1569,11 +1572,7 @@ impl<'a> Executor<'_, 'a> { if (1..=3).contains(&offset_value) { offset_value = resolve_offset(sequences, index); } - if index != first - && raw.lit_length <= 32 - && raw.match_length <= 64 - MINMATCH - && offset_value >= 16 + 3 - { + if index != first && raw.lit_length <= 32 && offset_value >= 16 + 3 { break; } exec.exec_sequence::(Sequence { From 80414584576ad7f67e61391a1f52b30cd044def4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:19:59 +0000 Subject: [PATCH 090/199] zstd: a streaming decoder refuses offsets beyond the window once its buffer has wrapped around The format does not allow them. 1.5.7 does not check, and gives what is in its buffer by then. --- src/zstd/decompress/block.rs | 34 +++++++++++++++++++++++++--------- src/zstd/decompress/mod.rs | 6 +++++- test/js/bun/util/zstd.test.ts | 9 +++++++++ 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 32e7999e282f..5533ee862aed 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -76,8 +76,12 @@ pub(crate) enum ExtDict<'a> { /// The content of a dictionary. Empty if there is none. Dict(&'a [u8]), /// The `len` bytes up to `end` of the buffer that the output goes to, which is the streaming decoder's and has - /// wrapped around. - Within { end: usize, len: usize }, + /// wrapped around. What is further back than `window` is being overwritten. + Within { + end: usize, + len: usize, + window: usize, + }, } impl ExtDict<'_> { @@ -89,6 +93,16 @@ impl ExtDict<'_> { } } + /// The largest offset there can be, `in_prefix` bytes after it. In 1.5.7 that is all there is in either case, and + /// a frame whose offsets go beyond its window gets whatever copies that go too far have left there. + #[inline(always)] + fn reach(&self, in_prefix: usize) -> usize { + match *self { + ExtDict::Dict(dict) => in_prefix + dict.len(), + ExtDict::Within { len, window, .. } => window.min(in_prefix + len), + } + } + /// `memmove(&buf[dst], dictEnd - back, len)` #[inline] fn copy_to(&self, buf: &mut [u8], dst: usize, back: usize, len: usize) -> Result<()> { @@ -1197,7 +1211,7 @@ impl<'a> Executor<'_, 'a> { let in_prefix = op - self.prefix_start; let mut match_pos = op.wrapping_sub(offset); if offset > in_prefix { - if offset > in_prefix + self.ext_dict.len() { + if offset > self.ext_dict.reach(in_prefix) { return Err(ErrorCode::CorruptionDetected); } let back = offset - in_prefix; @@ -1363,7 +1377,7 @@ impl<'a> Executor<'_, 'a> { /// The last offsets are those of the 3 sequences before, which are rewritten for that where it takes it. #[inline(never)] fn exec_batch(&mut self, sequences: &mut [RawSequence], index: usize) -> usize { - self.exec_batch_with::(sequences, index, &[], 0..0) + self.exec_batch_with::(sequences, index, &[], 0..0, 0) } /// The same where matches in `dict`, which comes before the output, are of the common kind too. @@ -1374,7 +1388,7 @@ impl<'a> Executor<'_, 'a> { index: usize, dict: &[u8], ) -> usize { - self.exec_batch_with::(sequences, index, dict, 0..0) + self.exec_batch_with::(sequences, index, dict, 0..0, 0) } /// The same for `within` of the buffer that the output goes to. @@ -1384,8 +1398,9 @@ impl<'a> Executor<'_, 'a> { sequences: &mut [RawSequence], index: usize, within: Range, + window: usize, ) -> usize { - self.exec_batch_with::(sequences, index, &[], within) + self.exec_batch_with::(sequences, index, &[], within, window) } /// What comes before the output is nothing of the common kind, `dict` or `within`, as `EXT` tells. @@ -1396,6 +1411,7 @@ impl<'a> Executor<'_, 'a> { mut index: usize, dict: &[u8], within: Range, + window: usize, ) -> usize { // Positions fit in 32 bits, so that the compiler sees that nothing added to them overflows. const MAX_BACK: usize = 3 << 30; @@ -1473,7 +1489,7 @@ impl<'a> Executor<'_, 'a> { if EXT == EXT_WITHIN && offset_value as usize > o_lit_end + 3 { let back = offset_value as usize - 3 - o_lit_end; // All that gets copied is of the older part, which is more than the match. - if back > within.len() || back < 64 { + if back > within.len() || back < 64 || offset_value as usize - 3 > window { break; } match_end += within.end; @@ -1651,8 +1667,8 @@ impl<'a> Executor<'_, 'a> { ExtDict::Dict(dict) if !dict.is_empty() => { self.exec_batch_after_dict(sequences, index, dict) } - ExtDict::Within { end, len } if len != 0 => { - self.exec_batch_after_wrap(sequences, index, end - len..end) + ExtDict::Within { end, len, window } if len != 0 => { + self.exec_batch_after_wrap(sequences, index, end - len..end, window) } _ => self.exec_batch(sequences, index), }; diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index f9c956ebd220..32ba78b676f9 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -798,7 +798,11 @@ impl DState { ) -> Result { let ext_dict = match self.ext { ExtKind::Dict => ExtDict::Dict(ddict.map_or(&[], |ddict| &*ddict.content)), - ExtKind::Within { end, len } => ExtDict::Within { end, len }, + ExtKind::Within { end, len } => ExtDict::Within { + end, + len, + window: self.f_params.window_size as usize, + }, }; self.block.decompress_block( BlockOutput { diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 19ebdf64ef3e..3b81bf54d6ee 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1459,6 +1459,15 @@ describe("bugs of zstd 1.5.7", () => { expect(() => Bun.zstdDecompressSync(frame)).toThrow(); }); + // No decoder checked that. A streaming one keeps little more than the window, and gave whatever was there by then. + it("an offset beyond the window is an error once a streaming decoder has let go of what it is for", () => { + // A window of 1 KB, 1 KB each of "a", "b" and "c", then "xyz" and 4 bytes from 1500 bytes back. + const frame = Buffer.from("28b52ffd00000220006102200062022000635d00001878797a0154030a01df05", "hex"); + expect(() => zlib.zstdDecompressSync(frame)).toThrow( + expect.objectContaining({ code: "ZSTD_error_corruption_detected" }), + ); + }); + // With workers, only a frame of one job was checked. Any other came out with the wrong size in its header. it.each([-1, 1])("with workers, input that is %i off the pledged size is an error", async difference => { const c = zlib.constants; From ddf9cc3be674080224b69825f3dd9ed6fc5abde6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:20:55 +0000 Subject: [PATCH 091/199] zstd: fewer copies and fewer batches for a small block --- src/zstd/decompress/block.rs | 8 +++++--- src/zstd/decompress/huf.rs | 16 ++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 7acc8a293b8d..e70c4a338340 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -803,7 +803,7 @@ struct PaddedStart(OnceCell<[u8; Self::PADDING + Self::MAX_SIZE]>); impl PaddedStart { const PADDING: usize = 32; - const MAX_SIZE: usize = 32; + const MAX_SIZE: usize = 224; const fn new() -> Self { Self(OnceCell::new()) @@ -1741,8 +1741,10 @@ impl<'a> Executor<'_, 'a> { } self.out.extend_to(self.op + size, self.op + size); for part in parts { - self.out.bytes().write_at(self.op, part)?; - self.op += part.len(); + if !part.is_empty() { + self.out.bytes().write_at(self.op, part)?; + self.op += part.len(); + } } Ok(()) } diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index e6741a7450cc..27348ee7df98 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -369,7 +369,7 @@ impl HufDTable { return false; }; stream.decode_most(&self.table, self.log); - self.decode_tails([stream]) + self.decode_tails(&mut [stream]) } /// Four streams after their sizes. Whether each is exactly its share of `dst.len()` symbols. @@ -386,7 +386,7 @@ impl HufDTable { if !streams.0.iter().all(Stream::fits_tail) { return self.decode_stragglers(streams); } - self.decode_4_tails(streams) + self.decode_4_tails(&mut streams) } #[inline(never)] @@ -438,19 +438,19 @@ impl HufDTable { } self.decode_most(&mut decoding); } - self.decode_4_tails(streams) + self.decode_4_tails(&mut streams) } #[inline(never)] - fn decode_4_tails(&self, streams: Streams<'_>) -> bool { - self.decode_tails(streams.0) + fn decode_4_tails(&self, streams: &mut Streams<'_>) -> bool { + self.decode_tails(&mut streams.0) } /// Decodes what is left of streams. Whether each turns out to be exactly as many symbols as its output takes. #[inline(always)] - fn decode_tails(&self, mut streams: [Stream<'_>; N]) -> bool { + fn decode_tails(&self, streams: &mut [Stream<'_>; N]) -> bool { let mut tails = [const { Tail::new() }; N]; - for (tail, stream) in tails.iter_mut().zip(&streams) { + for (tail, stream) in tails.iter_mut().zip(&*streams) { if !tail.set(stream) { return false; } @@ -478,7 +478,7 @@ impl HufDTable { } } let mut exact = true; - for (tail, stream) in tails.iter().zip(&mut streams) { + for (tail, stream) in tails.iter().zip(streams) { exact &= tail.get(stream, self.code.table_log); } exact From 9e0d3954c256c36615d211fc94942c8f3a934e4d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:27:32 +0000 Subject: [PATCH 092/199] zstd: workers reset their tables and make their contexts like anyone else --- src/zstd/compress/cctx.rs | 6 ------ src/zstd/compress/mt.rs | 7 +++---- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 790216715118..9f998b06cafd 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -159,8 +159,6 @@ pub(crate) fn reset_table( Ok(()) } -pub(crate) use reset_table as reset_struct_table; - /// `ZSTD_cwksp_mark_tables_dirty` and then `ZSTD_cwksp_mark_tables_clean` fn forget_unused_indices(ms: &mut MatchState) { ms.hash_table.forget_rest(); @@ -877,10 +875,6 @@ impl FrameCompressor { } } - pub(crate) fn for_worker() -> Self { - Self::new() - } - /// `ZSTD_resetCCtx_internal` fn reset_internal( &mut self, diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 864572177a6f..12a2160e7c55 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -13,8 +13,7 @@ use std::sync::mpmc::{Receiver, Sender, TryRecvError, channel}; use std::thread::JoinHandle; use super::cctx::{ - CDict, DictContentType, FrameCompressor, Input, Span, reset_struct_table, reset_table, - write_last_empty_block, + CDict, DictContentType, FrameCompressor, Input, Span, reset_table, write_last_empty_block, }; use super::internal::{ BufId, Buffers, DictTableLoadMethod, LdmParams, LdmState, ParamSwitch, RawSeq, RawSeqStore, @@ -67,7 +66,7 @@ impl SerialState { if ldm_params.enable_ldm == ParamSwitch::Enable { let bucket_log = ldm_params.hash_log - ldm_params.bucket_size_log; self.ldm_state.window = Window::new(); - reset_struct_table( + reset_table( &mut self.ldm_state.hash_table, 1 << ldm_params.hash_log, true, @@ -301,7 +300,7 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - fn worker_main(jobs: &Receiver, cancelled: &AtomicBool) { let mut state = WorkerState { - cctx: FrameCompressor::for_worker(), + cctx: FrameCompressor::new(), seq: Vec::new(), dst: Vec::new(), }; From bc18f7720ff6eeb1459d137c1cfddac5d7deb3fa Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:34:59 +0000 Subject: [PATCH 093/199] zstd: with matches far back, what few sequences a block has are known ahead of time too, and so are the last ones --- src/zstd/decompress/block.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 04e27a75c9c6..a30f8bd84d5b 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -613,15 +613,16 @@ impl BlockDecoder { small_output, } = &mut **scratch; let dict_tables = dict_entropy.map(|dict| &dict.seq); - let tables = if nb_seq <= MAX_ONE_BY_ONE { + let is_history_long = total_history_size > (1 << 24); + let tables = if nb_seq <= MAX_ONE_BY_ONE && !use_prefetch_decoder && !is_history_long { entropy.seq.pick(self.seq_sources, dict_tables) } else { entropy.seq.gather(&mut self.seq_sources, dict_tables) }; - // With matches far back in a long history, it pays to know the sequences ahead of time. Nothing is prefetched - // here, but which of two problems in a damaged block is reported depends on the order. - if !use_prefetch_decoder && total_history_size > (1 << 24) && nb_seq > 8 { + // With matches far back in a long history, it pays to know the sequences ahead of time. Which of two problems in + // a damaged block is reported depends on that too. + if !use_prefetch_decoder && is_history_long && nb_seq > 8 { const MIN_SHARE: u32 = 7; use_prefetch_decoder = long_offset_share(tables.of) >= MIN_SHARE; } @@ -1623,9 +1624,10 @@ impl<'a> Executor<'_, 'a> { let mut remaining = nb_seq; // Long copies go on while the next sequences are decoded, if that is what comes next. And a batch of a few // sequences is not worth it. - if (seq_state.tables.matches_are_long() || nb_seq <= MAX_ONE_BY_ONE) && !self.split { + if (seq_state.tables.matches_are_long() || nb_seq <= MAX_ONE_BY_ONE) && !LONG && !self.split + { let mut exec = self.fork(); - while remaining > ahead.len() { + while remaining > 0 { let sequence = seq_state.decode_sequence(remaining == 1); remaining -= 1; exec.exec_sequence::(sequence)?; @@ -1685,9 +1687,15 @@ impl<'a> Executor<'_, 'a> { } } } + let history = &mut self.history(); for sequence in &mut *ahead { *sequence = seq_state.decode_sequence(remaining == 1); remaining -= 1; + history.prefetch( + sequence.lit_length, + sequence.match_length - MINMATCH, + (sequence.offset as u32).saturating_add(3), + ); } let is_at_end = match left_at { Some(_) => PaddedStart::is_reached(&seq_state.bit_d), From b5f24e01563a05e5b9364de17f99c9b1c47805b1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:38:07 +0000 Subject: [PATCH 094/199] zstd: without rare symbols, the states of a table find their symbols where these are listed in order --- src/zstd/common/fse_decompress.rs | 94 +++++++++++++++++++++++-------- src/zstd/decompress/seq_table.rs | 20 +++---- 2 files changed, 78 insertions(+), 36 deletions(-) diff --git a/src/zstd/common/fse_decompress.rs b/src/zstd/common/fse_decompress.rs index 9d83c5b935b1..cc9da08043d0 100644 --- a/src/zstd/common/fse_decompress.rs +++ b/src/zstd/common/fse_decompress.rs @@ -87,9 +87,10 @@ pub(crate) const MAX_SPREAD_SIZE: usize = 1 << 9; pub(crate) struct Spread { /// The symbols, each as many times as it has states. in_order: [u8; MAX_SPREAD_SIZE + 8], - pub(crate) symbols: [u8; MAX_SPREAD_SIZE], - /// For each symbol, what to give [`NextState::at`] for its first state. - pub(crate) symbol_next: [u16; FSE_MAX_SYMBOL_VALUE as usize + 1], + /// The symbol of each state. + by_state: [u8; MAX_SPREAD_SIZE + 8], + /// For each symbol, what to give [`NextState::at`] for its next state. + symbol_next: [u16; FSE_MAX_SYMBOL_VALUE as usize + 1], /// The symbols of low probability. rare: [u8; FSE_MAX_SYMBOL_VALUE as usize + 1], /// Room for the counts that all that is made from. @@ -100,7 +101,7 @@ impl Spread { pub(crate) const fn new() -> Self { Self { in_order: [0; _], - symbols: [0; _], + by_state: [0; _], symbol_next: [0; _], rare: [0; _], counts: [0; _], @@ -109,8 +110,15 @@ impl Spread { /// The first two passes of `FSE_buildDTable` and `ZSTD_buildFSETable`, with few branches to mispredict, for the /// first `nb_symbols` of `counts`. They add up to the size of the table, which is at most [`MAX_SPREAD_SIZE`]. - pub(crate) fn set(&mut self, nb_symbols: usize, table_log: u32) -> Result<()> { + pub(crate) fn states(&mut self, nb_symbols: usize, table_log: u32) -> Result> { let table_size = 1usize << table_log; + let table_mask = table_size - 1; + let step = fse_table_step(table_size); + // What `step` times this is 1 more than a multiple of the size of the table. + let mut stride = step; + for _ in 0..2 { + stride = stride.wrapping_mul(2usize.wrapping_sub(step.wrapping_mul(stride))); + } let mut nb_rare = 0usize; let mut nb_in_order = 0usize; let mut eight_times = 0u64; @@ -138,25 +146,73 @@ impl Spread { eight_times += 0x0101_0101_0101_0101; } + if nb_rare == 0 { + // One symbol after the other goes `step` states further, so the symbol of the next state is `stride` symbols + // further. + return Ok(States { + symbols: &self.in_order, + index: 0, + stride: stride & table_mask, + mask: table_mask, + symbol_next: &mut self.symbol_next, + }); + } + // The states from here on are those of the rare symbols. Others that would go there go to the next place. let first_rare = table_size - nb_rare.min(table_size); - let table_mask = table_size - 1; - let step = fse_table_step(table_size); - let symbols = self.symbols.sub_mut(..table_size)?; + let by_state = self.by_state.sub_mut(..table_size)?; let mut position = 0usize; let mut from = 0usize; for _ in 0..table_size / 2 { for position in [position, (position + step) & table_mask] { let symbol = self.in_order.get(from % MAX_SPREAD_SIZE).copied(); - symbols.put(position, symbol.unwrap_or(0))?; + by_state.put(position, symbol.unwrap_or(0))?; from += usize::from(position < first_rare); } position = (position + 2 * step) & table_mask; } - for (symbol, &rare) in symbols.iter_mut().rev().zip(self.rare.sub(..nb_rare)?) { + for (symbol, &rare) in by_state.iter_mut().rev().zip(self.rare.sub(..nb_rare)?) { *symbol = rare; } - Ok(()) + Ok(States { + symbols: &self.by_state, + index: 0, + stride: 1, + mask: table_mask, + symbol_next: &mut self.symbol_next, + }) + } +} + +/// The states of a table, from the first on. +pub(crate) struct States<'a> { + symbols: &'a [u8; MAX_SPREAD_SIZE + 8], + /// Where in `symbols` the symbol of the next state is. + index: usize, + /// How much further the one after it is, around `mask`. + stride: usize, + mask: usize, + symbol_next: &'a mut [u16; FSE_MAX_SYMBOL_VALUE as usize + 1], +} + +/// A state of a table. +pub(crate) struct State { + pub(crate) symbol: u8, + /// What to give [`NextState::at`]. + pub(crate) next: u16, +} + +impl States<'_> { + #[inline(always)] + pub(crate) fn next(&mut self) -> State { + let symbol = (self.symbols.get(self.index % MAX_SPREAD_SIZE)).map_or(0, |&symbol| symbol); + self.index = (self.index + self.stride) & self.mask; + let mut next = 0; + if let Some(symbol_next) = self.symbol_next.get_mut(usize::from(symbol)) { + next = *symbol_next; + *symbol_next = next.wrapping_add(1); + } + State { symbol, next } } } @@ -197,18 +253,10 @@ fn build_dtable( return Err(ErrorCode::TableLogTooLarge); } dt.table_log = table_log; - spread.set(max_symbol_value as usize + 1, table_log)?; - for (decode, &symbol) in dt - .table - .sub_mut(..1 << table_log)? - .iter_mut() - .zip(&spread.symbols) - { - let Some(next) = spread.symbol_next.get_mut(usize::from(symbol)) else { - continue; - }; - let next_state = NEXT_STATES.get(usize::from(*next) % NEXT_STATES.len()); - *next = next.wrapping_add(1); + let mut states = spread.states(max_symbol_value as usize + 1, table_log)?; + for decode in dt.table.sub_mut(..1 << table_log)? { + let State { symbol, next } = states.next(); + let next_state = NEXT_STATES.get(usize::from(next) % NEXT_STATES.len()); *decode = FseDecode { symbol, ..*next_state.unwrap_or(&FseDecode::EMPTY) diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs index 771b6068e919..cfdb803b89e0 100644 --- a/src/zstd/decompress/seq_table.rs +++ b/src/zstd/decompress/seq_table.rs @@ -1,6 +1,6 @@ //! The FSE tables that sequences are decoded with: `ZSTD_seqSymbol` and `ZSTD_buildFSETable`. -use crate::common::fse_decompress::{MAX_SPREAD_SIZE, NextState, Spread}; +use crate::common::fse_decompress::{MAX_SPREAD_SIZE, NextState, Spread, State}; use crate::common::{ LL_BITS, LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, ML_BITS, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM_LOG, @@ -331,18 +331,12 @@ impl SeqTable { self.table_log = table_log; self.with_more_bits = share_with_more_bits(spread.counts.sub(..nb_symbols)?, codes, table_log); - spread.set(nb_symbols, table_log)?; - - for (cell, &symbol) in table_decode.iter_mut().zip(&spread.symbols) { - let symbol = usize::from(symbol); - let (Some(next), Some(code)) = - (spread.symbol_next.get_mut(symbol), codes.get(symbol % 64)) - else { - continue; - }; - let next_state = C::NEXT_STATES.get(usize::from(*next) % (4 * MAX_TABLE_SIZE)); - *next = next.wrapping_add(1); - *cell = C::from_bits(code | next_state.copied().unwrap_or(0)); + let mut states = spread.states(nb_symbols, table_log)?; + for cell in table_decode { + let State { symbol, next } = states.next(); + let code = codes.get(usize::from(symbol) % 64).copied(); + let next_state = C::NEXT_STATES.get(usize::from(next) % (4 * MAX_TABLE_SIZE)); + *cell = C::from_bits(code.unwrap_or(0) | next_state.copied().unwrap_or(0)); } Ok(()) } From 916f38a982e14caeec0e132a339f2525dd5fb18d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:39:41 +0000 Subject: [PATCH 095/199] node:zlib: a zstd dictionary that cannot be loaded is an error As in Node, the constructor throws ERR_ZLIB_INITIALIZATION_FAILED. The stream used to end without output. --- src/runtime/node/zlib/NativeZstd.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/runtime/node/zlib/NativeZstd.rs b/src/runtime/node/zlib/NativeZstd.rs index 5047cec4902f..110dcbb82d46 100644 --- a/src/runtime/node/zlib/NativeZstd.rs +++ b/src/runtime/node/zlib/NativeZstd.rs @@ -293,8 +293,15 @@ mod _impl { .stream .with_mut(|s| s.init(pledged_src_size, dictionary)); if err.is_error() { + // SAFETY: is_error() ⇔ msg is non-null; it points at a NUL-terminated C string. + let msg = unsafe { bun_core::ffi::cstr(err.msg) }.to_bytes(); self.emit_error(global, this_value, err); - return Ok(false); + return Err(global + .err( + jsc::ErrorCode::ZLIB_INITIALIZATION_FAILED, + format_args!("{}", bstr::BStr::new(msg)), + ) + .throw()); } let Some(mut params_) = init_params_array_value.as_array_buffer(global) else { From b8025bff4221fd109780174fe6c6e25cb71cee1d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:39:42 +0000 Subject: [PATCH 096/199] zstd: the total of the Huffman weights comes from how many there are of each --- src/zstd/common/entropy.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/zstd/common/entropy.rs b/src/zstd/common/entropy.rs index 50d2e0a188fa..433c5ec890ae 100644 --- a/src/zstd/common/entropy.rs +++ b/src/zstd/common/entropy.rs @@ -315,30 +315,34 @@ pub(crate) fn huf_read_stats_wksp( // Four counts of each weight: one that is incremented again and again has to be waited for. let mut counts = [[0u32; 16]; 4]; - let mut weight_total: u32 = 0; - let mut max_weight = 0; + let mut all_bits = 0; let (fours, others) = huff_weight.sub(..o_size)?.as_chunks::<4>(); for weights in fours.iter().chain(&[pad(others)]) { + all_bits |= u32::from_le_bytes(*weights); for (&weight, counts) in weights.iter().zip(&mut counts) { if let Some(count) = counts.get_mut(usize::from(weight) % 16) { *count += 1; } - weight_total += (1 << (weight % 16)) >> 1; - max_weight = max_weight.max(weight); } } - if u32::from(max_weight) > HUF_TABLELOG_MAX { - return Err(ErrorCode::CorruptionDetected); - } let [counts0, counts1, counts2, counts3] = &counts; - for (weight, rank) in rank_stats.iter_mut().enumerate() { + let mut ranks = [0u32; 16]; + for (weight, rank) in ranks.iter_mut().enumerate() { *rank = counts0.at(weight)? + counts1.at(weight)? + counts2.at(weight)? + counts3.at(weight)?; } + let (valid, too_large) = ranks.split_at(rank_stats.len()); + if all_bits & 0xF0F0_F0F0 != 0 || too_large.iter().any(|&rank| rank != 0) { + return Err(ErrorCode::CorruptionDetected); + } + rank_stats.copy_from_slice(valid); // Those that complete the last four. if let Some(rank) = rank_stats.first_mut() { *rank -= (4 - others.len()) as u32; } + let weight_total = (rank_stats.iter().enumerate().skip(1)) + .map(|(weight, &rank)| rank << (weight - 1)) + .sum::(); if weight_total == 0 { return Err(ErrorCode::CorruptionDetected); } From dcbcaf150f956f74d9e7bf46ef5ffc1a8e1fd157 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:42:48 +0000 Subject: [PATCH 097/199] zstd: long matches far back are still copied while the next sequences are decoded --- src/zstd/decompress/block.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index a30f8bd84d5b..59ee9d61f2b3 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -1624,10 +1624,10 @@ impl<'a> Executor<'_, 'a> { let mut remaining = nb_seq; // Long copies go on while the next sequences are decoded, if that is what comes next. And a batch of a few // sequences is not worth it. - if (seq_state.tables.matches_are_long() || nb_seq <= MAX_ONE_BY_ONE) && !LONG && !self.split + if (seq_state.tables.matches_are_long() || nb_seq <= MAX_ONE_BY_ONE && !LONG) && !self.split { let mut exec = self.fork(); - while remaining > 0 { + while remaining > ahead.len() { let sequence = seq_state.decode_sequence(remaining == 1); remaining -= 1; exec.exec_sequence::(sequence)?; From 977ef37a1171fb8c9fb7824214551d43f2bd18ef Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:44:54 +0000 Subject: [PATCH 098/199] zstd: fast and dfast have one version for all parameters but the default ones, each of which has its own --- src/zstd/compress/double_fast.rs | 189 +++++++++++++++---------------- src/zstd/compress/fast.rs | 125 ++++++++++---------- src/zstd/compress/internal.rs | 14 +-- 3 files changed, 158 insertions(+), 170 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index a639145ad099..0e85fe129685 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -5,10 +5,10 @@ use super::fast::{ restore_rep, }; use super::internal::{ - BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, MatchState, Near, Prefix, - REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, hash_of, hash_ptr, - hash8, hashed_get, hashed_put, hashed_swap, le64_at, offset_to_offbase, table_of_size, - table_prefetch, write_tagged_index, + BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, Hasher, K_SEARCH_STRENGTH, MatchState, Near, + Prefix, REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, hash_ptr, + hashed_get, hashed_put, hashed_swap, le64_at, offset_to_offbase, table_of_size, table_prefetch, + write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -109,17 +109,17 @@ pub(crate) fn fill_double_hash_table( /// Has what a search from `pos` on looks at first in the tables on its way. `pos` is the end of a match that was just /// found: there is enough to do about that one in the meantime. #[inline(always)] -fn look_ahead( +fn look_ahead( near: Near<'_>, pos: u32, hash_long: &[u32], - h_bits_l: u32, + long: Hasher, hash_small: &[u32], - h_bits_s: u32, + small: Hasher, ) { if let Some(bytes) = near.read64(pos) { - table_prefetch(hash_long, hash8(bytes, h_bits_l, 0)); - table_prefetch(hash_small, hash_of::(bytes, h_bits_s)); + table_prefetch(hash_long, long.hash(bytes)); + table_prefetch(hash_small, small.hash(bytes)); } } @@ -127,8 +127,8 @@ fn look_ahead( struct Finder { /// The index of the first position. base_index: u32, - h_bits_l: u32, - h_bits_s: u32, + long: Hasher, + small: Hasher, /// Where to start. first: u32, offset_1: u32, @@ -141,7 +141,7 @@ impl Finder { /// The C code computes the long hash and reads its table one position ahead. Here that is left to the processor: a /// turn of the inner loop only takes the position and the 8 bytes there, `bytes0`, from the one before. /// - /// `LONG` and `SMALL` are `h_bits_l` and `h_bits_s`, or both 0 for any: see [`table_of_size`]. + /// `MLS`, `LONG` and `SMALL` are what `small` and `long` are made with, or all 0 for any: see [`table_of_size`]. #[inline(never)] fn find_matches( &mut self, @@ -152,15 +152,19 @@ impl Finder { const K_STEP_INCR: u32 = 1 << K_SEARCH_STRENGTH; let Self { base_index, - h_bits_l, - h_bits_s, + long, + small, first: mut ip, mut offset_1, mut offset_2, } = *self; let near = sink.near; - let h_bits_l = if LONG > 0 { LONG } else { h_bits_l }; - let h_bits_s = if SMALL > 0 { SMALL } else { h_bits_s }; + let long = if LONG > 0 { Hasher::new(LONG, 8) } else { long }; + let small = if SMALL > 0 { + Hasher::new(SMALL, MLS) + } else { + small + }; let Some(hash_long) = table_of_size::(hash_long) else { return; }; @@ -184,15 +188,15 @@ impl Finder { break 'block; }; curr = base_index.wrapping_add(ip); - let idxl0 = hashed_swap(hash_long, hash8(bytes0, h_bits_l, 0), curr); - let idxs0 = hashed_swap(hash_small, hash_of::(bytes0, h_bits_s), curr); + let idxl0 = hashed_swap(hash_long, long.hash(bytes0), curr); + let idxs0 = hashed_swap(hash_small, small.hash(bytes0), curr); if sink.read32_before(ip + 1, offset_1) == Some((bytes0 >> 8) as u32) && offset_1 > 0 { ip += 1; let end = ip + 4 + sink.count_before(ip + 4, offset_1); - look_ahead::(near, end, hash_long, h_bits_l, hash_small, h_bits_s); + look_ahead(near, end, hash_long, long, hash_small, small); sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); break end; } @@ -205,7 +209,7 @@ impl Finder { } else if near.has::<0xFFFF_FFFF>(matchs0, bytes0) { // A short match: a longer one at the next position is preferred. let end = ip + 4 + near.count(ip + 4, matchs0 + 4); - let idxl1 = hashed_get(hash_long, hash8(bytes1, h_bits_l, 0)); + let idxl1 = hashed_get(hash_long, long.hash(bytes1)); let matchl1 = idxl1.wrapping_sub(base_index); if near.has::<{ u64::MAX }>(matchl1, bytes1) && matchl1 > 0 { let end1 = ip1 + 8 + near.count(ip1 + 8, matchl1 + 8); @@ -230,16 +234,12 @@ impl Finder { continue; }; - look_ahead::(near, end, hash_long, h_bits_l, hash_small, h_bits_s); + look_ahead(near, end, hash_long, long, hash_small, small); offset_2 = offset_1; offset_1 = ip - match0; // With a larger step, ip1 may be past the end of the match. if step < 4 { - hashed_put( - hash_long, - hash8(bytes1, h_bits_l, 0), - base_index.wrapping_add(ip1), - ); + hashed_put(hash_long, long.hash(bytes1), base_index.wrapping_add(ip1)); } let back = near.count_backwards(ip, sink.anchor, match0, 0); sink.store::<16>(ip - back, end, offset_to_offbase(offset_1)); @@ -257,21 +257,17 @@ impl Finder { }; let index_to_insert = curr + 2; if let Some(to_insert) = near.read64(index_to_insert.wrapping_sub(base_index)) { - hashed_put(hash_long, hash8(to_insert, h_bits_l, 0), index_to_insert); - hashed_put( - hash_small, - hash_of::(to_insert, h_bits_s), - index_to_insert, - ); + hashed_put(hash_long, long.hash(to_insert), index_to_insert); + hashed_put(hash_small, small.hash(to_insert), index_to_insert); } hashed_put( hash_long, - hash8(le64_at::<0, _>(around), h_bits_l, 0), + long.hash(le64_at::<0, _>(around)), base_index.wrapping_add(ip - 2), ); hashed_put( hash_small, - hash_of::(le64_at::<1, _>(around), h_bits_s), + small.hash(le64_at::<1, _>(around)), base_index.wrapping_add(ip - 1), ); @@ -279,8 +275,8 @@ impl Finder { let end = ip + 4 + sink.count_before(ip + 4, offset_2); core::mem::swap(&mut offset_1, &mut offset_2); let index = base_index.wrapping_add(ip); - hashed_put(hash_small, hash_of::(bytes0, h_bits_s), index); - hashed_put(hash_long, hash8(bytes0, h_bits_l, 0), index); + hashed_put(hash_small, small.hash(bytes0), index); + hashed_put(hash_long, long.hash(bytes0), index); sink.store_repeat(end); ip = end; let Some(bytes) = near.read64(ip) else { @@ -307,8 +303,8 @@ pub(crate) fn compress_block_double_fast( let h_bits_s = ms.c_params.chain_log; let mut finder = Finder { base_index: block.prefix_lowest_index, - h_bits_l, - h_bits_s, + long: Hasher::new(h_bits_l, 8), + small: Hasher::new(h_bits_s, ms.c_params.min_match), first: block.first, offset_1: block.rep.offset_1, offset_2: block.rep.offset_2, @@ -318,15 +314,20 @@ pub(crate) fn compress_block_double_fast( let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; // First, what levels 3 and 4 come with if the input is not known to be small, and level 3 if it is. match (ms.c_params.min_match, h_bits_l, h_bits_s) { - (5, 17, 16) => finder.find_matches::<5, 17, 16>(hash_long, hash_small, &mut sink), (5, 18, 18) => finder.find_matches::<5, 18, 18>(hash_long, hash_small, &mut sink), - (4, 16, 16) => finder.find_matches::<4, 16, 16>(hash_long, hash_small, &mut sink), + (5, 17, 16) => finder.find_matches::<5, 17, 16>(hash_long, hash_small, &mut sink), (5, 16, 15) => finder.find_matches::<5, 16, 15>(hash_long, hash_small, &mut sink), + (5, 14, 14) => finder.find_matches::<5, 14, 14>(hash_long, hash_small, &mut sink), + (4, 17, 17) => finder.find_matches::<4, 17, 17>(hash_long, hash_small, &mut sink), + (4, 17, 16) => finder.find_matches::<4, 17, 16>(hash_long, hash_small, &mut sink), + (4, 16, 16) => finder.find_matches::<4, 16, 16>(hash_long, hash_small, &mut sink), + (4, 16, 15) => finder.find_matches::<4, 16, 15>(hash_long, hash_small, &mut sink), (4, 15, 14) => finder.find_matches::<4, 15, 14>(hash_long, hash_small, &mut sink), - (5, ..) => finder.find_matches::<5, 0, 0>(hash_long, hash_small, &mut sink), - (6, ..) => finder.find_matches::<6, 0, 0>(hash_long, hash_small, &mut sink), - (7, ..) => finder.find_matches::<7, 0, 0>(hash_long, hash_small, &mut sink), - _ => finder.find_matches::<4, 0, 0>(hash_long, hash_small, &mut sink), + (4, 14, 13) => finder.find_matches::<4, 14, 13>(hash_long, hash_small, &mut sink), + (4, 13, 12) => finder.find_matches::<4, 13, 12>(hash_long, hash_small, &mut sink), + (4, 12, 11) => finder.find_matches::<4, 12, 11>(hash_long, hash_small, &mut sink), + (4, 11, 10) => finder.find_matches::<4, 11, 10>(hash_long, hash_small, &mut sink), + _ => finder.find_matches::<0, 0, 0>(hash_long, hash_small, &mut sink), } let last_literals = sink.last_literals()?; restore_rep( @@ -342,7 +343,7 @@ pub(crate) fn compress_block_double_fast( impl Finder { /// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` #[inline(never)] - fn find_matches_dict( + fn find_matches_dict( &mut self, hash_long: &mut [u32], hash_small: &mut [u32], @@ -352,8 +353,8 @@ impl Finder { ) { let Self { base_index, - h_bits_l, - h_bits_s, + long, + small, first: mut ip, mut offset_1, mut offset_2, @@ -368,8 +369,8 @@ impl Finder { let bytes0 = le64_at::<0, _>(here); let curr = base_index.wrapping_add(ip); let match_long = - hashed_swap(hash_long, hash8(bytes0, h_bits_l, 0), curr).wrapping_sub(base_index); - let match_index_s = hashed_swap(hash_small, hash_of::(bytes0, h_bits_s), curr); + hashed_swap(hash_long, long.hash(bytes0), curr).wrapping_sub(base_index); + let match_index_s = hashed_swap(hash_small, small.hash(bytes0), curr); if window.read32_before(ip + 1, offset_1) == Some((bytes0 >> 8) as u32) { ip += 1; @@ -383,7 +384,7 @@ impl Finder { if near.has::<{ u64::MAX }>(match_long, bytes0) { break 'found (window.place_of(match_long), 8); } - if let Some(place) = dict_tables.long.get::<8>(bytes0) + if let Some(place) = dict_tables.long.get(bytes0) && window.older.read64(place) == Some(bytes0) { break 'found (place, 8); @@ -393,7 +394,7 @@ impl Finder { let short = if match_index_s > base_index { Some(window.place_of(match_index_s - base_index)) } else { - dict_tables.short.get::(bytes0) + dict_tables.short.get(bytes0) }; let Some(place_short) = short.filter(|&place| window.read32(place) == Some(bytes0 as u32)) @@ -404,13 +405,13 @@ impl Finder { // A short match: a long one at the next position is preferred. let bytes1 = le64_at::<1, _>(here); - let match_long1 = hashed_swap(hash_long, hash8(bytes1, h_bits_l, 0), curr + 1) + let match_long1 = hashed_swap(hash_long, long.hash(bytes1), curr + 1) .wrapping_sub(base_index); if near.has::<{ u64::MAX }>(match_long1, bytes1) { ip += 1; break 'found (window.place_of(match_long1), 8); } - if let Some(place) = dict_tables.long.get::<8>(bytes1) + if let Some(place) = dict_tables.long.get(bytes1) && window.older.read64(place) == Some(bytes1) { ip += 1; @@ -432,21 +433,17 @@ impl Finder { }; let index_to_insert = curr + 2; if let Some(to_insert) = near.read64(index_to_insert.wrapping_sub(base_index)) { - hashed_put(hash_long, hash8(to_insert, h_bits_l, 0), index_to_insert); - hashed_put( - hash_small, - hash_of::(to_insert, h_bits_s), - index_to_insert, - ); + hashed_put(hash_long, long.hash(to_insert), index_to_insert); + hashed_put(hash_small, small.hash(to_insert), index_to_insert); } hashed_put( hash_long, - hash8(le64_at::<0, _>(around), h_bits_l, 0), + long.hash(le64_at::<0, _>(around)), base_index.wrapping_add(ip - 2), ); hashed_put( hash_small, - hash_of::(le64_at::<1, _>(around), h_bits_s), + small.hash(le64_at::<1, _>(around)), base_index.wrapping_add(ip - 1), ); @@ -457,8 +454,8 @@ impl Finder { let end = ip + 4 + window.count(ip + 4, rep_place + 4); core::mem::swap(&mut offset_1, &mut offset_2); let index = base_index.wrapping_add(ip); - hashed_put(hash_small, hash_of::(bytes, h_bits_s), index); - hashed_put(hash_long, hash8(bytes, h_bits_l, 0), index); + hashed_put(hash_small, small.hash(bytes), index); + hashed_put(hash_long, long.hash(bytes), index); sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); ip = end; } @@ -490,11 +487,13 @@ pub(crate) fn compress_block_double_fast_dict_match_state( long: TaggedTable::new( &dms.ms.hash_table, dms.ms.tagged_hash_log(), + 8, block.dict_start_index, )?, short: TaggedTable::new( &dms.ms.chain_table, dms.ms.tagged_chain_log(), + ms.c_params.min_match, block.dict_start_index, )?, }; @@ -503,8 +502,8 @@ pub(crate) fn compress_block_double_fast_dict_match_state( let [offset_1, offset_2, _] = *rep; let mut finder = Finder { base_index: block.prefix_start_index, - h_bits_l, - h_bits_s, + long: Hasher::new(h_bits_l, 8), + small: Hasher::new(h_bits_s, ms.c_params.min_match), first: block.first(), offset_1, offset_2, @@ -512,12 +511,7 @@ pub(crate) fn compress_block_double_fast_dict_match_state( let sink = &mut MatchSink::with_near(seq_store, block.window.near, block.istart); let long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let small = ms.chain_table.sub_mut(..1 << h_bits_s)?; - match ms.c_params.min_match { - 5 => finder.find_matches_dict::<5>(long, small, sink, block.window, tables), - 6 => finder.find_matches_dict::<6>(long, small, sink, block.window, tables), - 7 => finder.find_matches_dict::<7>(long, small, sink, block.window, tables), - _ => finder.find_matches_dict::<4>(long, small, sink, block.window, tables), - } + finder.find_matches_dict(long, small, sink, block.window, tables); let last_literals = sink.last_literals()?; let [rep0, rep1, _] = rep; *rep0 = finder.offset_1; @@ -526,7 +520,7 @@ pub(crate) fn compress_block_double_fast_dict_match_state( } impl Finder { - /// `ZSTD_compressBlock_doubleFast_extDict_generic`. `LONG` and `SMALL` are as for [`Self::find_matches`]. + /// `ZSTD_compressBlock_doubleFast_extDict_generic`. `MLS`, `LONG` and `SMALL` are as for [`Self::find_matches`]. #[inline(never)] fn find_matches_ext_dict( &mut self, @@ -537,8 +531,8 @@ impl Finder { ) { let Self { base_index, - h_bits_l, - h_bits_s, + long, + small, first: mut ip, mut offset_1, mut offset_2, @@ -547,8 +541,12 @@ impl Finder { let older_len = window.older.len(); // The index of the first place. let origin_index = base_index.wrapping_sub(older_len); - let h_bits_l = if LONG > 0 { LONG } else { h_bits_l }; - let h_bits_s = if SMALL > 0 { SMALL } else { h_bits_s }; + let long = if LONG > 0 { Hasher::new(LONG, 8) } else { long }; + let small = if SMALL > 0 { + Hasher::new(SMALL, MLS) + } else { + small + }; let Some(hash_long) = table_of_size::(hash_long) else { return; }; @@ -561,9 +559,9 @@ impl Finder { let bytes0 = le64_at::<0, _>(here); let curr = base_index.wrapping_add(ip); let place_long = - hashed_swap(hash_long, hash8(bytes0, h_bits_l, 0), curr).wrapping_sub(origin_index); - let place_short = hashed_swap(hash_small, hash_of::(bytes0, h_bits_s), curr) - .wrapping_sub(origin_index); + hashed_swap(hash_long, long.hash(bytes0), curr).wrapping_sub(origin_index); + let place_short = + hashed_swap(hash_small, small.hash(bytes0), curr).wrapping_sub(origin_index); if window.read32_before(ip + 1, offset_1) == Some((bytes0 >> 8) as u32) && offset_1 <= window.place_of(ip + 1) @@ -571,7 +569,7 @@ impl Finder { ip += 1; let rep_place = window.place_of(ip) - offset_1; let end = ip + 4 + window.count(ip + 4, rep_place + 4); - look_ahead::(near, end, hash_long, h_bits_l, hash_small, h_bits_s); + look_ahead(near, end, hash_long, long, hash_small, small); sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); ip = end; } else { @@ -582,7 +580,7 @@ impl Finder { } else if window.has::<0xFFFF_FFFF>(place_short, bytes0) && place_short > 0 { // A short match: a long one at the next position is preferred. let bytes1 = le64_at::<1, _>(here); - let place3 = hashed_swap(hash_long, hash8(bytes1, h_bits_l, 0), curr + 1) + let place3 = hashed_swap(hash_long, long.hash(bytes1), curr + 1) .wrapping_sub(origin_index); if window.has::<{ u64::MAX }>(place3, bytes1) && place3 > 0 { ip += 1; @@ -595,7 +593,7 @@ impl Finder { continue; }; let end = ip + known + window.count(ip + known, place + known); - look_ahead::(near, end, hash_long, h_bits_l, hash_small, h_bits_s); + look_ahead(near, end, hash_long, long, hash_small, small); let back = window.count_backwards(ip, sink.anchor, place); offset_2 = offset_1; offset_1 = window.place_of(ip) - place; @@ -609,21 +607,17 @@ impl Finder { }; let index_to_insert = curr + 2; if let Some(to_insert) = near.read64(index_to_insert.wrapping_sub(base_index)) { - hashed_put(hash_long, hash8(to_insert, h_bits_l, 0), index_to_insert); - hashed_put( - hash_small, - hash_of::(to_insert, h_bits_s), - index_to_insert, - ); + hashed_put(hash_long, long.hash(to_insert), index_to_insert); + hashed_put(hash_small, small.hash(to_insert), index_to_insert); } hashed_put( hash_long, - hash8(le64_at::<0, _>(around), h_bits_l, 0), + long.hash(le64_at::<0, _>(around)), base_index.wrapping_add(ip - 2), ); hashed_put( hash_small, - hash_of::(le64_at::<1, _>(around), h_bits_s), + small.hash(le64_at::<1, _>(around)), base_index.wrapping_add(ip - 1), ); @@ -637,8 +631,8 @@ impl Finder { let end = ip + 4 + window.count(ip + 4, rep_place + 4); core::mem::swap(&mut offset_1, &mut offset_2); let index = base_index.wrapping_add(ip); - hashed_put(hash_small, hash_of::(bytes, h_bits_s), index); - hashed_put(hash_long, hash8(bytes, h_bits_l, 0), index); + hashed_put(hash_small, small.hash(bytes), index); + hashed_put(hash_long, long.hash(bytes), index); sink.store_repeat(end); ip = end; } @@ -665,8 +659,8 @@ pub(crate) fn compress_block_double_fast_ext_dict( let [offset_1, offset_2, _] = *rep; let mut finder = Finder { base_index: block.prefix_start_index, - h_bits_l, - h_bits_s, + long: Hasher::new(h_bits_l, 8), + small: Hasher::new(h_bits_s, ms.c_params.min_match), first: block.istart, offset_1, offset_2, @@ -676,12 +670,9 @@ pub(crate) fn compress_block_double_fast_ext_dict( let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; let (long, small, sink) = (hash_long, hash_small, &mut sink); match (ms.c_params.min_match, h_bits_l, h_bits_s) { - (5, 17, 16) => finder.find_matches_ext_dict::<5, 17, 16>(long, small, sink, block.window), (5, 18, 18) => finder.find_matches_ext_dict::<5, 18, 18>(long, small, sink, block.window), - (5, ..) => finder.find_matches_ext_dict::<5, 0, 0>(long, small, sink, block.window), - (6, ..) => finder.find_matches_ext_dict::<6, 0, 0>(long, small, sink, block.window), - (7, ..) => finder.find_matches_ext_dict::<7, 0, 0>(long, small, sink, block.window), - _ => finder.find_matches_ext_dict::<4, 0, 0>(long, small, sink, block.window), + (5, 17, 16) => finder.find_matches_ext_dict::<5, 17, 16>(long, small, sink, block.window), + _ => finder.find_matches_ext_dict::<0, 0, 0>(long, small, sink, block.window), } let last_literals = sink.last_literals()?; let [rep0, rep1, _] = rep; diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index ed767e21fa50..d59f7f8a4312 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -1,10 +1,11 @@ //! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. use super::internal::{ - BlockSrc, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, K_SEARCH_STRENGTH, - MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, - ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, hash_of, hash_ptr, hashed_get, hashed_put, - hashed_swap, le32_at, le64_at, offset_to_offbase, table_of_size, write_tagged_index, + BlockSrc, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, Hasher, + K_SEARCH_STRENGTH, MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, + TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, hash_ptr, hashed_get, + hashed_put, hashed_swap, le32_at, le64_at, offset_to_offbase, table_of_size, + write_tagged_index, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -430,7 +431,7 @@ struct Finder { /// The index of the first position. base_index: u32, step_size: u32, - hlog: u32, + hasher: Hasher, /// Where to start. first: u32, rep_offset1: u32, @@ -444,7 +445,7 @@ impl Finder { /// at `ip2` for a repeat. The C code computes hashes and reads the table one position ahead. Here that is left to /// the processor: a turn only takes the position and the 8 bytes there, `bytes0`, from the one before. /// - /// `HLOG` is `hlog`, or 0 for any: see [`table_of_size`]. + /// `MLS` and `HLOG` are what `hasher` is made with, or both 0 for any: see [`table_of_size`]. #[inline(never)] fn find_matches( &mut self, @@ -455,13 +456,17 @@ impl Finder { let Self { base_index, step_size, - hlog, + hasher, first: mut ip0, mut rep_offset1, mut rep_offset2, } = *self; let near = sink.near; - let hlog = if HLOG > 0 { HLOG } else { hlog }; + let hasher = if HLOG > 0 { + Hasher::new(HLOG, MLS) + } else { + hasher + }; let Some(hash_table) = table_of_size::(hash_table) else { return; }; @@ -481,24 +486,24 @@ impl Finder { let Some(ahead) = near.window::<{ HASH_READ_SIZE + 2 }>(ip2) else { break 'block; }; - let idx0 = hashed_swap(hash_table, hash_of::(bytes0, hlog), current0); + let idx0 = hashed_swap(hash_table, hasher.hash(bytes0), current0); if sink.read32_before(ip2, rep_offset1) == Some(le32_at::<0, _>(ahead)) && rep_offset1 > 0 { - hashed_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); + hashed_put(hash_table, hasher.hash(bytes0 >> 8), current0 + 1); let back = u32::from(sink.same_byte_before(ip2, rep_offset1)); let end = ip2 + 4 + sink.count_before(ip2 + 4, rep_offset1); break (ip2 - back, end, REPCODE1_TO_OFFBASE); } let mut match0 = idx0.wrapping_sub(base_index); if near.has::<0xFFFF_FFFF>(match0, bytes0) { - hashed_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); + hashed_put(hash_table, hasher.hash(bytes0 >> 8), current0 + 1); } else { // As far as they are hashed or compared. let bytes1 = bytes0 >> 8; current0 += 1; - let idx1 = hashed_swap(hash_table, hash_of::(bytes1, hlog), current0); + let idx1 = hashed_swap(hash_table, hasher.hash(bytes1), current0); match0 = idx1.wrapping_sub(base_index); if !near.has::<0xFFFF_FFFF>(match0, bytes1) { current0 = base_index.wrapping_add(ip2); @@ -515,7 +520,7 @@ impl Finder { if step <= 4 { hashed_put( hash_table, - hash_of::(le64_at::<0, _>(ahead), hlog), + hasher.hash(le64_at::<0, _>(ahead)), base_index.wrapping_add(ip2), ); } @@ -537,12 +542,12 @@ impl Finder { }; bytes0 = bytes; if let Some(to_insert) = near.read64((current0 + 2).wrapping_sub(base_index)) { - hashed_put(hash_table, hash_of::(to_insert, hlog), current0 + 2); + hashed_put(hash_table, hasher.hash(to_insert), current0 + 2); } if let Some(before) = near.read64(ip0 - 2) { hashed_put( hash_table, - hash_of::(before, hlog), + hasher.hash(before), base_index.wrapping_add(ip0 - 2), ); } @@ -552,7 +557,7 @@ impl Finder { core::mem::swap(&mut rep_offset1, &mut rep_offset2); hashed_put( hash_table, - hash_of::(bytes0, hlog), + hasher.hash(bytes0), base_index.wrapping_add(ip0), ); sink.store::<32>(ip0, end, REPCODE1_TO_OFFBASE); @@ -582,7 +587,7 @@ impl Finder { let Self { base_index, step_size, - hlog, + hasher, first: mut ip0, mut rep_offset1, mut rep_offset2, @@ -591,7 +596,11 @@ impl Finder { let older_len = window.older.len(); // The index of the first place. let origin_index = base_index.wrapping_sub(older_len); - let hlog = if HLOG > 0 { HLOG } else { hlog }; + let hasher = if HLOG > 0 { + Hasher::new(HLOG, MLS) + } else { + hasher + }; let Some(hash_table) = table_of_size::(hash_table) else { return; }; @@ -610,7 +619,7 @@ impl Finder { let Some(ahead) = near.window::<{ HASH_READ_SIZE + 2 }>(ip2) else { break 'block; }; - let idx0 = hashed_swap(hash_table, hash_of::(bytes0, hlog), current0); + let idx0 = hashed_swap(hash_table, hasher.hash(bytes0), current0); // The 4 bytes have to be in one segment, with a byte before them. let rep_place = window.place_of(ip2).wrapping_sub(rep_offset1); @@ -618,7 +627,7 @@ impl Finder { && rep_place != older_len && rep_offset1 > 0 { - hashed_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); + hashed_put(hash_table, hasher.hash(bytes0 >> 8), current0 + 1); let back = u32::from(near.read8(ip2 - 1) == window.read8(rep_place.wrapping_sub(1))); break (ip2, rep_place, back, REPCODE1_TO_OFFBASE, u32::MAX); @@ -626,11 +635,11 @@ impl Finder { let mut place = idx0.wrapping_sub(origin_index); let mut next = u32::MAX; if window.has::<0xFFFF_FFFF>(place, bytes0) { - hashed_put(hash_table, hash_of::(bytes0 >> 8, hlog), current0 + 1); + hashed_put(hash_table, hasher.hash(bytes0 >> 8), current0 + 1); } else { let bytes1 = bytes0 >> 8; current0 += 1; - let idx1 = hashed_swap(hash_table, hash_of::(bytes1, hlog), current0); + let idx1 = hashed_swap(hash_table, hasher.hash(bytes1), current0); place = idx1.wrapping_sub(origin_index); if !window.has::<0xFFFF_FFFF>(place, bytes1) { current0 = base_index.wrapping_add(ip2); @@ -660,7 +669,7 @@ impl Finder { { hashed_put( hash_table, - hash_of::(bytes, hlog), + hasher.hash(bytes), base_index.wrapping_add(next), ); } @@ -671,12 +680,12 @@ impl Finder { }; bytes0 = bytes; if let Some(to_insert) = near.read64((current0 + 2).wrapping_sub(base_index)) { - hashed_put(hash_table, hash_of::(to_insert, hlog), current0 + 2); + hashed_put(hash_table, hasher.hash(to_insert), current0 + 2); } if let Some(before) = near.read64(ip0 - 2) { hashed_put( hash_table, - hash_of::(before, hlog), + hasher.hash(before), base_index.wrapping_add(ip0 - 2), ); } @@ -687,7 +696,7 @@ impl Finder { core::mem::swap(&mut rep_offset1, &mut rep_offset2); hashed_put( hash_table, - hash_of::(bytes0, hlog), + hasher.hash(bytes0), base_index.wrapping_add(ip0), ); sink.store::<32>(ip0, end, REPCODE1_TO_OFFBASE); @@ -717,7 +726,7 @@ pub(crate) fn compress_block_fast( base_index: block.prefix_lowest_index, // The least step is 2, as two positions are looked at per turn. step_size: least_step(&ms.c_params) + 1, - hlog, + hasher: Hasher::new(hlog, ms.c_params.min_match), first: block.first, rep_offset1: block.rep.offset_1, rep_offset2: block.rep.offset_2, @@ -728,13 +737,19 @@ pub(crate) fn compress_block_fast( match (ms.c_params.min_match, hlog) { (7, 14) => finder.find_matches::<7, 14>(hash_table, &mut sink), (6, 16) => finder.find_matches::<6, 16>(hash_table, &mut sink), - (6, 13) => finder.find_matches::<6, 13>(hash_table, &mut sink), (6, 14) => finder.find_matches::<6, 14>(hash_table, &mut sink), + (6, 13) => finder.find_matches::<6, 13>(hash_table, &mut sink), (5, 15) => finder.find_matches::<5, 15>(hash_table, &mut sink), - (5, _) => finder.find_matches::<5, 0>(hash_table, &mut sink), - (6, _) => finder.find_matches::<6, 0>(hash_table, &mut sink), - (7, _) => finder.find_matches::<7, 0>(hash_table, &mut sink), - _ => finder.find_matches::<4, 0>(hash_table, &mut sink), + (5, 14) => finder.find_matches::<5, 14>(hash_table, &mut sink), + (5, 13) => finder.find_matches::<5, 13>(hash_table, &mut sink), + (5, 12) => finder.find_matches::<5, 12>(hash_table, &mut sink), + (5, 11) => finder.find_matches::<5, 11>(hash_table, &mut sink), + (4, 15) => finder.find_matches::<4, 15>(hash_table, &mut sink), + (4, 14) => finder.find_matches::<4, 14>(hash_table, &mut sink), + (4, 13) => finder.find_matches::<4, 13>(hash_table, &mut sink), + (4, 12) => finder.find_matches::<4, 12>(hash_table, &mut sink), + (4, 11) => finder.find_matches::<4, 11>(hash_table, &mut sink), + _ => finder.find_matches::<0, 0>(hash_table, &mut sink), } let last_literals = sink.last_literals()?; restore_rep( @@ -784,26 +799,27 @@ impl<'a> DictBlock<'a> { #[derive(Clone, Copy)] pub(super) struct TaggedTable<'a> { table: &'a [u32], - /// How many bits to hash to, tag included. - h_bits: u32, + /// To a hash and a tag. + hasher: Hasher, /// The index that the first place has in the table. start_index: u32, } impl<'a> TaggedTable<'a> { - pub(super) fn new(table: &'a [u32], h_bits: u32, start_index: u32) -> Result { + /// `mls` is how many bytes are hashed. + pub(super) fn new(table: &'a [u32], h_bits: u32, mls: u32, start_index: u32) -> Result { Ok(Self { table: table.sub(..1 << (h_bits - ZSTD_SHORT_CACHE_TAG_BITS))?, - h_bits, + hasher: Hasher::new(h_bits, mls), start_index, }) } - /// The place that the table has for the `MLS` first bytes of `bytes`. `None` if the tag tells that it is for + /// The place that the table has for the first bytes of `bytes`. `None` if the tag tells that it is for /// something else, or if it is the first place, which stands for none. #[inline(always)] - pub(super) fn get(self, bytes: u64) -> Option { - let hash_and_tag = hash_of::(bytes, self.h_bits); + pub(super) fn get(self, bytes: u64) -> Option { + let hash_and_tag = self.hasher.hash(bytes); let index_and_tag = hashed_get(self.table, hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS); let index = index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; (compare_packed_tags(index_and_tag as usize, hash_and_tag) && index > self.start_index) @@ -814,7 +830,7 @@ impl<'a> TaggedTable<'a> { impl Finder { /// `ZSTD_compressBlock_fast_dictMatchState_generic` #[inline(never)] - fn find_matches_dict( + fn find_matches_dict( &mut self, hash_table: &mut [u32], sink: &mut MatchSink<'_, '_>, @@ -825,7 +841,7 @@ impl Finder { let Self { base_index, step_size, - hlog, + hasher, first: mut ip0, mut rep_offset1, mut rep_offset2, @@ -850,7 +866,7 @@ impl Finder { break 'block; }; curr = base_index.wrapping_add(ip0); - let match_index = hashed_swap(hash_table, hash_of::(bytes0, hlog), curr); + let match_index = hashed_swap(hash_table, hasher.hash(bytes0), curr); if window.read32_before(ip0 + 1, rep_offset1) == Some((bytes0 >> 8) as u32) { ip0 += 1; @@ -862,7 +878,7 @@ impl Finder { } // A match in the dictionary is only taken if there is no candidate in the prefix. - if let Some(place) = dict_table.get::(bytes0) + if let Some(place) = dict_table.get(bytes0) && window.older.read32(place) == Some(bytes0 as u32) && match_index <= base_index { @@ -898,12 +914,12 @@ impl Finder { }; bytes0 = bytes; if let Some(to_insert) = near.read64((curr + 2).wrapping_sub(base_index)) { - hashed_put(hash_table, hash_of::(to_insert, hlog), curr + 2); + hashed_put(hash_table, hasher.hash(to_insert), curr + 2); } if let Some(before) = near.read64(ip0 - 2) { hashed_put( hash_table, - hash_of::(before, hlog), + hasher.hash(before), base_index.wrapping_add(ip0 - 2), ); } @@ -914,7 +930,7 @@ impl Finder { core::mem::swap(&mut rep_offset1, &mut rep_offset2); hashed_put( hash_table, - hash_of::(bytes0, hlog), + hasher.hash(bytes0), base_index.wrapping_add(ip0), ); sink.store::<32>(ip0, end, REPCODE1_TO_OFFBASE); @@ -944,6 +960,7 @@ pub(crate) fn compress_block_fast_dict_match_state( let dict_table = TaggedTable::new( &dms.ms.hash_table, dms.ms.tagged_hash_log(), + ms.c_params.min_match, block.dict_start_index, )?; let hlog = ms.c_params.hash_log; @@ -951,19 +968,14 @@ pub(crate) fn compress_block_fast_dict_match_state( let mut finder = Finder { base_index: block.prefix_start_index, step_size: least_step(&ms.c_params), - hlog, + hasher: Hasher::new(hlog, ms.c_params.min_match), first: block.first(), rep_offset1, rep_offset2, }; let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; - match ms.c_params.min_match { - 5 => finder.find_matches_dict::<5>(hash_table, &mut sink, block.window, dict_table), - 6 => finder.find_matches_dict::<6>(hash_table, &mut sink, block.window, dict_table), - 7 => finder.find_matches_dict::<7>(hash_table, &mut sink, block.window, dict_table), - _ => finder.find_matches_dict::<4>(hash_table, &mut sink, block.window, dict_table), - } + finder.find_matches_dict(hash_table, &mut sink, block.window, dict_table); let last_literals = sink.last_literals()?; let [rep0, rep1, _] = rep; *rep0 = finder.rep_offset1; @@ -986,7 +998,7 @@ pub(crate) fn compress_block_fast_ext_dict( let mut finder = Finder { base_index: block.prefix_start_index, step_size: least_step(&ms.c_params) + 1, - hlog, + hasher: Hasher::new(hlog, ms.c_params.min_match), first: block.istart, rep_offset1: block.rep.offset_1, rep_offset2: block.rep.offset_2, @@ -997,10 +1009,7 @@ pub(crate) fn compress_block_fast_ext_dict( (7, 14) => finder.find_matches_ext_dict::<7, 14>(hash_table, &mut sink, block.window), (6, 16) => finder.find_matches_ext_dict::<6, 16>(hash_table, &mut sink, block.window), (6, 13) => finder.find_matches_ext_dict::<6, 13>(hash_table, &mut sink, block.window), - (5, _) => finder.find_matches_ext_dict::<5, 0>(hash_table, &mut sink, block.window), - (6, _) => finder.find_matches_ext_dict::<6, 0>(hash_table, &mut sink, block.window), - (7, _) => finder.find_matches_ext_dict::<7, 0>(hash_table, &mut sink, block.window), - _ => finder.find_matches_ext_dict::<4, 0>(hash_table, &mut sink, block.window), + _ => finder.find_matches_ext_dict::<0, 0>(hash_table, &mut sink, block.window), } let last_literals = sink.last_literals()?; restore_rep( diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 4a7cff824102..57da01a0c3e2 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1580,7 +1580,7 @@ pub(crate) const fn hash3_of(bytes: u32, h: u32) -> usize { hash3(bytes, h, 0) as usize } -/// [`hash_of`], when `mls` is not a constant. +/// [`hash_ptr`], out of the 8 bytes at the position. #[derive(Clone, Copy)] pub(crate) struct Hasher { /// Takes the bytes that do not count out, on the left. @@ -1615,18 +1615,6 @@ impl Hasher { } } -/// [`hash_ptr`] of the position that the 8 bytes `bytes` were read at. -#[inline(always)] -pub(crate) const fn hash_of(bytes: u64, h_bits: u32) -> usize { - match MLS { - 5 => hash5(bytes, h_bits, 0), - 6 => hash6(bytes, h_bits, 0), - 7 => hash7(bytes, h_bits, 0), - 8 => hash8(bytes, h_bits, 0), - _ => hash4(bytes as u32, h_bits, 0) as usize, - } -} - /// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, /// else 8. #[inline(always)] From 2fa2eb454df3d3d5a74df772f47b63da9498d258 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:46:05 +0000 Subject: [PATCH 099/199] zstd: dfast only asks ahead for what is in large tables --- src/zstd/compress/double_fast.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 0e85fe129685..98dd326a971c 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -196,7 +196,9 @@ impl Finder { { ip += 1; let end = ip + 4 + sink.count_before(ip + 4, offset_1); - look_ahead(near, end, hash_long, long, hash_small, small); + if LONG == 0 || LONG >= 17 { + look_ahead(near, end, hash_long, long, hash_small, small); + } sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); break end; } @@ -234,7 +236,9 @@ impl Finder { continue; }; - look_ahead(near, end, hash_long, long, hash_small, small); + if LONG == 0 || LONG >= 17 { + look_ahead(near, end, hash_long, long, hash_small, small); + } offset_2 = offset_1; offset_1 = ip - match0; // With a larger step, ip1 may be past the end of the match. From 2f442d15fc726171cedcf1367d9073c6c58f795d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:39:41 +0000 Subject: [PATCH 100/199] zstd: test for the bugs that zstd has had --- test/js/bun/util/zstd.test.ts | 512 ++++++++++++++++++++++++++++++++++ 1 file changed, 512 insertions(+) diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 3b81bf54d6ee..7cdaafc4fe55 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1483,3 +1483,515 @@ describe("bugs of zstd 1.5.7", () => { expect(await promise).toMatchObject({ code: "ZSTD_error_srcSize_wrong" }); }); }); + +// Bugs that zstd has had, from its history up to October 2026. zstd 1.5.7 has none of them, and what is expected is what +// it gives: they are here because the port does not follow the C code line by line. So Bun 1.4.4 passes them too, but for +// bugs of its own: streams with workers lose their output, and a dictionary that a decoder cannot load goes unnoticed. +describe("bugs that zstd has had", () => { + const c = zlib.constants; + const hex = (digits: string) => Buffer.from(digits.replaceAll(" ", ""), "hex"); + const base64 = (text: string) => Buffer.from(text, "base64"); + const repeated = (digits: string, count: number) => Buffer.alloc((digits.length / 2) * count, hex(digits)); + const skippable = (size: number) => { + const frame = Buffer.alloc(8 + size, 7); + frame.writeUInt32LE(0x184d2a50, 0); + frame.writeUInt32LE(size, 4); + return frame; + }; + /** xorshift32 */ + function noise(length: number, seed: number, mask = 255) { + const bytes = Buffer.alloc(length); + for (let i = 0, x = seed >>> 0; i < length; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + bytes[i] = x & mask; + } + return bytes; + } + function through(stream: zlib.ZstdCompress | zlib.ZstdDecompress, chunks: Iterable) { + const { promise, resolve } = Promise.withResolvers(); + const output: Buffer[] = []; + stream.on("data", chunk => output.push(chunk)); + stream.on("error", (error: any) => resolve(error.code)); + stream.on("end", () => resolve(Buffer.concat(output))); + for (const chunk of chunks) stream.write(chunk); + stream.end(); + return promise; + } + function* pieces(bytes: Buffer, size: number) { + for (let i = 0; i < bytes.length; i += size) yield bytes.subarray(i, i + size); + } + const attempt = (decompress: () => Uint8Array) => { + try { + return Buffer.from(decompress()); + } catch (error: any) { + return error.code as string; + } + }; + /** By `Bun.zstdDecompressSync`, by `zlib.zstdDecompressSync`, and by a stream that gets little at a time. */ + const decoded = async (frame: Buffer, options?: zlib.ZstdOptions) => [ + attempt(() => Bun.zstdDecompressSync(frame)), + attempt(() => zlib.zstdDecompressSync(frame, options)), + await through(zlib.createZstdDecompress(options), pieces(frame, frame.length > 3000 ? 1000 : 1)), + ]; + const hello = hex("28b52ffd200529000068656c6c6f"); + const corrupt = "ZSTD_error_corruption_detected"; + + it.each<[string, Buffer, Buffer | string]>([ + ["an empty compressed block", hex("28b52ffd00001500000000"), ""], + ["an empty raw block", hex("28b52ffd0000 010000"), ""], + ["an empty run", hex("28b52ffd0000 030000 09"), ""], + ["empty blocks between others", hex("28b52ffd0000 180000616263 000000 0200007a 190000646566"), "abcdef"], + [ + "a run for a first block", + hex("28b52ffda400001000" + "020010 00".repeat(7) + "030010 00 f13e16e1"), + Buffer.alloc(1 << 20), + ], + ["no sequences, said in 2 bytes", hex("28b52ffd00008500006848656c6c6f20576f726c64210a8000"), "Hello World!\n"], + ["no sequences", hex("28b52ffd0000 2d0000 1861626300"), "abc"], + [ + "a compressed block of 128 KB", + Buffer.concat([hex("28b52ffd0038050010ccff1f"), Buffer.alloc(131068), hex("00")]), + Buffer.alloc(131068), + ], + [ + "copies that go too far at the end of a stream's buffer", + hex("28b52ffd04004c00001061610100002a800544000008620100002a20045d00000003400000646027b0e00c6762cee0"), + Buffer.concat([Buffer.alloc(517, "a"), Buffer.alloc(543, "b"), Buffer.from("aaa")]), + ], + [ + "sequence sections of 3 and 4 bytes", + base64("KLUv/SQ8NQEA8IUIwsRwz9fAln5Ma6mLvMW22X9M8QWmVO+saZSJHANECgcAtASAQAqk"), + hex("8508c2c470cfd7c0967e".repeat(2) + "4c6ba98bbcc5b6d97f4c".repeat(2) + "f105a654efac6994891c".repeat(2)), + ], + [ + "skippable frames before, between and after frames", + hex("502a4d1803000000010203 28b52ffd0000190000616263 5f2a4d1800000000 28b52ffd0000190000646566"), + "abcdef", + ], + ["a skippable frame longer than any buffer", Buffer.concat([skippable(200_000), hello]), "hello"], + ["a window of 128 MB", hex("28b52ffd0088190000616263"), "abc"], + ["a match for the byte before", hex("28b52ffd8000040000004d00001c0000610150010040"), "aaaa"], + ["a match that overlaps itself", hex("28b52ffd80000c00000045000010616201006d0e0b"), "abababababab"], + [ + "a raw block of the size of the window", + Buffer.concat([hex("28b52ffd0000 012000"), Buffer.alloc(1024, "A")]), + Buffer.alloc(1024, "A"), + ], + ])("%s is decoded", async (_, frame, content) => { + const expected = Buffer.from(content); + expect(await decoded(frame)).toEqual([expected, expected, expected]); + }); + + it.each<[string, Buffer, string]>([ + ["an offset of 0", hex("28b52ffd0000450000080002002f430bae"), corrupt], + ["a repeat offset that comes to 0", hex("28b52ffd000040000061626364656667683d000000015400010003"), corrupt], + ["a Huffman state that is cut short", hex("28b52ffd000055000072800104207e1f02aa00"), corrupt], + [ + "bytes after no sequences, said in 2 bytes", + hex("28b52ffd00009500006848656c6c6f20576f726c64210a80000000"), + corrupt, + ], + ["a byte after no sequences", hex("28b52ffd0000 350000 186162630000"), corrupt], + ["less content than the header says", hex("28b52ffd200629000068656c6c6f"), corrupt], + ["more content than the header says", hex("28b52ffd200429000068656c6c6f"), corrupt], + [ + "a raw block larger than the window", + Buffer.concat([hex("28b52ffd0000 092000"), Buffer.alloc(1025, "A")]), + corrupt, + ], + [ + "the Huffman table of the block before, in the first", + hex("28b52ffd0000350000434000000000"), + "ZSTD_error_dictionary_corrupted", + ], + ["a window of 256 MB", hex("28b52ffd0090190000616263"), "ZSTD_error_frameParameter_windowTooLarge"], + ["the reserved bit", hex("28b52ffd0800190000616263"), "ZSTD_error_frameParameter_unsupported"], + ["the reserved bit of a frame in one segment", hex("28b52ffd2800010000"), "ZSTD_error_frameParameter_unsupported"], + ["the largest window there is a code for", hex("28b52ffd00ff010000"), "ZSTD_error_frameParameter_windowTooLarge"], + ["the sequence tables of the block before, in the first", hex("28b52ffd8000040000002d0000086101fc01"), corrupt], + ["a literal length code of 36", hex("28b52ffd800004000000350000086101402401"), corrupt], + ["an offset code of 32", hex("28b52ffd800004000000350000086101102001"), corrupt], + ["a match length code of 53", hex("28b52ffd800004000000350000086101043501"), corrupt], + [ + "4 streams for 5 literals", + hex("28b52ffd8000050000004d0000568001811100000000"), + "ZSTD_error_literals_headerWrong", + ], + [ + "Huffman weights that add up to more than 2^12", + hex("28b52ffd80001400000085000042010383cccc01010101010101010100"), + corrupt, + ], + ["the first repeat offset less 1, in the first sequence", hex("28b52ffd800003000000350000000150000181"), corrupt], + ["an offset of 2 after 1 byte", hex("28b52ffd8000040000003d0000096101440100b9"), corrupt], + ["a run of literals without its byte", hex("28b52ffd8000280000001d00008d0200"), corrupt], + ["compressed literals without a table", hex("28b52ffd80002800000025000002000000"), corrupt], + [ + // CVE-2026-116591 of another decoder, where it got through on top of what the block before had left. + "Huffman weights with a table of 2^15 states", + Buffer.concat([ + hex("28b52ffd003874060042b8320d1a00000090affaff030080ff7f"), + repeated("a594524a29", 37), + hex("a5941200ad5603eeffdfb31a1c1a00e0ff1f0fb19c5acf4de41d7bab995e0151a473d8b1f3394b67f6ab1aab1aab1a"), + ...[repeated("8c31c61863", 1365), hex("8c31"), repeated("638c31c618", 1365), hex("632c")], + ...[repeated("31c618638c", 1365), hex("3146"), repeated("31c618638c", 1365), hex("310600")], + ]), + corrupt, + ], + ])("%s is an error", async (_, frame, code) => { + expect(await decoded(frame)).toEqual(["ERR_ZSTD", code, code]); + }); + + it("what follows the last frame is refused or left alone", async () => { + const abc = Buffer.from("abc"); + const frame = hex("28b52ffd0000190000616263"); + const after = (rest: string) => decoded(Buffer.concat([frame, hex(rest)])); + expect(await after("ff")).toEqual(["ERR_ZSTD", abc, "ZSTD_error_prefix_unknown"]); + expect(await after("502a4d18ffffffff")).toEqual(["ERR_ZSTD", abc, abc]); + // 8 more than that wraps around. + expect(await decoded(hex("502a4d18f8ffffff"))).toEqual(["ERR_ZSTD", Buffer.alloc(0), Buffer.alloc(0)]); + }); + + it("a first block cannot repeat the tables of the frame that was decoded before it", async () => { + const text = zlib.zstdCompressSync(noise(20_000, 18, 15)); + for (const [frame, code] of [ + [hex("28b52ffd800008000000350000834000010100"), "ZSTD_error_dictionary_corrupted"], + [hex("28b52ffd8000040000002d0000086101fc01"), corrupt], + ] as const) { + expect(Bun.zstdDecompressSync(text).length).toBe(20_000); + expect(zlib.zstdDecompressSync(text).length).toBe(20_000); + expect(await decoded(frame)).toEqual(["ERR_ZSTD", code, code]); + expect(await through(zlib.createZstdDecompress(), [text, frame])).toBe(code); + } + }); + + it("offsets up to the window are decoded after a stream's buffer has wrapped around", async () => { + for (const distance of [1000, 1017, 1023, 1024]) { + const input = Buffer.alloc(100_000, noise(distance, distance)); + for (const level of [5, 13, 19]) { + const params = { [c.ZSTD_c_windowLog]: 10, [c.ZSTD_c_compressionLevel]: level }; + const frame = zlib.zstdCompressSync(input, { params }); + expect(frame.length).toBeLessThan(5000); + expect(await through(zlib.createZstdDecompress(), pieces(frame, 1))).toEqual(input); + expect(await through(zlib.createZstdDecompress(), [frame])).toEqual(input); + } + } + }); + + it("a larger window is decoded if allowed", async () => { + const [, sync, streamed] = await decoded(hex("28b52ffd0090190000616263"), { + params: { [c.ZSTD_d_windowLogMax]: 28 }, + }); + expect([sync, streamed]).toEqual([Buffer.from("abc"), Buffer.from("abc")]); + }); + + it("each bit of the checksum counts", async () => { + const frame = zlib.zstdCompressSync(noise(3000, 1, 15), { params: { [c.ZSTD_c_checksumFlag]: 1 } }); + for (let bit = 0; bit < 32; bit++) { + const damaged = Buffer.from(frame); + damaged[frame.length - 4 + (bit >> 3)] ^= 1 << (bit & 7); + expect(attempt(() => zlib.zstdDecompressSync(damaged))).toBe("ZSTD_error_checksum_wrong"); + } + }); + + it("a raw block comes out as it comes in", async () => { + const content = noise(100_000, 2); + const frame = zlib.zstdCompressSync(content); + const stream = zlib.createZstdDecompress(); + const { promise, resolve } = Promise.withResolvers(); + stream.once("data", chunk => resolve(chunk.length)); + stream.write(frame.subarray(0, 5000), () => stream.flush()); + expect(await promise).toBeWithin(4000, 5000); + stream.destroy(); + }); + + const roundTrip = (input: Buffer, params: Record, dictionary?: Buffer) => { + const frame = zlib.zstdCompressSync(input, { params, dictionary }); + expect(zlib.zstdDecompressSync(frame, { dictionary }).equals(input)).toBe(true); + return frame; + }; + const levels = (from: number, to: number) => Array.from({ length: to - from + 1 }, (_, i) => from + i); + + // facebook/zstd#3517 + it("a block is split where 64 KB of literals come before a repeat offset", () => { + const input = noise(256 << 10, 2654435761, 127); + noise(32 << 10, 0xcafebabe, 3).copy(input); + const put = (at: number, values: number[]) => { + for (const skip of [0, 4]) values.forEach((value, i) => input.writeUInt32LE(value, (at + skip + i) * 4)); + }; + put(8 << 10, [0xffffffff, 0xeeeeeeee]); + put((24 << 10) + 2, [0xdddddddd, 0xcccccccc]); + put(32 << 10, [0xbbbbbbbb, 0xaaaaaaaa]); + expect(roundTrip(input, { [c.ZSTD_c_compressionLevel]: 19, [c.ZSTD_c_minMatch]: 7 }).length).toBe(210649); + }); + + // facebook/zstd#3212 + const ZSTD_c_targetCBlockSize = 130; + it("literals and matches of more than 64 KB go in blocks of a target size", () => { + const input = Buffer.alloc(270_000); + noise(70_000, 12345, 127).copy(input); + for (let i = 70_000; i < 200_000; i++) input[i] = input[i - 1000]; + noise(70_000, 777, 127).copy(input, 200_000); + for (const level of [1, 3, 7, 19]) { + roundTrip(input, { [c.ZSTD_c_compressionLevel]: level, [ZSTD_c_targetCBlockSize]: 1340 }); + } + }); + + it("runs are coded as runs, but for the first block", () => { + const twoRuns = Buffer.concat([Buffer.alloc((256 << 10) - 2, "B"), Buffer.alloc(100 << 10, "A")]); + expect(Buffer.from(Bun.zstdCompressSync(twoRuns, { level: 1 })).toString("hex")).toBe( + "28b52ffda0fe8f05005400001042420100fbff39c0025c0000184241410100faff391002f37f0c41", + ); + const runs = "02001000".repeat(6) + "03120a00"; + const zeros = (level: number) => + Buffer.from(Bun.zstdCompressSync(Buffer.alloc(1_000_000), { level })).toString("hex"); + expect(zeros(1)).toBe("28b52ffd804840420f005400001000000100fbff39c002" + runs); + expect(zeros(3)).toBe("28b52ffda040420f005400001000000100fbff39c002" + runs); + expect(zeros(19)).toBe("28b52ffda040420f004c000008000100fcff391002" + runs); + }); + + it("what does not compress is not split", () => { + const input = noise(256 << 10, 99); + for (let chunk = 0; chunk < 32; chunk++) { + for (let i = chunk * 8192; i < (chunk + 1) * 8192; i += 43) + input[i] = chunk % 2 ? input[i] & 0x80 : input[i] | 0x80; + } + for (const level of [1, 3, 5, 9, 13, 16, 19, 22]) { + expect(Bun.zstdCompressSync(input, { level }).length).toBe(input.length + 9 + 2 * 3); + } + }); + + it("literals that Huffman coding makes longer are left as they are", () => { + const input = base64( + "KLUv/QRQFQQAZoYeHZByBTMAgCylAQAAAAAQ3rSu67renkkEAACAAACAFgAXABYAPAbsctt3bakp+jCTL8TBUCAMBAHAB/qINH48GBJqFKKYQCuWaqejAYm/HkwYO8iHxTOx0WQulspEBGEVN1ttJot1X2V//vv1OxoLTidSkpxSsxzjcgEBAFsWJDvL4y4=", + ); + for (const level of levels(1, 22)) + expect(roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }).length).toBe(152); + expect(Bun.zstdDecompressSync(input).length).toBe(142); + }); + + it("there is no empty block after input of the pledged size", async () => { + const block = Buffer.alloc(128 << 10, "a"); + const compressed = async (options: zlib.ZstdOptions) => + ((await through(zlib.createZstdCompress(options), [block])) as Buffer).toString("hex"); + expect(await compressed({ pledgedSrcSize: block.length })).toBe("28b52ffda0000002005500001061610100fbff39c002"); + expect(await compressed({})).toBe("28b52ffd00585400001061610100fbff39c002010000"); + }); + + it("tiny inputs", () => { + const compressed = (input: string | Buffer, level: number) => + Buffer.from(Bun.zstdCompressSync(input, { level })).toString("hex"); + for (const level of [1, 3, 19]) { + expect(compressed(Buffer.alloc(100), level)).toBe("28b52ffd206445000010000001003f012c"); + expect(compressed("", level)).toBe("28b52ffd2000010000"); + expect(compressed("a", level)).toBe("28b52ffd200109000061"); + } + const abab = Buffer.alloc(20, "ab"); + expect([1, 3, 19].map(level => compressed(abab, level))).toEqual([ + "28b52ffd20144500001061620100790e0b", + "28b52ffd20144500001061620100790e0b", + "28b52ffd20144d0000186162610100cd6e08", + ]); + for (const length of levels(0, 40)) { + for (const input of [Buffer.alloc(length), Buffer.from(levels(1, length)), Buffer.alloc(length, "ab")]) { + for (const level of [-5, 1, 3, 6, 13, 19, 22]) roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }); + } + } + }); + + it("127, 128 and 129 sequences", () => { + const made = (total: number) => { + let x = 1; + const next = () => ((x = (Math.imul(x, 1103515245) + 12345) & 0x7fffffff), (x >> 16) & 255); + const bytes = Array.from({ length: 16 }, next); + while (bytes.length < total) { + const distance = 4 + (next() % 9); + for (let i = 0; i < 3; i++) bytes.push(bytes.at(-distance)!); + const byte = next(); + bytes.push(byte === bytes.at(-distance) ? (byte + 1) & 255 : byte); + } + const frame = roundTrip(Buffer.from(bytes), { [c.ZSTD_c_compressionLevel]: 19 }); + return `${frame.length} ${Bun.hash.xxHash64(frame).toString(16)}`; + }; + expect([532, 536, 540].map(made)).toEqual(["237 675545151ba98499", "239 18edd3092b7b0659", "241 bf78b419d4d9bdc1"]); + }); + + it("literals and matches of about 64 KB", () => { + for (const literals of [65_535, 65_536, 65_537]) { + for (const match of [65_538, 65_539]) { + const input = Buffer.concat([ + noise(literals + 10, literals), + noise(literals + 10, literals).subarray(0, match), + ]); + for (const level of [1, 3, 9, 19]) roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }); + } + } + }); + + it("level 0 is level 3", () => { + const input = noise(20_000, 3, 7); + expect(roundTrip(input, { [c.ZSTD_c_compressionLevel]: 0 })).toEqual( + roundTrip(input, { [c.ZSTD_c_compressionLevel]: 3 }), + ); + }); + + it("the size of the output buffer does not show", () => { + for (const input of [Buffer.alloc(0), Buffer.from("a"), noise(200, 4, 7), noise(5000, 5, 7), noise(5000, 6)]) { + for (const level of [1, 3, 9, 19]) { + const params = { [c.ZSTD_c_compressionLevel]: level }; + expect(zlib.zstdCompressSync(input, { params, chunkSize: 64 })).toEqual(roundTrip(input, params)); + } + } + }); + + it("a match for the first position is found alike in one call and in a stream", () => { + const input = Buffer.from([...levels(0, 19), ...levels(0, 9)]); + for (const level of levels(1, 19)) { + expect(Buffer.from(Bun.zstdCompressSync(input, { level }))).toEqual( + roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }), + ); + } + }); + + it("the first block that counts comes after blocks too small to", async () => { + const input = noise(5000, 7, 15); + const stream = zlib.createZstdCompress({ + pledgedSrcSize: input.length, + params: { [c.ZSTD_c_compressionLevel]: 22 }, + }); + const { promise, resolve } = Promise.withResolvers(); + const output: Buffer[] = []; + stream.on("data", chunk => output.push(chunk)).on("end", () => resolve(Buffer.concat(output))); + for (const [from, to] of [ + [0, 4], + [4, 9], + [9, 20], + ]) { + stream.write(input.subarray(from, to)); + stream.flush(); + } + stream.end(input.subarray(20)); + expect(zlib.zstdDecompressSync(await promise)).toEqual(input); + }); + + it("raw blocks between compressed ones", () => { + const input = Buffer.concat(levels(1, 3).flatMap(i => [noise(128 << 10, i, 7), noise(128 << 10, i + 10)])); + for (const level of [1, 3, 9, 19]) roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }); + }); + + describe("dictionaries", () => { + // `tests/dict-files/zero-weight-dict` + const zeroWeights = base64( + "N6Qw7P8g/wA2EBsBAS69U+gVvLDIxkg0+R20Td9JXrqkGW5ZfDAqCkIVpJJ/1SYRHsHu2M4pkpKSkpKSkjIF0wIgDAoHxYOiAYE+uwUEIMKpoI4m1OVZksMoCiFkjDGEECAAAEAAQGREAAAAABRwrVMUAAAATCGEjEHGDM3IDAAAAAEAAAAEAAAACAAAACAgjibU5VmSwygK", + ); + + it("whose Huffman table has symbols of weight 0", () => { + const skewed = noise(5000, 8).map((byte, i, all) => Math.min(byte, all[(i + 1) % all.length])); + for (const level of levels(1, 22)) { + for (const input of [zeroWeights, Buffer.from("0000000000000000000000000\n"), Buffer.from(skewed)]) { + roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }, zeroWeights); + } + } + }); + + // A compressor only looks at its dictionary when it starts, and has one error for all that can go wrong with it. + const refused = ["ZSTD_error_memory_allocation", "ERR_ZLIB_INITIALIZATION_FAILED"]; + const taken = (dictionary: Buffer) => + [ + attempt(() => zlib.zstdCompressSync("hello", { dictionary })), + attempt(() => zlib.zstdDecompressSync(hello, { dictionary })), + ].map(result => (typeof result === "string" ? result : "ok")); + + it("whose repeat offsets reach no further than their content", () => { + const withOffset = (offset: number, at: number) => { + const dictionary = Buffer.from(zeroWeights); + dictionary.writeUInt32LE(offset, at); + return taken(dictionary); + }; + for (const at of [130, 134, 138]) { + expect([withOffset(11, at), withOffset(12, at), withOffset(0, at)]).toEqual([["ok", "ok"], refused, refused]); + } + }); + + it("that are short", () => { + const input = noise(300, 9, 7); + const plain = zlib.zstdCompressSync(input); + for (const length of levels(1, 20)) { + const raw = noise(length, length, 7); + const frame = roundTrip(input, {}, raw); + if (length < 8) expect(frame).toEqual(plain); + // With the magic number, 8 bytes and more have to be a whole dictionary. + hex("37a430ec").copy(raw); + expect(taken(raw)).toEqual(length < 8 ? ["ok", "ok"] : refused); + } + }); + + it("that leave the window", async () => { + const dictionary = noise(10240, 10, 15); + const input = Buffer.alloc(10240); + for (let i = 0, x = 11; i < input.length; x = (x * 1103515245 + 12345) >>> 0) { + const length = 4 + ((x >>> 8) % 57); + const distance = [7, 300, 1000][(x >>> 16) % 3]; + const from = (x >>> 4) % (dictionary.length - 64); + for (let k = 0; k < length && i < input.length; k++, i++) { + input[i] = + x % 5 === 0 + ? (x >>> (k % 24)) & 15 + : x % 5 === 1 || i < distance + ? dictionary[from + k] + : input[i - distance]; + } + } + for (const strategy of levels(1, 9)) { + const params = { [c.ZSTD_c_windowLog]: 10, [c.ZSTD_c_strategy]: strategy, [c.ZSTD_c_checksumFlag]: 1 }; + const frame = (await through(zlib.createZstdCompress({ dictionary, params }), pieces(input, 512))) as Buffer; + const decoder = zlib.createZstdDecompress({ dictionary, params: { [c.ZSTD_d_windowLogMax]: 10 } }); + expect(await through(decoder, pieces(frame, 100))).toEqual(input); + } + }); + }); + + describe("workers", () => { + it("whose overlap starts like a dictionary", async () => { + const input = noise(5 << 20, 12, 7); + for (const job of levels(1, 4)) hex("37a430ec").copy(input, (job << 20) - (1 << 14)); + const params = { + [c.ZSTD_c_nbWorkers]: 2, + [c.ZSTD_c_jobSize]: 1 << 20, + [c.ZSTD_c_windowLog]: 17, + [c.ZSTD_c_overlapLog]: 6, + }; + const frame = (await through(zlib.createZstdCompress({ params }), pieces(input, 300_000))) as Buffer; + expect(zlib.zstdDecompressSync(frame).equals(input)).toBe(true); + }); + + it("with long distance matching and a tiny dictionary", () => { + const input = noise(600_000, 13, 7); + for (const length of [1, 8, 63]) { + roundTrip(input, { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_enableLongDistanceMatching]: 1 }, noise(length, 14)); + } + }); + + // What Node 24 gives, with zstd 1.5.7. + it("make the same of input however it is cut", async () => { + const input = noise(3 << 20, 15, 7); + const options = { pledgedSrcSize: input.length, params: { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_checksumFlag]: 1 } }; + const cut = async (size: number) => { + const frame = (await through(zlib.createZstdCompress(options), pieces(input, size))) as Buffer; + return `${frame.length} ${Bun.MD5.hash(frame, "hex").slice(0, 8)}`; + }; + expect(await cut(input.length)).toBe("1398674 d981d509"); + for (const size of [1 << 20, 64 << 10, 1000]) expect(await cut(size)).toBe("1398687 a871b693"); + }); + + it("with nothing left for the last job", async () => { + const input = noise(2 << 20, 16, 7); + const params = { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 512 << 10, [c.ZSTD_c_checksumFlag]: 1 }; + const dictionary = noise(1000, 17, 7); + const frame = (await through(zlib.createZstdCompress({ params, dictionary }), [input])) as Buffer; + expect(zlib.zstdDecompressSync(frame, { dictionary }).equals(input)).toBe(true); + }); + }); +}); From bee54ea41f2ec772265c134703156d322b84faee Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:54:48 +0000 Subject: [PATCH 101/199] zstd: fast checks once for what it reads around the end of a match, and copies no literals where there are none --- src/zstd/compress/fast.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index d59f7f8a4312..216f75ee81f6 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -537,20 +537,18 @@ impl Finder { sink.store::<32>(start, end, off_base); // `ip0` is not past the last position that can be looked at. - let Some(bytes) = near.read64(ip0) else { + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { break 'block; }; - bytes0 = bytes; + bytes0 = le64_at::<2, _>(around); if let Some(to_insert) = near.read64((current0 + 2).wrapping_sub(base_index)) { hashed_put(hash_table, hasher.hash(to_insert), current0 + 2); } - if let Some(before) = near.read64(ip0 - 2) { - hashed_put( - hash_table, - hasher.hash(before), - base_index.wrapping_add(ip0 - 2), - ); - } + hashed_put( + hash_table, + hasher.hash(le64_at::<0, _>(around)), + base_index.wrapping_add(ip0 - 2), + ); while rep_offset2 > 0 && sink.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) { let end = ip0 + 4 + sink.count_before(ip0 + 4, rep_offset2); @@ -560,7 +558,7 @@ impl Finder { hasher.hash(bytes0), base_index.wrapping_add(ip0), ); - sink.store::<32>(ip0, end, REPCODE1_TO_OFFBASE); + sink.store_repeat(end); ip0 = end; let Some(bytes) = near.read64(ip0) else { break 'block; From 8e241a5c619f07f3be6d1fb073b88f473cb13629 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:58:49 +0000 Subject: [PATCH 102/199] zstd: so does dfast --- src/zstd/compress/double_fast.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 98dd326a971c..e8e9a0e0b0b2 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -252,13 +252,10 @@ impl Finder { ip = end; // `ip` is not past the last position that can be looked at. - let Some(bytes) = near.read64(ip) else { - break 'block; - }; - bytes0 = bytes; - let Some(around) = near.window::<{ 1 + HASH_READ_SIZE }>(ip - 2) else { + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip - 2) else { break 'block; }; + bytes0 = le64_at::<2, _>(around); let index_to_insert = curr + 2; if let Some(to_insert) = near.read64(index_to_insert.wrapping_sub(base_index)) { hashed_put(hash_long, long.hash(to_insert), index_to_insert); From eb2aea28503c09460148fa8e4d8038264ac42881 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:04:31 +0000 Subject: [PATCH 103/199] zstd: with a dictionary's match state, how many bytes are hashed is known to the compiler again --- src/zstd/compress/double_fast.rs | 17 ++++++++++++----- src/zstd/compress/fast.rs | 18 ++++++++++++------ src/zstd/compress/internal.rs | 8 ++++++++ 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index e8e9a0e0b0b2..4bd285de2982 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -344,7 +344,7 @@ pub(crate) fn compress_block_double_fast( impl Finder { /// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` #[inline(never)] - fn find_matches_dict( + fn find_matches_dict( &mut self, hash_long: &mut [u32], hash_small: &mut [u32], @@ -361,6 +361,8 @@ impl Finder { mut offset_2, } = *self; let near = window.near; + let long = long.with_mls(8); + let small = small.with_mls(MLS); if hash_long.is_empty() || hash_small.is_empty() { return; } @@ -385,7 +387,7 @@ impl Finder { if near.has::<{ u64::MAX }>(match_long, bytes0) { break 'found (window.place_of(match_long), 8); } - if let Some(place) = dict_tables.long.get(bytes0) + if let Some(place) = dict_tables.long.get::<8>(bytes0) && window.older.read64(place) == Some(bytes0) { break 'found (place, 8); @@ -395,7 +397,7 @@ impl Finder { let short = if match_index_s > base_index { Some(window.place_of(match_index_s - base_index)) } else { - dict_tables.short.get(bytes0) + dict_tables.short.get::(bytes0) }; let Some(place_short) = short.filter(|&place| window.read32(place) == Some(bytes0 as u32)) @@ -412,7 +414,7 @@ impl Finder { ip += 1; break 'found (window.place_of(match_long1), 8); } - if let Some(place) = dict_tables.long.get(bytes1) + if let Some(place) = dict_tables.long.get::<8>(bytes1) && window.older.read64(place) == Some(bytes1) { ip += 1; @@ -512,7 +514,12 @@ pub(crate) fn compress_block_double_fast_dict_match_state( let sink = &mut MatchSink::with_near(seq_store, block.window.near, block.istart); let long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let small = ms.chain_table.sub_mut(..1 << h_bits_s)?; - finder.find_matches_dict(long, small, sink, block.window, tables); + match ms.c_params.min_match { + 5 => finder.find_matches_dict::<5>(long, small, sink, block.window, tables), + 6 => finder.find_matches_dict::<6>(long, small, sink, block.window, tables), + 7 => finder.find_matches_dict::<7>(long, small, sink, block.window, tables), + _ => finder.find_matches_dict::<4>(long, small, sink, block.window, tables), + } let last_literals = sink.last_literals()?; let [rep0, rep1, _] = rep; *rep0 = finder.offset_1; diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 216f75ee81f6..230807b09bb7 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -813,11 +813,11 @@ impl<'a> TaggedTable<'a> { }) } - /// The place that the table has for the first bytes of `bytes`. `None` if the tag tells that it is for + /// The place that the table has for the `MLS` first bytes of `bytes`, as many as it was made for. `None` if the tag tells that it is for /// something else, or if it is the first place, which stands for none. #[inline(always)] - pub(super) fn get(self, bytes: u64) -> Option { - let hash_and_tag = self.hasher.hash(bytes); + pub(super) fn get(self, bytes: u64) -> Option { + let hash_and_tag = self.hasher.with_mls(MLS).hash(bytes); let index_and_tag = hashed_get(self.table, hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS); let index = index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; (compare_packed_tags(index_and_tag as usize, hash_and_tag) && index > self.start_index) @@ -828,7 +828,7 @@ impl<'a> TaggedTable<'a> { impl Finder { /// `ZSTD_compressBlock_fast_dictMatchState_generic` #[inline(never)] - fn find_matches_dict( + fn find_matches_dict( &mut self, hash_table: &mut [u32], sink: &mut MatchSink<'_, '_>, @@ -845,6 +845,7 @@ impl Finder { mut rep_offset2, } = *self; let near = window.near; + let hasher = hasher.with_mls(MLS); if hash_table.is_empty() || dict_table.table.is_empty() { return; } @@ -876,7 +877,7 @@ impl Finder { } // A match in the dictionary is only taken if there is no candidate in the prefix. - if let Some(place) = dict_table.get(bytes0) + if let Some(place) = dict_table.get::(bytes0) && window.older.read32(place) == Some(bytes0 as u32) && match_index <= base_index { @@ -973,7 +974,12 @@ pub(crate) fn compress_block_fast_dict_match_state( }; let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; - finder.find_matches_dict(hash_table, &mut sink, block.window, dict_table); + match ms.c_params.min_match { + 5 => finder.find_matches_dict::<5>(hash_table, &mut sink, block.window, dict_table), + 6 => finder.find_matches_dict::<6>(hash_table, &mut sink, block.window, dict_table), + 7 => finder.find_matches_dict::<7>(hash_table, &mut sink, block.window, dict_table), + _ => finder.find_matches_dict::<4>(hash_table, &mut sink, block.window, dict_table), + } let last_literals = sink.last_literals()?; let [rep0, rep1, _] = rep; *rep0 = finder.rep_offset1; diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 57da01a0c3e2..e3b1ee9b844b 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1608,6 +1608,14 @@ impl Hasher { } } + /// The same of `mls` bytes. + pub(crate) const fn with_mls(self, mls: u32) -> Self { + Self { + shift: self.shift, + ..Self::new(0, mls) + } + } + /// [`hash_ptr`] of the position that the 8 bytes `bytes` were read at. #[inline(always)] pub(crate) const fn hash(self, bytes: u64) -> usize { From fd951f43429c44ff5903c5b70dc1f3d23b9363aa Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:04:33 +0000 Subject: [PATCH 104/199] zstd: nobody waits for workers to leave Closing a stream whose workers were busy took as long as they needed for what they were at, on the thread that closed it. --- src/zstd/compress/mt.rs | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 12a2160e7c55..a1a849bdf9fe 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -10,7 +10,6 @@ use std::collections::VecDeque; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpmc::{Receiver, Sender, TryRecvError, channel}; -use std::thread::JoinHandle; use super::cctx::{ CDict, DictContentType, FrameCompressor, Input, Span, reset_table, write_last_empty_block, @@ -379,7 +378,8 @@ impl PendingJob { /// `ZSTDMT_CCtx` pub(crate) struct MtCtx { - workers: Vec>, + /// They are on their own: they share nothing, and leave once there is no queue or the frame is given up. + nb_workers: usize, /// Whichever worker is free takes the next job, so that a job does not wait for one thread to be scheduled. queue: Sender, /// The other end, for workers to come. @@ -450,7 +450,7 @@ impl MtCtx { pub(crate) fn new(nb_workers: u32) -> Result { let (queue, queue_end) = channel(); let mut mtctx = Self { - workers: Vec::new(), + nb_workers: 0, queue, queue_end, cancelled: Arc::new(AtomicBool::new(false)), @@ -476,31 +476,25 @@ impl MtCtx { /// `ZSTDMT_resize`: no job is under way. fn resize(&mut self, nb_workers: usize) -> Result<()> { - if nb_workers < self.workers.len() { - self.stop_workers(); + if nb_workers < self.nb_workers { + // They leave when nothing can be sent to the queue they have. + (self.queue, self.queue_end) = channel(); + self.nb_workers = 0; } - while self.workers.len() < nb_workers { + while self.nb_workers < nb_workers { let queue_end = self.queue_end.clone(); let cancelled = Arc::clone(&self.cancelled); - let worker = std::thread::Builder::new() + std::thread::Builder::new() .name("zstd".into()) .stack_size(2 << 20) .spawn(move || worker_main(&queue_end, &cancelled)) .map_err(|_| ErrorCode::MemoryAllocation)?; - self.workers.push(worker); + self.nb_workers += 1; } self.params.nb_workers = nb_workers as i32; Ok(()) } - fn stop_workers(&mut self) { - // They leave when nothing can be sent to the queue they have. - (self.queue, self.queue_end) = channel(); - for worker in self.workers.drain(..) { - let _ = worker.join(); - } - } - /// `ZSTDMT_waitForAllJobsCompleted` and `ZSTDMT_releaseAllJobResources`, for a frame that is given up. fn abandon_jobs(&mut self) { self.cancelled.store(true, Ordering::SeqCst); @@ -585,7 +579,7 @@ impl MtCtx { /// The most jobs at a time, which bounds memory as the size of the round buffer does in the C code. fn max_jobs(&self) -> usize { - self.workers.len() + 2 + self.nb_workers + 2 } /// `ZSTDMT_createCompressionJob`: of the input there is in `in_buff`. @@ -820,7 +814,7 @@ impl MtCtx { impl Drop for MtCtx { fn drop(&mut self) { + // Nobody waits for them, which could take as long as a job does. self.cancelled.store(true, Ordering::SeqCst); - self.stop_workers(); } } From e4de7dfefb9ce876b2e079347c3b1a11e4faefb9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:04:34 +0000 Subject: [PATCH 105/199] zstd: test for the code of an error --- test/js/bun/util/zstd.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 7cdaafc4fe55..678a3d779530 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1456,7 +1456,7 @@ describe("bugs of zstd 1.5.7", () => { expect(() => zlib.zstdDecompressSync(frame, { chunkSize: 262086 })).toThrow( expect.objectContaining({ code: "ZSTD_error_corruption_detected" }), ); - expect(() => Bun.zstdDecompressSync(frame)).toThrow(); + expect(() => Bun.zstdDecompressSync(frame)).toThrow(expect.objectContaining({ code: "ERR_ZSTD" })); }); // No decoder checked that. A streaming one keeps little more than the window, and gave whatever was there by then. From 8e1228962e5d4804ee2169b81ef0352c2f846c5e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:05:03 +0000 Subject: [PATCH 106/199] zstd: fast with a dictionary's match state checks once around the end of a match too --- src/zstd/compress/fast.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 230807b09bb7..fea3628ecf82 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -908,20 +908,18 @@ impl Finder { } // `ip0` is not past the last position that can be looked at. - let Some(bytes) = near.read64(ip0) else { + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { break 'block; }; - bytes0 = bytes; + bytes0 = le64_at::<2, _>(around); if let Some(to_insert) = near.read64((curr + 2).wrapping_sub(base_index)) { hashed_put(hash_table, hasher.hash(to_insert), curr + 2); } - if let Some(before) = near.read64(ip0 - 2) { - hashed_put( - hash_table, - hasher.hash(before), - base_index.wrapping_add(ip0 - 2), - ); - } + hashed_put( + hash_table, + hasher.hash(le64_at::<0, _>(around)), + base_index.wrapping_add(ip0 - 2), + ); while window.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) { let rep_place = window.place_of(ip0).wrapping_sub(rep_offset2); @@ -932,7 +930,7 @@ impl Finder { hasher.hash(bytes0), base_index.wrapping_add(ip0), ); - sink.store::<32>(ip0, end, REPCODE1_TO_OFFBASE); + sink.store_repeat(end); ip0 = end; let Some(bytes) = near.read64(ip0) else { break 'block; From 3c5a4774c1c9acc246ebf5e71893cac2e9a2bd4c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:05:43 +0000 Subject: [PATCH 107/199] zstd: and with two segments --- src/zstd/compress/fast.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index fea3628ecf82..0fbf98c216d2 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -673,20 +673,18 @@ impl Finder { } // `ip0` is not past the last position that can be looked at. - let Some(bytes) = near.read64(ip0) else { + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { break 'block; }; - bytes0 = bytes; + bytes0 = le64_at::<2, _>(around); if let Some(to_insert) = near.read64((current0 + 2).wrapping_sub(base_index)) { hashed_put(hash_table, hasher.hash(to_insert), current0 + 2); } - if let Some(before) = near.read64(ip0 - 2) { - hashed_put( - hash_table, - hasher.hash(before), - base_index.wrapping_add(ip0 - 2), - ); - } + hashed_put( + hash_table, + hasher.hash(le64_at::<0, _>(around)), + base_index.wrapping_add(ip0 - 2), + ); while window.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) && rep_offset2 > 0 { let rep_place = window.place_of(ip0).wrapping_sub(rep_offset2); @@ -697,7 +695,7 @@ impl Finder { hasher.hash(bytes0), base_index.wrapping_add(ip0), ); - sink.store::<32>(ip0, end, REPCODE1_TO_OFFBASE); + sink.store_repeat(end); ip0 = end; let Some(bytes) = near.read64(ip0) else { break 'block; From 06433fdde2648cd16c4e7e262bbb53151600b44d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:14:50 +0000 Subject: [PATCH 108/199] zstd: a thread's next decoder takes over no output either --- src/zstd/decompress/block.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 59ee9d61f2b3..6d90594080b4 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -235,8 +235,8 @@ struct Scratch { } thread_local! { - /// That of the last decoder that was dropped, without its literals, for the next one. Nothing depends on what is - /// in the tables, and it takes longer to clear new ones than to decode a small frame. + /// That of the last decoder that was dropped, without its literals and output, for the next one. Nothing depends on + /// what is in the tables, and it takes longer to clear new ones than to decode a small frame. static SPARE_SCRATCH: cell::Cell>> = const { cell::Cell::new(None) }; } @@ -263,6 +263,7 @@ impl Drop for BlockDecoder { fn drop(&mut self) { if let Some(mut scratch) = self.scratch.take() { scratch.lit_buffer.clear(); + scratch.small_output.fill(0); if scratch.lit_buffer.capacity() > Scratch::MAX_SPARE_LIT_BUFFER { scratch.lit_buffer = Vec::new(); } From 0ffd40af661336119fb12d248ae7521b076f1a1a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:15:49 +0000 Subject: [PATCH 109/199] zstd: fast and dfast index a dictionary by 10 bytes at a time, and with no look at what is in the tables --- src/zstd/compress/double_fast.rs | 88 +++++++------------ src/zstd/compress/fast.rs | 141 ++++++++++++++++++++++--------- src/zstd/compress/internal.rs | 12 --- 3 files changed, 135 insertions(+), 106 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 4bd285de2982..8aa8fea29916 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -1,54 +1,37 @@ //! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. use super::fast::{ - DictBlock, ExtDictBlock, FAST_HASH_FILL_STEP, MatchSink, NoDictBlock, Segments, TaggedTable, - restore_rep, + DictBlock, ExtDictBlock, MatchSink, NoDictBlock, Segments, TaggedTable, ThirdPositions, + put_tagged, put_tagged_in_between, restore_rep, }; use super::internal::{ BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, Hasher, K_SEARCH_STRENGTH, MatchState, Near, - Prefix, REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, hash_ptr, - hashed_get, hashed_put, hashed_swap, le64_at, offset_to_offbase, table_of_size, table_prefetch, - write_tagged_index, + Prefix, REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, hashed_get, hashed_put, hashed_swap, + le64_at, offset_to_offbase, table_get, table_of_size, table_prefetch, table_put, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; use crate::util::{BUG, SliceExt}; -/// `ZSTD_fillDoubleHashTableForCDict` +/// `ZSTD_fillDoubleHashTableForCDict`: the tables are empty. fn fill_double_hash_table_for_cdict( ms: &mut MatchState, prefix: Prefix<'_>, end: usize, dtlm: DictTableLoadMethod, ) -> Result<()> { - let h_bits_l = ms.tagged_hash_log(); - let h_bits_s = ms.tagged_chain_log(); - let mls = ms.c_params.min_match; + let long = Hasher::new(ms.tagged_hash_log(), 8); + let small = Hasher::new(ms.tagged_chain_log(), ms.c_params.min_match); + let positions = ThirdPositions::new(prefix, ms.next_to_update, end)?; let hash_large = ms.hash_table.as_mut_slice(); let hash_small = ms.chain_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - // The positions in between only go in the table of long hashes, where there is nothing yet. - while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { - let curr = prefix.index_of(ip); - for i in 0..FAST_HASH_FILL_STEP { - let sm_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; - let lg_hash_and_tag = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; - if i == 0 { - write_tagged_index(hash_small, sm_hash_and_tag, curr + i)?; - } - if i == 0 || hash_large.at(lg_hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { - write_tagged_index(hash_large, lg_hash_and_tag, curr + i)?; - } - if dtlm == DictTableLoadMethod::Fast { - break; - } - } - ip += FAST_HASH_FILL_STEP as usize; + if dtlm == DictTableLoadMethod::Full { + put_tagged_in_between(hash_large, long, positions.clone()); + } + for at in positions { + let bytes = le64_at::<0, _>(at.bytes); + put_tagged(hash_small, small.hash(bytes), at.index); + put_tagged(hash_large, long.hash(bytes), at.index); } Ok(()) } @@ -60,34 +43,27 @@ fn fill_double_hash_table_for_cctx( end: usize, dtlm: DictTableLoadMethod, ) -> Result<()> { - let h_bits_l = ms.c_params.hash_log; - let h_bits_s = ms.c_params.chain_log; - let mls = ms.c_params.min_match; + let long = Hasher::new(ms.c_params.hash_log, 8); + let small = Hasher::new(ms.c_params.chain_log, ms.c_params.min_match); + let positions = ThirdPositions::new(prefix, ms.next_to_update, end)?; let hash_large = ms.hash_table.as_mut_slice(); let hash_small = ms.chain_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - // The positions in between only go in the table of long hashes, where there is nothing yet. - while ip + FAST_HASH_FILL_STEP as usize - 1 <= iend { - let curr = prefix.index_of(ip); - for i in 0..FAST_HASH_FILL_STEP { - let sm_hash = hash_ptr(buf, ip + i as usize, h_bits_s, mls)?; - let lg_hash = hash_ptr(buf, ip + i as usize, h_bits_l, 8)?; - if i == 0 { - hash_small.put(sm_hash, curr + i)?; - } - if i == 0 || hash_large.at(lg_hash)? == 0 { - hash_large.put(lg_hash, curr + i)?; - } - if dtlm == DictTableLoadMethod::Fast { - break; + for at in positions { + let bytes = le64_at::<0, _>(at.bytes); + table_put(hash_small, small.hash(bytes), at.index); + table_put(hash_large, long.hash(bytes), at.index); + // The positions in between only go in the table of long hashes, where there is nothing yet. + if dtlm == DictTableLoadMethod::Full { + for (bytes, index) in [ + (le64_at::<1, _>(at.bytes), at.index + 1), + (le64_at::<2, _>(at.bytes), at.index + 2), + ] { + let hash = long.hash(bytes); + if table_get(hash_large, hash) == 0 { + table_put(hash_large, hash, index); + } } } - ip += FAST_HASH_FILL_STEP as usize; } Ok(()) } diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 0fbf98c216d2..dd0990217a8c 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -3,15 +3,14 @@ use super::internal::{ BlockSrc, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, Hasher, K_SEARCH_STRENGTH, MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, - TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, hash_ptr, hashed_get, - hashed_put, hashed_swap, le32_at, le64_at, offset_to_offbase, table_of_size, - write_tagged_index, + TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, hashed_get, hashed_put, + hashed_swap, le32_at, le64_at, offset_to_offbase, table_of_size, table_put, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; use crate::util::{BUG, SliceExt}; -pub(super) const FAST_HASH_FILL_STEP: u32 = 3; +const FAST_HASH_FILL_STEP: u32 = 3; /// The repeat offsets at the start of a block. Those that reach further than `max_rep` are 0, and saved. pub(super) struct SavedRep { @@ -71,54 +70,120 @@ pub(super) fn restore_rep( }; } -/// `ZSTD_fillHashTableForCDict` +/// What `fast` and `dfast` index a dictionary by: every third position, with the 10 bytes there, which the 8 bytes at +/// it and at the two positions in between are among. +#[derive(Clone)] +pub(super) struct ThirdPositions<'a> { + /// From the first position that is left up to the end of the 10 bytes at the last one. + bytes: &'a [u8], + /// The index of the first position that is left. + index: u32, +} + +/// One of [`ThirdPositions`]. +pub(super) struct ThirdPosition<'a> { + pub(super) index: u32, + pub(super) bytes: &'a [u8; THIRD_POSITION_SIZE], +} + +const THIRD_POSITION_SIZE: usize = FAST_HASH_FILL_STEP as usize - 1 + HASH_READ_SIZE; + +impl<'a> ThirdPositions<'a> { + /// From `first_index` up to the position `end` of the segment. + pub(super) fn new(prefix: Prefix<'a>, first_index: u32, end: usize) -> Result { + let bytes = prefix.buf.sub(prefix.pos_of(first_index)..end)?; + let len = match bytes.len().checked_sub(THIRD_POSITION_SIZE) { + Some(more) => bytes.len() - more % FAST_HASH_FILL_STEP as usize, + None => 0, + }; + Ok(Self { + bytes: bytes.sub(..len)?, + index: first_index, + }) + } +} + +impl<'a> Iterator for ThirdPositions<'a> { + type Item = ThirdPosition<'a>; + + #[inline(always)] + fn next(&mut self) -> Option { + let item = ThirdPosition { + index: self.index, + bytes: self.bytes.first_chunk()?, + }; + self.bytes = self.bytes.get(FAST_HASH_FILL_STEP as usize..)?; + self.index += FAST_HASH_FILL_STEP; + Some(item) + } +} + +impl DoubleEndedIterator for ThirdPositions<'_> { + #[inline(always)] + fn next_back(&mut self) -> Option { + let (before, bytes) = self.bytes.split_last_chunk()?; + let item = ThirdPosition { + index: self.index + before.len() as u32, + bytes, + }; + let len = self.bytes.len() - FAST_HASH_FILL_STEP as usize; + self.bytes = self.bytes.get(..len)?; + Some(item) + } +} + +/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. The size of +/// `table` is a power of 2 that the hash is less than. +#[inline(always)] +pub(super) fn put_tagged(table: &mut [u32], hash_and_tag: usize, index: u32) { + let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + if let Some(cell) = table.get_mut(hash & table.len().wrapping_sub(1)) { + *cell = (index << ZSTD_SHORT_CACHE_TAG_BITS) | u32::from(hash_and_tag as u8); + } +} + +/// Indexes the positions between `positions`, for which the C code only has the cells that are empty when it gets to +/// them, on its way from the first position to the last. As `table` is empty, it comes to the same to go from the last +/// to the first, and to index `positions` afterwards. Whether a cell is empty, which is as good as random, is then of +/// no concern. +#[inline(always)] +pub(super) fn put_tagged_in_between( + table: &mut [u32], + hasher: Hasher, + positions: ThirdPositions<'_>, +) { + for at in positions.rev() { + put_tagged(table, hasher.hash(le64_at::<2, _>(at.bytes)), at.index + 2); + put_tagged(table, hasher.hash(le64_at::<1, _>(at.bytes)), at.index + 1); + } +} + +/// `ZSTD_fillHashTableForCDict`: the table is empty. fn fill_hash_table_for_cdict( ms: &mut MatchState, prefix: Prefix<'_>, end: usize, dtlm: DictTableLoadMethod, ) -> Result<()> { - let h_bits = ms.tagged_hash_log(); - let mls = ms.c_params.min_match; + let hasher = Hasher::new(ms.tagged_hash_log(), ms.c_params.min_match); + let positions = ThirdPositions::new(prefix, ms.next_to_update, end)?; let hash_table = ms.hash_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { - let curr = prefix.index_of(ip); - write_tagged_index(hash_table, hash_ptr(buf, ip, h_bits, mls)?, curr)?; - if dtlm == DictTableLoadMethod::Full { - // The positions in between only go where there is nothing yet. - for p in 1..FAST_HASH_FILL_STEP { - let hash_and_tag = hash_ptr(buf, ip + p as usize, h_bits, mls)?; - if hash_table.at(hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS)? == 0 { - write_tagged_index(hash_table, hash_and_tag, curr + p)?; - } - } - } - ip += FAST_HASH_FILL_STEP as usize; + if dtlm == DictTableLoadMethod::Full { + put_tagged_in_between(hash_table, hasher, positions.clone()); + } + for at in positions { + put_tagged(hash_table, hasher.hash(le64_at::<0, _>(at.bytes)), at.index); } Ok(()) } /// `ZSTD_fillHashTableForCCtx`, which is only for `ZSTD_dtlm_fast`. fn fill_hash_table_for_cctx(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) -> Result<()> { - let h_bits = ms.c_params.hash_log; - let mls = ms.c_params.min_match; + let hasher = Hasher::new(ms.c_params.hash_log, ms.c_params.min_match); + let positions = ThirdPositions::new(prefix, ms.next_to_update, end)?; let hash_table = ms.hash_table.as_mut_slice(); - let buf = prefix.buf; - let mut ip = prefix.pos_of(ms.next_to_update); - let Some(iend) = end.checked_sub(HASH_READ_SIZE) else { - return Ok(()); - }; - - while ip + (FAST_HASH_FILL_STEP as usize) < iend + 2 { - let curr = prefix.index_of(ip); - hash_table.put(hash_ptr(buf, ip, h_bits, mls)?, curr)?; - ip += FAST_HASH_FILL_STEP as usize; + for at in positions { + table_put(hash_table, hasher.hash(le64_at::<0, _>(at.bytes)), at.index); } Ok(()) } diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index e3b1ee9b844b..2fcc9d30cb6f 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1914,18 +1914,6 @@ pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32 pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; -/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. -#[inline(always)] -pub(crate) fn write_tagged_index( - hash_table: &mut [u32], - hash_and_tag: usize, - index: u32, -) -> Result<()> { - let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - let tag = hash_and_tag as u32 & ZSTD_SHORT_CACHE_TAG_MASK; - hash_table.put(hash, (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag) -} - /// `ZSTD_comparePackedTags` #[inline(always)] pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { From 271748524cb5098c812d3aff7649025766c69243 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:19:17 +0000 Subject: [PATCH 110/199] zstd: nothing to index is no error --- src/zstd/compress/double_fast.rs | 4 ++-- src/zstd/compress/fast.rs | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 8aa8fea29916..958be79feebf 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -22,7 +22,7 @@ fn fill_double_hash_table_for_cdict( ) -> Result<()> { let long = Hasher::new(ms.tagged_hash_log(), 8); let small = Hasher::new(ms.tagged_chain_log(), ms.c_params.min_match); - let positions = ThirdPositions::new(prefix, ms.next_to_update, end)?; + let positions = ThirdPositions::new(prefix, ms.next_to_update, end); let hash_large = ms.hash_table.as_mut_slice(); let hash_small = ms.chain_table.as_mut_slice(); if dtlm == DictTableLoadMethod::Full { @@ -45,7 +45,7 @@ fn fill_double_hash_table_for_cctx( ) -> Result<()> { let long = Hasher::new(ms.c_params.hash_log, 8); let small = Hasher::new(ms.c_params.chain_log, ms.c_params.min_match); - let positions = ThirdPositions::new(prefix, ms.next_to_update, end)?; + let positions = ThirdPositions::new(prefix, ms.next_to_update, end); let hash_large = ms.hash_table.as_mut_slice(); let hash_small = ms.chain_table.as_mut_slice(); for at in positions { diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index dd0990217a8c..9b4cfad86bb1 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -89,17 +89,18 @@ pub(super) struct ThirdPosition<'a> { const THIRD_POSITION_SIZE: usize = FAST_HASH_FILL_STEP as usize - 1 + HASH_READ_SIZE; impl<'a> ThirdPositions<'a> { - /// From `first_index` up to the position `end` of the segment. - pub(super) fn new(prefix: Prefix<'a>, first_index: u32, end: usize) -> Result { - let bytes = prefix.buf.sub(prefix.pos_of(first_index)..end)?; + /// From `first_index` up to the position `end` of the segment: none if that is before. + pub(super) fn new(prefix: Prefix<'a>, first_index: u32, end: usize) -> Self { + let bytes = prefix.buf.get(prefix.pos_of(first_index)..end); + let bytes = bytes.unwrap_or_default(); let len = match bytes.len().checked_sub(THIRD_POSITION_SIZE) { Some(more) => bytes.len() - more % FAST_HASH_FILL_STEP as usize, None => 0, }; - Ok(Self { - bytes: bytes.sub(..len)?, + Self { + bytes: bytes.get(..len).unwrap_or_default(), index: first_index, - }) + } } } @@ -166,7 +167,7 @@ fn fill_hash_table_for_cdict( dtlm: DictTableLoadMethod, ) -> Result<()> { let hasher = Hasher::new(ms.tagged_hash_log(), ms.c_params.min_match); - let positions = ThirdPositions::new(prefix, ms.next_to_update, end)?; + let positions = ThirdPositions::new(prefix, ms.next_to_update, end); let hash_table = ms.hash_table.as_mut_slice(); if dtlm == DictTableLoadMethod::Full { put_tagged_in_between(hash_table, hasher, positions.clone()); @@ -180,7 +181,7 @@ fn fill_hash_table_for_cdict( /// `ZSTD_fillHashTableForCCtx`, which is only for `ZSTD_dtlm_fast`. fn fill_hash_table_for_cctx(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) -> Result<()> { let hasher = Hasher::new(ms.c_params.hash_log, ms.c_params.min_match); - let positions = ThirdPositions::new(prefix, ms.next_to_update, end)?; + let positions = ThirdPositions::new(prefix, ms.next_to_update, end); let hash_table = ms.hash_table.as_mut_slice(); for at in positions { table_put(hash_table, hasher.hash(le64_at::<0, _>(at.bytes)), at.index); From 3c893fc59edcb58774b57c1e0a04f93df3622936 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:15:07 +0000 Subject: [PATCH 111/199] zstd: the lazy parsers weigh a repeat offset only where it gives a match --- src/zstd/compress/lazy.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 55dde0b96b1c..9b2513cc8f26 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -1889,12 +1889,14 @@ fn compress_block_lazy_generic<'a, F: Finder<'a>, const DEPTH: u32, const DICT_M } ip += 1; let rep_len = rep_length::(block, ip, offset_1)?; - let weight = 2 + since_best; - let gain2 = (rep_len * weight) as i32; - let gain1 = (best.len * weight) as i32 - best.cost() + 1; - if rep_len >= 4 && gain2 > gain1 { - best = Found::repeat(rep_len); - start = ip; + if rep_len >= 4 { + let weight = 2 + since_best; + let gain2 = (rep_len * weight) as i32; + let gain1 = (best.len * weight) as i32 - best.cost() + 1; + if gain2 > gain1 { + best = Found::repeat(rep_len); + start = ip; + } } wanted = Wanted { longer_than: 3, From 573e4c09853680bd76c85d9c2d5e146020982560 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:15:08 +0000 Subject: [PATCH 112/199] zstd: fetch all of a row of 64 ahead of its search --- src/zstd/compress/lazy.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 9b2513cc8f26..7979c11b82ed 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -1174,6 +1174,11 @@ fn row_prefetch( let start = rel_row::(hash); prefetch_l1(indices, start); prefetch_l1(indices, start + 16); + if ROW_ENTRIES == 64 { + // The C code leaves this half out. + prefetch_l1(indices, start + 32); + prefetch_l1(indices, start + 48); + } let start = rel_tag_row::(hash); prefetch_l1(tags, start); if SECOND_TAGS { From 0db59a615ad9ab5a9a83a080d2da4f82b0f0479a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:15:09 +0000 Subject: [PATCH 113/199] zstd: faster hash chains, most of all for short inputs A position is added to the chains with the hash and the head that its own search found. Runs of literals go through a loop of their own, which hands the first candidate that matches over to the search. A block where every index is within reach and in the chain table has its own version of the finder, without the checks for that. A search for a match of a given length is given up if the end of such a match has no position in the hash table. Positions are added two at a time after a match. --- src/zstd/compress/internal.rs | 6 + src/zstd/compress/lazy.rs | 256 +++++++++++++++++++++++++++++----- 2 files changed, 227 insertions(+), 35 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 4a7cff824102..4c2e57a4bcc1 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1682,6 +1682,12 @@ impl Hash8 { pub(crate) const fn of(self, bytes: u64) -> u32 { (bytes.wrapping_mul(self.mul) ^ self.salt).wrapping_shr(self.shift) as u32 } + + /// [`Self::of`], when the salt is known to be 0. + #[inline(always)] + pub(crate) const fn of_unsalted(self, bytes: u64) -> u32 { + bytes.wrapping_mul(self.mul).wrapping_shr(self.shift) as u32 + } } // ── the window ──────────────────────────────────────────────────────────── diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 7979c11b82ed..ea805d342ed6 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -606,6 +606,8 @@ struct Longest<'a> { ip: u32, /// A match has to be more than 3 bytes longer than this. tail_start: u32, + /// `segment` from `tail_start` on. + tails: Near<'a>, /// What such a match has in common with `ip`: the 4 bytes from `tail_start` on. tail: u32, /// 0 as long as there is no match. @@ -623,6 +625,7 @@ impl<'a> Longest<'a> { start_index, ip, tail_start, + tails: segment.tail(tail_start)?, tail: segment.read32(ip + tail_start)?, off_base: best.off_base, }) @@ -634,9 +637,13 @@ impl<'a> Longest<'a> { if len > self.tail_start + 3 { self.tail_start = len - 3; self.off_base = off_base; - let Some(tail) = self.segment.read32(self.ip + self.tail_start) else { + let (Some(tails), Some(tail)) = ( + self.segment.tail(self.tail_start), + self.segment.read32(self.ip + self.tail_start), + ) else { return Offered::Longest; }; + self.tails = tails; self.tail = tail; } Offered::Considered @@ -664,12 +671,18 @@ impl<'a> Longest<'a> { Ok(self.offer_match(len, offset_to_offbase(offset))) } - /// There may be a match at `match_index`. + /// There may be a match at `match_index`. `IN_TAILS` is which of two ways its tail is read: the one that takes fewer + /// instructions, or the one that has less to wait for after a match. #[inline(always)] - fn offer(&mut self, match_index: u32) -> Offered { + fn offer(&mut self, match_index: u32) -> Offered { let match_pos = match_index.wrapping_sub(self.start_index); - let tail_pos = match_pos as usize + self.tail_start as usize; - let Some(tail) = self.segment.read32_at(tail_pos) else { + let tail = if IN_TAILS { + self.tails.read32(match_pos) + } else { + self.segment + .read32_at(match_pos as usize + self.tail_start as usize) + }; + let Some(tail) = tail else { return Offered::Outside; }; if tail != self.tail { @@ -688,6 +701,24 @@ impl<'a> Longest<'a> { } } +/// Where the parser is, when it has no match. +#[derive(Clone, Copy)] +struct LiteralRun { + ip: u32, + /// Where the literals start. + anchor: u32, + /// Positions are searched up to this one, which is not. + ilimit: u32, + offset_1: u32, +} + +/// Where the parser goes on after [`Finder::skip_literals`]. +struct Skipped { + ip: u32, + /// The repeat offset is known to give no match at the next position. + no_repeat: bool, +} + /// `ZSTD_searchMax`, with the state of the search in local variables for as long as a block lasts. trait Finder<'a>: Sized { /// How far from the end of the block the last search is. @@ -707,6 +738,16 @@ trait Finder<'a>: Sized { skipping: bool, ) -> Result>; + /// Goes past positions where the parser would find nothing, neither with [`Self::find`] nor with the repeat offset + /// at the next position: goes on from `run.ip`, or from a later position that the parser would come to. + #[inline(always)] + fn skip_literals(&mut self, run: LiteralRun) -> Result { + Ok(Skipped { + ip: run.ip, + no_repeat: false, + }) + } + /// Every position is indexed again from now on. fn stop_skipping(&mut self) -> Result<()>; @@ -819,66 +860,121 @@ impl Chains<'_> { /// Adds every position that has 8 bytes in `positions`. The first one has the index `index`. #[inline(never)] - fn insert_all(mut self, hasher: Hash8, positions: &[u8], mut index: u32) { - for bytes in positions.array_windows() { - self.insert(hasher, u64::from_le_bytes(*bytes), index); - index = index.wrapping_add(1); + fn insert_all(self, hasher: Hash8, positions: &[u8], index: u32) { + let (Some(hash_mask), Some(chain_mask)) = ( + self.hash_table.len().checked_sub(1), + self.chain_table.len().checked_sub(1), + ) else { + return; + }; + for (n, bytes) in positions.array_windows().enumerate() { + let index = index as usize + n; + let hash = hasher.of_unsalted(u64::from_le_bytes(*bytes)) as usize; + if let Some(head) = self.hash_table.get_mut(hash & hash_mask) + && let Some(next) = self.chain_table.get_mut(index & chain_mask) + { + *next = core::mem::replace(head, index as u32); + } } } } -/// `ZSTD_HcFindBestMatch` -struct ChainFinder<'a, const DICT_MODE: u8> { +/// Where the search of a position is at. +#[derive(Clone, Copy)] +struct Candidate { + /// The first candidate that is a match. + match_index: u32, + /// How many there may be, with that one. + nb_attempts: u32, +} + +/// `ZSTD_HcFindBestMatch`. `SIMPLE` tells that all along the block, every index from that of the position 0 on is within +/// reach and in the chain table, and none below it is valid. +struct ChainFinder<'a, const DICT_MODE: u8, const SIMPLE: bool> { block: LazyBlock<'a>, chains: Chains<'a>, hasher: Hash8, + mls: u32, nb_attempts: u32, /// `ms.next_to_update`, as a position. next_to_update: u32, + /// The hash of `next_to_update`, if it was searched. Else it is out of the hash table. + next_hash: usize, + /// What the hash table had for it, and still has. + next_head: u32, + /// The index from which every position is in the chains, up to `next_to_update`. + dense_from: u32, + /// Where the search of `next_to_update` is at, if [`Finder::skip_literals`] started it. + started: Option, ms_next_to_update: &'a mut u32, dms: Option>, } -impl ChainFinder<'_, DICT_MODE> { +/// Whether [`ChainFinder`] can be `SIMPLE`. +fn chains_are_simple(ms: &MatchState, block: &LazyBlock<'_>) -> bool { + let reach = block.reach.max_distance.min(ms.chain_table.len() as u32); + block.reach.lowest_valid == block.base_index && block.near.len() <= reach +} + +impl ChainFinder<'_, DICT_MODE, SIMPLE> { /// `ZSTD_insertAndFindFirstIndex_internal`: returns the index of the first candidate for `ip`. #[inline(always)] fn insert_and_find_first_index(&mut self, ip: u32, skipping: bool) -> Result { let near = self.block.near; let first = self.next_to_update; - let index = self.block.base_index.wrapping_add(first); - // While skipping, not every position is inserted. - if first.wrapping_add(1) == ip || (skipping && first < ip) { - let bytes = near.read64(first).ok_or(BUG)?; - self.chains.insert(self.hasher, bytes, index); - } else if first < ip { - let positions = near.bytes().sub(first as usize..ip as usize + 7)?; - let chains = Chains { - hash_table: self.chains.hash_table, - chain_table: self.chains.chain_table, - }; - chains.insert_all(self.hasher, positions, index); + if first < ip { + let index = self.block.base_index.wrapping_add(first); + if let Some(head) = self.chains.hash_table.get_mut(self.next_hash) { + *head = index; + table_put(self.chains.chain_table, index as usize, self.next_head); + } else { + let bytes = near.read64(first).ok_or(BUG)?; + self.chains.insert(self.hasher, bytes, index); + } + if skipping { + // Not every position is inserted. + self.dense_from = self.block.base_index.wrapping_add(ip); + } else if first + 1 < ip { + let positions = near.bytes().sub(first as usize + 1..ip as usize + 7)?; + let chains = Chains { + hash_table: self.chains.hash_table, + chain_table: self.chains.chain_table, + }; + chains.insert_all(self.hasher, positions, index.wrapping_add(1)); + } } self.next_to_update = ip; let hash = self.hasher.of(near.read64(ip).ok_or(BUG)?) as usize; - Ok(table_get(self.chains.hash_table, hash)) + let hash = hash & self.chains.hash_table.len().wrapping_sub(1); + let head = self.chains.hash_table.get(hash).copied().unwrap_or(0); + self.next_hash = hash; + self.next_head = head; + Ok(head) } } -impl<'a, const DICT_MODE: u8> Finder<'a> for ChainFinder<'a, DICT_MODE> { +impl<'a, const DICT_MODE: u8, const SIMPLE: bool> Finder<'a> + for ChainFinder<'a, DICT_MODE, SIMPLE> +{ const MARGIN: u32 = 8; #[inline(always)] fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result { let mls = ms.c_params.min_match.clamp(4, 6); Ok(Self { + started: None, block, chains: Chains { hash_table: &mut ms.hash_table, chain_table: &mut ms.chain_table, }, hasher: Hash8::new(mls, ms.c_params.hash_log, 0), + mls, nb_attempts: 1 << ms.c_params.search_log, next_to_update: ms.next_to_update.wrapping_sub(block.base_index), + next_hash: usize::MAX, + next_head: 0, + dense_from: ms.next_to_update, ms_next_to_update: &mut ms.next_to_update, dms: src.dms.as_ref().map(|dms| ChainDict::new(dms, mls)), }) @@ -893,12 +989,22 @@ impl<'a, const DICT_MODE: u8> Finder<'a> for ChainFinder<'a, DICT_MODE> { ) -> Result> { let block = self.block; let curr = block.base_index.wrapping_add(ip); - let low_limit = block.reach.low_limit(curr); + let low_limit = if SIMPLE { + block.base_index + } else { + block.reach.low_limit(curr) + }; let min_chain = curr.saturating_sub(self.chains.chain_table.len() as u32); let mut nb_attempts = self.nb_attempts; - let mut match_index = self.insert_and_find_first_index(ip, skipping)?; - if DICT_MODE == NO_DICT && match_index < low_limit { - return Ok(None); + let mut match_index; + if let Some(started) = self.started.take() { + match_index = started.match_index; + nb_attempts = started.nb_attempts; + } else { + match_index = self.insert_and_find_first_index(ip, skipping)?; + if DICT_MODE == NO_DICT && match_index < low_limit { + return Ok(None); + } } let closest = if DICT_MODE == DICT_MATCH_STATE { @@ -919,11 +1025,22 @@ impl<'a, const DICT_MODE: u8> Finder<'a> for ChainFinder<'a, DICT_MODE> { len: wanted.floor::(curr, closest), off_base: 0, }; + // A match that long has a position in the chains for each of its bytes, but the last ones. + if BY_GAIN + && DICT_MODE == NO_DICT + && best.len > 3 + && low_limit >= self.dense_from + && let Some(end) = (best.len + 1).checked_sub(self.mls) + && let Some(bytes) = block.near.read64(ip + end) + && table_get(self.chains.hash_table, self.hasher.of(bytes) as usize) < low_limit + { + return Ok(None); + } let Some(mut longest) = Longest::new(segment, start_index, curr, best) else { return Ok(None); }; while nb_attempts > 0 { - let mut offered = longest.offer(match_index); + let mut offered = longest.offer::(match_index); if offered == Offered::Outside { if DICT_MODE != EXT_DICT || match_index < low_limit { break; @@ -931,7 +1048,7 @@ impl<'a, const DICT_MODE: u8> Finder<'a> for ChainFinder<'a, DICT_MODE> { let match_pos = (match_index - block.dict.start_index) as usize; offered = longest.offer_in_dict(&block, ip, match_pos)?; } - if offered == Offered::Longest || match_index <= min_chain { + if offered == Offered::Longest || (!SIMPLE && match_index <= min_chain) { break; } match_index = table_get(self.chains.chain_table, match_index as usize); @@ -959,6 +1076,51 @@ impl<'a, const DICT_MODE: u8> Finder<'a> for ChainFinder<'a, DICT_MODE> { Ok(longest.best().found()) } + #[inline(always)] + fn skip_literals(&mut self, run: LiteralRun) -> Result { + let near = self.block.near; + let mut ip = run.ip; + let no_repeat = false; + if !SIMPLE { + return Ok(Skipped { ip, no_repeat }); + } + while ip < run.ilimit { + let step = (ip - run.anchor) >> K_SEARCH_STRENGTH; + let bytes = near.read64(ip).ok_or(BUG)?; + if step + 1 > K_LAZY_SKIPPING_STEP + || (run.offset_1 > 0 + && near.read32((ip + 1).wrapping_sub(run.offset_1)) + == Some((bytes >> 8) as u32)) + { + break; + } + + let mut match_index = self.insert_and_find_first_index(ip, false)?; + let mut nb_attempts = self.nb_attempts; + while let Some(match_bytes) = + near.read32(match_index.wrapping_sub(self.block.base_index)) + { + if match_bytes == bytes as u32 { + self.started = Some(Candidate { + match_index, + nb_attempts, + }); + return Ok(Skipped { + ip, + no_repeat: true, + }); + } + nb_attempts -= 1; + if nb_attempts == 0 { + break; + } + match_index = table_get(self.chains.chain_table, match_index as usize); + } + ip += step + 1; + } + Ok(Skipped { ip, no_repeat }) + } + #[inline(always)] fn stop_skipping(&mut self) -> Result<()> { Ok(()) @@ -1492,7 +1654,7 @@ impl RowSearch<'_> { while candidates.matches != 0 && nb_attempts > 0 { let match_index = candidates.first(); candidates.remove_first(); - let mut offered = longest.offer(match_index); + let mut offered = longest.offer::(match_index); if offered == Offered::Outside { if DICT_MODE != EXT_DICT || match_index < low_limit { break; @@ -1847,8 +2009,27 @@ fn compress_block_lazy_generic<'a, F: Finder<'a>, const DEPTH: u32, const DICT_M let mut skipping = false; 'parse: while ip < ilimit { + let mut no_repeat = false; + if !skipping { + let skipped = finder.skip_literals(LiteralRun { + ip, + anchor, + ilimit, + offset_1, + })?; + ip = skipped.ip; + no_repeat = skipped.no_repeat; + if ip >= ilimit { + break; + } + } + // Check the repeat offset. - let mut best = Found::repeat(rep_length::(block, ip + 1, offset_1)?); + let mut best = Found::repeat(if DEPTH == 0 && no_repeat { + 0 + } else { + rep_length::(block, ip + 1, offset_1)? + }); let mut start = ip + 1; if DEPTH > 0 || best.len == 0 { @@ -1988,7 +2169,12 @@ fn compress_block_lazy_hashed( use_row: bool, ) -> Result { if !use_row { - return compress_block_lazy_generic::, DEPTH, DICT_MODE>(args); + if DICT_MODE == NO_DICT && chains_are_simple(args.ms, &args.block) { + return compress_block_lazy_generic::, DEPTH, NO_DICT>(args); + } + return compress_block_lazy_generic::, DEPTH, DICT_MODE>( + args, + ); } // With a dictionary, the second tags are given up: how many candidates there are matters if it is searched next. if DICT_MODE != NO_DICT { From c2216a51dca016c649d7b39166ae1cb9ed08f42d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:23:20 +0000 Subject: [PATCH 114/199] zstd: a match right after another one is not looked at for what is before it --- src/zstd/compress/double_fast.rs | 2 +- src/zstd/compress/fast.rs | 14 ++++++++++++-- src/zstd/compress/internal.rs | 4 ++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 958be79feebf..2820d0f6f9a9 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -578,7 +578,7 @@ impl Finder { }; let end = ip + known + window.count(ip + known, place + known); look_ahead(near, end, hash_long, long, hash_small, small); - let back = window.count_backwards(ip, sink.anchor, place); + let back = window.count_backwards_if_any(ip, sink.anchor, place); offset_2 = offset_1; offset_1 = window.place_of(ip) - place; sink.store::<16>(ip - back, end, offset_to_offbase(offset_1)); diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 9b4cfad86bb1..af98cf1d8799 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -484,6 +484,16 @@ impl<'a> Segments<'a> { len + self.near.count(ip + len, 0) } + /// [`Self::count_backwards`], with a branch on whether `ip` is `anchor`: whoever got there without a step is likely + /// to do so again. + #[inline(always)] + pub(super) fn count_backwards_if_any(self, ip: u32, anchor: u32, place: u32) -> u32 { + if ip == anchor { + return 0; + } + self.count_backwards(ip, anchor, place) + } + /// How many bytes before the position `ip`, down to `anchor`, are the same as those before `place` in its segment. #[inline(always)] pub(super) fn count_backwards(self, ip: u32, anchor: u32, place: u32) -> u32 { @@ -721,7 +731,7 @@ impl Finder { let ip = current0.wrapping_sub(base_index); rep_offset2 = rep_offset1; rep_offset1 = window.place_of(ip) - place; - let back = window.count_backwards(ip, sink.anchor, place); + let back = window.count_backwards_if_any(ip, sink.anchor, place); break (ip, place, back, offset_to_offbase(rep_offset1), next); }; @@ -964,7 +974,7 @@ impl Finder { if let Some(place) = place { let end = ip0 + 4 + window.count(ip0 + 4, place + 4); - let back = window.count_backwards(ip0, sink.anchor, place); + let back = window.count_backwards_if_any(ip0, sink.anchor, place); rep_offset2 = rep_offset1; rep_offset1 = window.place_of(ip0) - place; sink.store::<32>(ip0 - back, end, offset_to_offbase(rep_offset1)); diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 2fcc9d30cb6f..7b2ae517385c 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -948,6 +948,10 @@ impl<'a> Near<'a> { match_pos: u32, match_low: u32, ) -> u32 { + // Whoever got here without a step is likely to do so again. + if pos == anchor { + return 0; + } let most = (pos - anchor).min(match_pos - match_low); self.count_backwards_in(pos, self, match_pos, most) } From 0e01a3749feae0cd4f5669834cc61ca5009758f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:24:53 +0000 Subject: [PATCH 115/199] zstd: say what the versions of fast and dfast are for --- src/zstd/compress/double_fast.rs | 4 +++- src/zstd/compress/fast.rs | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 2820d0f6f9a9..69995cf8486e 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -172,6 +172,7 @@ impl Finder { { ip += 1; let end = ip + 4 + sink.count_before(ip + 4, offset_1); + // Small tables are close by as it is. if LONG == 0 || LONG >= 17 { look_ahead(near, end, hash_long, long, hash_small, small); } @@ -289,7 +290,7 @@ pub(crate) fn compress_block_double_fast( let mut sink = MatchSink::new(seq_store, &block); let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; - // First, what levels 3 and 4 come with if the input is not known to be small, and level 3 if it is. + // What the levels come with, for an input of any size. match (ms.c_params.min_match, h_bits_l, h_bits_s) { (5, 18, 18) => finder.find_matches::<5, 18, 18>(hash_long, hash_small, &mut sink), (5, 17, 16) => finder.find_matches::<5, 17, 16>(hash_long, hash_small, &mut sink), @@ -653,6 +654,7 @@ pub(crate) fn compress_block_double_fast_ext_dict( let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; let (long, small, sink) = (hash_long, hash_small, &mut sink); + // What the levels come with, for an input that is large enough to get here. match (ms.c_params.min_match, h_bits_l, h_bits_s) { (5, 18, 18) => finder.find_matches_ext_dict::<5, 18, 18>(long, small, sink, block.window), (5, 17, 16) => finder.find_matches_ext_dict::<5, 17, 16>(long, small, sink, block.window), diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index af98cf1d8799..0ee5d133f271 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -805,7 +805,7 @@ pub(crate) fn compress_block_fast( }; let mut sink = MatchSink::new(seq_store, &block); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; - // First, what levels 1, 2 and those below 0 come with if the input is not known to be small, and level 1 if it is. + // What the levels come with, for an input of any size. match (ms.c_params.min_match, hlog) { (7, 14) => finder.find_matches::<7, 14>(hash_table, &mut sink), (6, 16) => finder.find_matches::<6, 16>(hash_table, &mut sink), @@ -887,8 +887,8 @@ impl<'a> TaggedTable<'a> { }) } - /// The place that the table has for the `MLS` first bytes of `bytes`, as many as it was made for. `None` if the tag tells that it is for - /// something else, or if it is the first place, which stands for none. + /// The place that the table has for the `MLS` first bytes of `bytes`, as many as it was made for. `None` if the tag + /// tells that it is for something else, or if it is the first place, which stands for none. #[inline(always)] pub(super) fn get(self, bytes: u64) -> Option { let hash_and_tag = self.hasher.with_mls(MLS).hash(bytes); @@ -1081,6 +1081,7 @@ pub(crate) fn compress_block_fast_ext_dict( }; let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; + // What the levels come with, for an input that is large enough to get here. match (ms.c_params.min_match, hlog) { (7, 14) => finder.find_matches_ext_dict::<7, 14>(hash_table, &mut sink, block.window), (6, 16) => finder.find_matches_ext_dict::<6, 16>(hash_table, &mut sink, block.window), From 3d4a5e7b1f6a95df5d486125bae25e007c97e853 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:32:53 +0000 Subject: [PATCH 116/199] zstd: test for more of the bugs that zstd has had --- test/js/bun/util/zstd.test.ts | 92 +++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 678a3d779530..0f633497ca52 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1654,6 +1654,23 @@ describe("bugs that zstd has had", () => { expect(await decoded(hex("502a4d18f8ffffff"))).toEqual(["ERR_ZSTD", Buffer.alloc(0), Buffer.alloc(0)]); }); + it("what no frame starts with is refused before there are 4 bytes of it", async () => { + const after = async (first: string) => { + const stream = zlib.createZstdDecompress(); + const { promise, resolve } = Promise.withResolvers(); + stream.on("error", (error: any) => resolve(error.code)); + stream.write(hex(first), error => resolve((error as any)?.code ?? "more")); + try { + return await promise; + } finally { + stream.destroy(); + } + }; + expect(await Promise.all(["28", "28b52f", "50", "5f2a4d"].map(after))).toEqual(["more", "more", "more", "more"]); + const unknown = "ZSTD_error_prefix_unknown"; + expect(await Promise.all(["000000", "29", "28b530", "4f2a4d"].map(after))).toEqual(Array(4).fill(unknown)); + }); + it("a first block cannot repeat the tables of the frame that was decoded before it", async () => { const text = zlib.zstdCompressSync(noise(20_000, 18, 15)); for (const [frame, code] of [ @@ -1881,6 +1898,47 @@ describe("bugs that zstd has had", () => { for (const level of [1, 3, 9, 19]) roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }); }); + it("a block of literals with a match for its last bytes", () => { + const input = noise(128 << 10, 19); + input.copy(input, input.length - 16, 1000, 1016); + for (const level of [16, 19, 22]) + expect(roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }).length).toBe(131084); + }); + + it("less input than the shortest long distance match", async () => { + const params = { [c.ZSTD_c_enableLongDistanceMatching]: 1, [c.ZSTD_c_ldmMinMatch]: 4096 }; + const frames = [0, 1, 7, 63, 64, 4095].flatMap(length => + [3, 19].map(level => roundTrip(noise(length, 21, 3), { ...params, [c.ZSTD_c_compressionLevel]: level })), + ); + expect(Bun.MD5.hash(Buffer.concat(frames), "hex")).toStartWith("033266ef"); + + const input = noise(300, 22, 3); + const stream = zlib.createZstdCompress({ params }); + const { promise, resolve } = Promise.withResolvers(); + const output: Buffer[] = []; + stream.on("data", chunk => output.push(chunk)).on("end", () => resolve(Buffer.concat(output))); + for (const piece of pieces(input, 5)) { + stream.write(piece); + stream.flush(); + } + stream.end(); + const frame = await promise; + expect(`${frame.length} ${Bun.MD5.hash(frame, "hex").slice(0, 8)}`).toBe("504 1f230d95"); + expect(zlib.zstdDecompressSync(frame)).toEqual(input); + }); + + it("the least and the most that parameters can be", () => { + const taken = (parameter: number, values: number[]) => + values.filter(value => typeof attempt(() => roundTrip(hello, { [parameter]: value })) !== "string"); + expect(taken(c.ZSTD_c_compressionLevel, [-200_000, -131_073, -131_072, 22, 23, 200_000])).toBeArrayOfSize(6); + expect(taken(c.ZSTD_c_windowLog, [9, 10, 31, 32])).toEqual([10, 31]); + expect(taken(c.ZSTD_c_enableLongDistanceMatching, [0, 1, 2, 3])).toEqual([0, 1, 2]); + expect(taken(c.ZSTD_c_ldmHashLog, [5, 6, 30, 31])).toEqual([6, 30]); + expect(taken(c.ZSTD_c_ldmMinMatch, [3, 4, 4096, 4097])).toEqual([4, 4096]); + expect(taken(c.ZSTD_c_ldmBucketSizeLog, [0, 1, 8, 9])).toEqual([0, 1, 8]); + expect(taken(c.ZSTD_c_ldmHashRateLog, [0, 25, 26])).toEqual([0, 25]); + }); + describe("dictionaries", () => { // `tests/dict-files/zero-weight-dict` const zeroWeights = base64( @@ -1928,6 +1986,22 @@ describe("bugs that zstd has had", () => { } }); + it("that do not make the window of a stream of unknown size smaller", async () => { + const chunks = [Buffer.from("abc"), Buffer.from("def")]; + const start = async (options: zlib.ZstdOptions) => + ((await through(zlib.createZstdCompress(options), chunks)) as Buffer).toString("hex", 0, 6); + const dictionary = noise(1000, 20, 7); + expect(await start({ dictionary })).toBe("28b52ffd0058"); + expect(await start({ dictionary, pledgedSrcSize: 6 })).toBe("28b52ffd2006"); + }); + + it("of which the tables have no room for all", () => { + const dictionary = noise(17 << 20, 24, 15); + const params = { [c.ZSTD_c_compressionLevel]: 1 }; + expect(roundTrip(dictionary.subarray(0, 10240), params, dictionary).length).toBe(5154); + expect(roundTrip(dictionary.subarray(-10240), params, dictionary).length).toBe(19); + }); + it("that leave the window", async () => { const dictionary = noise(10240, 10, 15); const input = Buffer.alloc(10240); @@ -1986,6 +2060,24 @@ describe("bugs that zstd has had", () => { for (const size of [1 << 20, 64 << 10, 1000]) expect(await cut(size)).toBe("1398687 a871b693"); }); + it("make the same twice", async () => { + const input = noise(3 << 19, 23, 7); + for (const longDistance of [0, 1]) { + for (const level of [3, 16]) { + const params = { + [c.ZSTD_c_nbWorkers]: 2, + [c.ZSTD_c_jobSize]: 512 << 10, + [c.ZSTD_c_compressionLevel]: level, + [c.ZSTD_c_enableLongDistanceMatching]: longDistance, + }; + const made = () => through(zlib.createZstdCompress({ params }), pieces(input, 100_000)) as Promise; + const [frame, again] = await Promise.all([made(), made()]); + expect(frame.equals(again)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(input)).toBe(true); + } + } + }); + it("with nothing left for the last job", async () => { const input = noise(2 << 20, 16, 7); const params = { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 512 << 10, [c.ZSTD_c_checksumFlag]: 1 }; From 034f39d1825247f523c9119cb2c6a3b0b9752bc7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:34:12 +0000 Subject: [PATCH 117/199] zlib: the sizes of zstd contexts as they are now --- test/js/node/zlib/zlib-estimated-size-gc.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/js/node/zlib/zlib-estimated-size-gc.test.ts b/test/js/node/zlib/zlib-estimated-size-gc.test.ts index 4d8db0a355f7..36b74e8afa86 100644 --- a/test/js/node/zlib/zlib-estimated-size-gc.test.ts +++ b/test/js/node/zlib/zlib-estimated-size-gc.test.ts @@ -12,7 +12,7 @@ import zlib from "node:zlib"; // estimateShallowMemoryUsageOf(cell) == sizeof(cell) + estimated_size(). `min` // is a floor below the per-mode footprint the constructor caches (brotli encode -// 5143, brotli decode 855, zstd compress about 14000, zstd decompress about 31000). +// 5143, brotli decode 855, zstd compress about 15000, zstd decompress about 36000). function checkFootprintStableAcrossClose(create: () => any, min: number) { const engine = create(); engine.on("error", () => {}); From 8f7a87e11a57ac14e507f4f3a34eac07229172fc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:40:02 +0000 Subject: [PATCH 118/199] verify-baseline: pass a per-test timeout to the emulated bun test run The same change as #45530. Under Intel SDE the first JPEG test of simd-baseline.test.ts takes 2 to 5.7 s, against a default timeout of 5 s. --- scripts/verify-baseline.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/verify-baseline.ts b/scripts/verify-baseline.ts index 4c10c15e1485..4e88770adfbe 100644 --- a/scripts/verify-baseline.ts +++ b/scripts/verify-baseline.ts @@ -235,7 +235,10 @@ if (values["skip-emulation"]) { } else { // Phase 1: SIMD code path verification const simdTestPath = join(repoRoot, "test", "js", "bun", "jsc-stress", "fixtures", "simd-baseline.test.ts"); - await runTest("SIMD baseline tests", ["test", simdTestPath], { live: true }); + // The default per-test timeout (5 s) assumes native speed. Intel SDE runs this file hundreds of + // times slower, so the emulated run gets a per-test timeout of its own. + const emulatedTestTimeoutMs = 90_000; + await runTest("SIMD baseline tests", ["test", `--timeout=${emulatedTestTimeoutMs}`, simdTestPath], { live: true }); } // Phase 2: JIT stress fixtures (only with --jit-stress, e.g. on WebKit changes) From 813eb784a1f6f2e9e5b0e854c9ca3b1cc7f3c5f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 22:08:04 +0000 Subject: [PATCH 119/199] zstd: workers that could not be started are started for the next frame After a change to fewer workers had failed, a frame with as many as before waited for workers that were gone. --- src/zstd/compress/mt.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index a1a849bdf9fe..8003d9ce8ab9 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -530,7 +530,7 @@ impl MtCtx { if !self.all_jobs_completed { self.abandon_jobs(); } - if params.nb_workers != self.params.nb_workers { + if params.nb_workers as usize != self.nb_workers { self.resize(params.nb_workers as usize)?; } From ec24e1d1a069fca6052fd8317c3388dc790da6cc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 22:08:04 +0000 Subject: [PATCH 120/199] zstd: indices start again where the C code replaces its workspace, and after a reset that failed Where indices start decides where they get to the limit and are brought down, which lets go of a dictionary that may still be within a large window. After 3.4 GB through a context, its output could so differ from that of the C code. A context whose tables could not be allocated went on with new indices and the old content of the other tables. --- src/zstd/compress/cctx.rs | 40 ++++++++++-- src/zstd/compress/mt.rs | 2 +- src/zstd/compress/params.rs | 126 ++++++++++++++++++++++++++++++++++-- 3 files changed, 157 insertions(+), 11 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 9f998b06cafd..9ddfc11d4908 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -19,9 +19,9 @@ use super::internal::{ }; use super::literals::min_literals_to_compress; use super::params::{ - CCtxParams, CParamMode, Format, MatchStateSizes, adjust_c_params_internal, - allocate_chain_table, cdict_indices_are_tagged, compress_bound, cycle_log, - get_c_params_from_cctx_params, max_nb_seq, resolve_row_match_finder_mode, + CCtxParams, CParamMode, Format, MatchStateSizes, Workspace, WorkspaceNeeds, + adjust_c_params_internal, allocate_chain_table, cdict_indices_are_tagged, compress_bound, + cycle_log, get_c_params_from_cctx_params, max_nb_seq, resolve_row_match_finder_mode, row_match_finder_used, }; use super::superblock::compress_super_block; @@ -295,7 +295,7 @@ fn overflow_correct_if_needed(ms: &mut MatchState, params: &CCtxParams, ip: usiz let correction = ms.window.correct_overflow(cycle_log, max_dist, ip); reduce_index(ms, params, correction); ms.next_to_update = ms.next_to_update.saturating_sub(correction); - // The dictionary is out of reach by now. + // Even if the dictionary is still within the window. ms.loaded_dict_end = 0; ms.has_dict_match_state = false; } @@ -810,6 +810,8 @@ pub(crate) struct FrameCompressor { pub(crate) applied_params: CCtxParams, pub(crate) dict_id: u32, pub(crate) dict_content_size: usize, + /// `None` for a worker, see [`Self::for_worker`]. + workspace: Option, /// For how many frames in a row the tables have been much larger than it takes. oversized_duration: usize, pub(crate) block_size_max: usize, @@ -857,6 +859,7 @@ impl FrameCompressor { applied_params: CCtxParams::new(0), dict_id: 0, dict_content_size: 0, + workspace: Some(Workspace::new()), oversized_duration: 0, block_size_max: 0, pledged_src_size_plus_one: 0, @@ -875,6 +878,15 @@ impl FrameCompressor { } } + /// For the jobs of one worker. Which context a job goes to is a matter of timing in the C code, so what its contexts + /// have been through cannot show in the output. + pub(crate) fn for_worker() -> Self { + Self { + workspace: None, + ..Self::new() + } + } + /// `ZSTD_resetCCtx_internal` fn reset_internal( &mut self, @@ -884,6 +896,8 @@ impl FrameCompressor { crp: CompResetPolicy, buffered: bool, ) -> Result { + // Until this is through: if it fails, the next frame starts from nothing. + let initialized = core::mem::take(&mut self.initialized); self.is_first_block = true; self.applied_params = *params; if self.applied_params.ldm_params.enable_ldm == ParamSwitch::Enable { @@ -912,8 +926,22 @@ impl FrameCompressor { let index_too_close = index_too_close_to_max(&self.match_state.window); let dict_too_big = loaded_dict_size > ZSTD_CHUNKSIZE_MAX as usize; - // Whether indices go on or start again does not show: what the tables hold from before is too low to be valid. - let needs_index_reset = if index_too_close || dict_too_big || !self.initialized { + // Where indices start shows in where they get to the limit, which is where a dictionary is let go of. + let new_workspace = self.workspace.as_mut().is_some_and(|workspace| { + if !initialized { + *workspace = Workspace::new(); + } + workspace.reset(&WorkspaceNeeds { + c_params: ¶ms.c_params, + ldm_params: ¶ms.ldm_params, + use_row_match_finder: params.use_row_match_finder, + buff_in_size: sizes.buff_in_size, + buff_out_size: sizes.buff_out_size, + block_size, + }) + }); + let needs_index_reset = if index_too_close || dict_too_big || !initialized || new_workspace + { IndexResetPolicy::Reset } else { IndexResetPolicy::Continue diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 8003d9ce8ab9..9f2ce5fa0499 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -299,7 +299,7 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - fn worker_main(jobs: &Receiver, cancelled: &AtomicBool) { let mut state = WorkerState { - cctx: FrameCompressor::new(), + cctx: FrameCompressor::for_worker(), seq: Vec::new(), dst: Vec::new(), }; diff --git a/src/zstd/compress/params.rs b/src/zstd/compress/params.rs index 9c091338d1ad..e003c587677b 100644 --- a/src/zstd/compress/params.rs +++ b/src/zstd/compress/params.rs @@ -3,13 +3,15 @@ use core::ops::RangeInclusive; use super::internal::{ - CompressionParameters, LdmParams, ParamSwitch, Strategy, ZSTD_SHORT_CACHE_TAG_BITS, + CompressionParameters, LdmParams, ParamSwitch, Strategy, ZSTD_OPT_SIZE, + ZSTD_SHORT_CACHE_TAG_BITS, }; use super::lazy::ZSTD_ROW_HASH_TAG_BITS; -use super::row_match_finder_supported; +use super::{ldm, row_match_finder_supported}; use crate::common::{ - ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_WINDOWLOG_ABSOLUTEMIN, - ZSTD_WINDOWLOG_LIMIT_DEFAULT, ZSTD_WINDOWLOG_MAX, + LITBITS, MAX_LL, MAX_ML, MAX_OFF, WILDCOPY_OVERLENGTH, ZSTD_BLOCKSIZE_MAX, + ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WINDOWLOG_LIMIT_DEFAULT, + ZSTD_WINDOWLOG_MAX, ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, }; use crate::error::{ErrorCode, Result}; use crate::util::highbit32; @@ -620,6 +622,25 @@ pub const fn compress_bound(src_size: usize) -> usize { src_size + (src_size >> 8) + margin } +// ── the size of the workspace ──────────────────────────────────────────────── +// +// The C code carves all the tables and buffers of a context out of one allocation. A context only goes on counting +// indexes from where the previous frame left them as long as that allocation is kept, so this keeps its books. + +const SIZEOF_COMPRESSED_BLOCK_STATE: usize = 5632; +const TMP_WORKSPACE_SIZE: usize = 8920; +const SIZEOF_SEQ_DEF: usize = 8; +const SIZEOF_MATCH: usize = 8; +const SIZEOF_OPTIMAL: usize = 28; +const SIZEOF_RAW_SEQ: usize = 12; +const SIZEOF_LDM_ENTRY: usize = 8; +const ZSTD_CWKSP_ALIGNMENT_BYTES: usize = 64; + +/// `ZSTD_cwksp_aligned64_alloc_size` +const fn aligned64(size: usize) -> usize { + size.next_multiple_of(ZSTD_CWKSP_ALIGNMENT_BYTES) +} + /// `ZSTD_maxNbSeq` pub(crate) const fn max_nb_seq(block_size: usize, min_match: u32) -> usize { block_size / if min_match == 3 { 3 } else { 4 } @@ -657,6 +678,103 @@ impl MatchStateSizes { } } +/// The books of a `ZSTD_cwksp`, as offsets from its start, which is taken to be aligned. +pub(crate) struct Workspace { + size: usize, + object_end: usize, + table_end: usize, + alloc_start: usize, + oversized_duration: usize, +} + +/// What a context needs room for. +pub(crate) struct WorkspaceNeeds<'a> { + pub(crate) c_params: &'a CompressionParameters, + pub(crate) ldm_params: &'a LdmParams, + pub(crate) use_row_match_finder: ParamSwitch, + pub(crate) buff_in_size: usize, + pub(crate) buff_out_size: usize, + pub(crate) block_size: usize, +} + +impl WorkspaceNeeds<'_> { + /// The tables, which the C code allocates from the bottom. + fn tables(&self) -> usize { + let sizes = MatchStateSizes::new(self.c_params, self.use_row_match_finder, true); + (sizes.chain_size + sizes.h_size + sizes.h3_size) * size_of::() + } + + /// Everything else, which the C code allocates from the top. + fn allocs(&self) -> usize { + let max_nb_seq = max_nb_seq(self.block_size, self.c_params.min_match); + let mut total = 0; + if row_match_finder_used(self.c_params.strat(), self.use_row_match_finder) { + total += aligned64(1 << self.c_params.hash_log); + } + if self.c_params.strat() >= Strategy::BtOpt { + total += aligned64((MAX_ML as usize + 1) * size_of::()) + + aligned64((MAX_LL as usize + 1) * size_of::()) + + aligned64((MAX_OFF as usize + 1) * size_of::()) + + aligned64((1 << LITBITS) * size_of::()) + + aligned64(ZSTD_OPT_SIZE * SIZEOF_MATCH) + + aligned64(ZSTD_OPT_SIZE * SIZEOF_OPTIMAL); + } + total += aligned64(max_nb_seq * SIZEOF_SEQ_DEF); + if self.ldm_params.enable_ldm == ParamSwitch::Enable { + total += aligned64((1usize << self.ldm_params.hash_log) * SIZEOF_LDM_ENTRY) + + aligned64( + ldm::get_max_nb_seq(*self.ldm_params, self.block_size) * SIZEOF_RAW_SEQ, + ) + + (1usize << (self.ldm_params.hash_log - self.ldm_params.bucket_size_log)); + } + total + + (self.block_size + WILDCOPY_OVERLENGTH) + + self.buff_in_size + + self.buff_out_size + + 3 * max_nb_seq + } + + /// `ZSTD_estimateCCtxSize_usingCCtxParams_internal` + fn needed_space(&self) -> usize { + let objects = TMP_WORKSPACE_SIZE + 2 * SIZEOF_COMPRESSED_BLOCK_STATE; + let slack_space = ZSTD_CWKSP_ALIGNMENT_BYTES * 2; + objects + self.tables() + self.allocs() + slack_space + } +} + +impl Workspace { + pub(crate) const fn new() -> Self { + Self { + size: 0, + object_end: 0, + table_end: 0, + alloc_start: 0, + oversized_duration: 0, + } + } + + /// What `ZSTD_resetCCtx_internal` does to the workspace. Returns whether it made a new one. + pub(crate) fn reset(&mut self, needs: &WorkspaceNeeds<'_>) -> bool { + let needed_space = needs.needed_space(); + self.oversized_duration += 1; + let too_small = self.size < needed_space; + let wasteful = self.alloc_start - self.table_end + >= needed_space.saturating_mul(ZSTD_WORKSPACETOOLARGE_FACTOR) + && self.oversized_duration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; + let resize = too_small || wasteful; + if resize { + self.size = needed_space; + self.object_end = aligned64(TMP_WORKSPACE_SIZE + 2 * SIZEOF_COMPRESSED_BLOCK_STATE); + self.oversized_duration = 0; + } + self.table_end = self.object_end + needs.tables(); + self.alloc_start = + (self.size - self.size % ZSTD_CWKSP_ALIGNMENT_BYTES).saturating_sub(needs.allocs()); + self.alloc_start = self.alloc_start.max(self.table_end); + resize + } +} + const fn cp( window_log: u32, chain_log: u32, From 6579103d925fece7aec96b2c546a8cc7fc81cdc6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 22:15:38 +0000 Subject: [PATCH 121/199] zstd: a unit test for the size of the table of long distance matches The table is not touched where it is not used, so how much memory a process has had says little about its size. --- scripts/rust-miri.ts | 1 + src/zstd/compress/ldm.rs | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/scripts/rust-miri.ts b/scripts/rust-miri.ts index e1befb7d144e..1ce61c22fc78 100644 --- a/scripts/rust-miri.ts +++ b/scripts/rust-miri.ts @@ -56,6 +56,7 @@ const MIRI_CRATES = [ "bun_threading", "bun_url", "bun_wyhash", + "bun_zstd", ]; function run(cmd: string, args: string[], opts: Parameters[2] = {}) { diff --git a/src/zstd/compress/ldm.rs b/src/zstd/compress/ldm.rs index f0b3e89c4f15..1b1cb9b0d3a8 100644 --- a/src/zstd/compress/ldm.rs +++ b/src/zstd/compress/ldm.rs @@ -734,3 +734,31 @@ pub(crate) fn block_compress( dict_mode, ) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_hash_rate_above_the_window_makes_for_the_smallest_table() { + let c_params = CompressionParameters { + window_log: 10, + chain_log: 10, + hash_log: 10, + search_log: 1, + min_match: 4, + target_length: 0, + strategy: Some(Strategy::Fast), + }; + let mut params = LdmParams { + enable_ldm: ParamSwitch::Enable, + hash_log: 0, + bucket_size_log: 0, + min_match_length: 0, + hash_rate_log: 11, + window_log: 0, + }; + adjust_parameters(&mut params, &c_params); + assert_eq!(params.hash_log, ZSTD_HASHLOG_MIN as u32); + } +} From 7e0e64bdc1b914088d36ff173fe04b8548f04b2c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 22:15:38 +0000 Subject: [PATCH 122/199] zstd: streams in tests report their errors --- test/js/bun/util/zstd.test.ts | 39 +++++++++++------------------------ 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 0f633497ca52..42ccb0d1d879 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1509,13 +1509,17 @@ describe("bugs that zstd has had", () => { } return bytes; } - function through(stream: zlib.ZstdCompress | zlib.ZstdDecompress, chunks: Iterable) { + function through(stream: zlib.ZstdCompress | zlib.ZstdDecompress, chunks: Iterable, flush = false) { const { promise, resolve } = Promise.withResolvers(); const output: Buffer[] = []; stream.on("data", chunk => output.push(chunk)); stream.on("error", (error: any) => resolve(error.code)); stream.on("end", () => resolve(Buffer.concat(output))); - for (const chunk of chunks) stream.write(chunk); + stream.on("close", () => resolve("closed")); + for (const chunk of chunks) { + stream.write(chunk); + if (flush) stream.flush(); + } stream.end(); return promise; } @@ -1717,8 +1721,8 @@ describe("bugs that zstd has had", () => { const content = noise(100_000, 2); const frame = zlib.zstdCompressSync(content); const stream = zlib.createZstdDecompress(); - const { promise, resolve } = Promise.withResolvers(); - stream.once("data", chunk => resolve(chunk.length)); + const { promise, resolve, reject } = Promise.withResolvers(); + stream.once("data", chunk => resolve(chunk.length)).on("error", reject); stream.write(frame.subarray(0, 5000), () => stream.flush()); expect(await promise).toBeWithin(4000, 5000); stream.destroy(); @@ -1878,19 +1882,9 @@ describe("bugs that zstd has had", () => { pledgedSrcSize: input.length, params: { [c.ZSTD_c_compressionLevel]: 22 }, }); - const { promise, resolve } = Promise.withResolvers(); - const output: Buffer[] = []; - stream.on("data", chunk => output.push(chunk)).on("end", () => resolve(Buffer.concat(output))); - for (const [from, to] of [ - [0, 4], - [4, 9], - [9, 20], - ]) { - stream.write(input.subarray(from, to)); - stream.flush(); - } - stream.end(input.subarray(20)); - expect(zlib.zstdDecompressSync(await promise)).toEqual(input); + const cuts = [0, 4, 9, 20, input.length]; + const chunks = cuts.slice(1).map((to, i) => input.subarray(cuts[i], to)); + expect(zlib.zstdDecompressSync((await through(stream, chunks, true)) as Buffer)).toEqual(input); }); it("raw blocks between compressed ones", () => { @@ -1913,16 +1907,7 @@ describe("bugs that zstd has had", () => { expect(Bun.MD5.hash(Buffer.concat(frames), "hex")).toStartWith("033266ef"); const input = noise(300, 22, 3); - const stream = zlib.createZstdCompress({ params }); - const { promise, resolve } = Promise.withResolvers(); - const output: Buffer[] = []; - stream.on("data", chunk => output.push(chunk)).on("end", () => resolve(Buffer.concat(output))); - for (const piece of pieces(input, 5)) { - stream.write(piece); - stream.flush(); - } - stream.end(); - const frame = await promise; + const frame = (await through(zlib.createZstdCompress({ params }), pieces(input, 5), true)) as Buffer; expect(`${frame.length} ${Bun.MD5.hash(frame, "hex").slice(0, 8)}`).toBe("504 1f230d95"); expect(zlib.zstdDecompressSync(frame)).toEqual(input); }); From 0c8383f13dc7da305647571240d6dc576d55575e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 22:44:23 +0000 Subject: [PATCH 123/199] zstd: fast and dfast with an attached dictionary branch on the tag alone, and count in the segment they found the match in --- src/zstd/compress/double_fast.rs | 49 +++++++------- src/zstd/compress/fast.rs | 109 +++++++++++++++++++++++-------- 2 files changed, 105 insertions(+), 53 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 69995cf8486e..fae0f1a607d5 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -1,7 +1,7 @@ //! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. use super::fast::{ - DictBlock, ExtDictBlock, MatchSink, NoDictBlock, Segments, TaggedTable, ThirdPositions, + DictBlock, ExtDictBlock, Found, MatchSink, NoDictBlock, Segments, TaggedTable, ThirdPositions, put_tagged, put_tagged_in_between, restore_rep, }; use super::internal::{ @@ -352,33 +352,30 @@ impl Finder { hashed_swap(hash_long, long.hash(bytes0), curr).wrapping_sub(base_index); let match_index_s = hashed_swap(hash_small, small.hash(bytes0), curr); - if window.read32_before(ip + 1, offset_1) == Some((bytes0 >> 8) as u32) { - ip += 1; - let rep_place = window.place_of(ip).wrapping_sub(offset_1); - let end = ip + 4 + window.count(ip + 4, rep_place + 4); - sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); + if let Some(end) = window.repeat_end(ip + 1, offset_1, (bytes0 >> 8) as u32) { + sink.store::<16>(ip + 1, end, REPCODE1_TO_OFFBASE); ip = end; } else { - // The place of the match, and how much of it is known. `ip` is moved to where it is for. - let (place, known) = 'found: { + // `ip` is moved to where the match is for. + let found = 'found: { if near.has::<{ u64::MAX }>(match_long, bytes0) { - break 'found (window.place_of(match_long), 8); + break 'found window.found_near(ip, sink.anchor, match_long, 8); } if let Some(place) = dict_tables.long.get::<8>(bytes0) && window.older.read64(place) == Some(bytes0) { - break 'found (place, 8); + break 'found window.found_older(ip, sink.anchor, place, 8); } // The dictionary is only looked up if there is no candidate in the prefix. let short = if match_index_s > base_index { - Some(window.place_of(match_index_s - base_index)) + let pos = match_index_s - base_index; + (near.read32(pos) == Some(bytes0 as u32)).then(|| window.place_of(pos)) } else { - dict_tables.short.get::(bytes0) + (dict_tables.short.get::(bytes0)) + .filter(|&place| window.older.read32(place) == Some(bytes0 as u32)) }; - let Some(place_short) = - short.filter(|&place| window.read32(place) == Some(bytes0 as u32)) - else { + let Some(place_short) = short else { ip += ((ip - sink.anchor) >> K_SEARCH_STRENGTH) + 1; continue 'search; }; @@ -389,22 +386,24 @@ impl Finder { .wrapping_sub(base_index); if near.has::<{ u64::MAX }>(match_long1, bytes1) { ip += 1; - break 'found (window.place_of(match_long1), 8); + break 'found window.found_near(ip, sink.anchor, match_long1, 8); } if let Some(place) = dict_tables.long.get::<8>(bytes1) && window.older.read64(place) == Some(bytes1) { ip += 1; - break 'found (place, 8); + break 'found window.found_older(ip, sink.anchor, place, 8); + } + Found { + end: ip + 4 + window.count(ip + 4, place_short + 4), + back: window.count_backwards(ip, sink.anchor, place_short), + offset: window.place_of(ip) - place_short, } - (place_short, 4) }; - let end = ip + known + window.count(ip + known, place + known); - let back = window.count_backwards(ip, sink.anchor, place); offset_2 = offset_1; - offset_1 = window.place_of(ip) - place; - sink.store::<16>(ip - back, end, offset_to_offbase(offset_1)); - ip = end; + offset_1 = found.offset; + sink.store::<16>(ip - found.back, found.end, offset_to_offbase(offset_1)); + ip = found.end; } // `ip` is not past the last position that can be looked at. @@ -428,10 +427,8 @@ impl Finder { ); while let Some(bytes) = near.read64(ip) - && window.read32_before(ip, offset_2) == Some(bytes as u32) + && let Some(end) = window.repeat_end(ip, offset_2, bytes as u32) { - let rep_place = window.place_of(ip).wrapping_sub(offset_2); - let end = ip + 4 + window.count(ip + 4, rep_place + 4); core::mem::swap(&mut offset_1, &mut offset_2); let index = base_index.wrapping_add(ip); hashed_put(hash_small, small.hash(bytes), index); diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 0ee5d133f271..90541cb6fd8a 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -1,5 +1,7 @@ //! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. +use core::sync::atomic::{Ordering, compiler_fence}; + use super::internal::{ BlockSrc, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, Hasher, K_SEARCH_STRENGTH, MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, @@ -447,13 +449,6 @@ impl<'a> Segments<'a> { spot.segment.read8(spot.pos) } - /// The 4 bytes at `place`, if they are in one segment. - #[inline(always)] - pub(super) fn read32(self, place: u32) -> Option { - let spot = self.spot(place); - spot.segment.read32(spot.pos) - } - /// Whether the bits of `MASK` are the same in `bytes` and in the 8 bytes at `place`, which are in one segment, as they /// are wherever a table leads to. #[inline(always)] @@ -484,6 +479,61 @@ impl<'a> Segments<'a> { len + self.near.count(ip + len, 0) } + /// [`Self::count`] for a place of the older segment. + #[inline(always)] + fn count_in_older(self, ip: u32, place: u32) -> u32 { + let len = self.near.count_in(ip, self.older, place); + if place + len != self.older.len() { + return len; + } + len + self.near.count(ip + len, 0) + } + + /// Where the match of the position `ip` with what is `offset` before it ends, if there is one: `bytes` are the 4 + /// bytes at `ip`, and those before have to be in one segment. + #[inline(always)] + pub(super) fn repeat_end(self, ip: u32, offset: u32, bytes: u32) -> Option { + let at = ip.wrapping_sub(offset); + let len = match self.near.read32(at) { + Some(there) if there == bytes => self.near.count(ip + 4, at + 4), + Some(_) => return None, + None => { + let place = self.place_of(at); + if self.older.read32(place) != Some(bytes) { + return None; + } + self.count_in_older(ip + 4, place + 4) + } + }; + Some(ip + 4 + len) + } + + /// The match at the position `ip` with what is at `pos` of the current segment, of which `known` bytes are known. + #[inline(always)] + pub(super) fn found_near(self, ip: u32, anchor: u32, pos: u32, known: u32) -> Found { + Found { + end: ip + known + self.near.count(ip + known, pos + known), + back: self.near.count_backwards(ip, anchor, pos, 0), + offset: ip - pos, + } + } + + /// The same with what is at `place`, which is in the older segment. + #[inline(always)] + pub(super) fn found_older(self, ip: u32, anchor: u32, place: u32, known: u32) -> Found { + let len = known + self.count_in_older(ip + known, place + known); + let back = if ip == anchor { + 0 + } else { + (self.near).count_backwards_in(ip, self.older, place, (ip - anchor).min(place)) + }; + Found { + end: ip + len, + back, + offset: self.place_of(ip) - place, + } + } + /// [`Self::count_backwards`], with a branch on whether `ip` is `anchor`: whoever got there without a step is likely /// to do so again. #[inline(always)] @@ -502,6 +552,14 @@ impl<'a> Segments<'a> { } } +/// A match at a position. +pub(super) struct Found { + pub(super) end: u32, + /// How many bytes before the position belong to it as well. + pub(super) back: u32, + pub(super) offset: u32, +} + /// What `fast` looks for matches with, when there is no dictionary and the window is one segment. struct Finder { /// The index of the first position. @@ -893,9 +951,13 @@ impl<'a> TaggedTable<'a> { pub(super) fn get(self, bytes: u64) -> Option { let hash_and_tag = self.hasher.with_mls(MLS).hash(bytes); let index_and_tag = hashed_get(self.table, hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS); + if !compare_packed_tags(index_and_tag as usize, hash_and_tag) { + return None; + } + // Costs nothing, and keeps the compiler from asking both questions before it goes on, as it mostly does here. + compiler_fence(Ordering::SeqCst); let index = index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; - (compare_packed_tags(index_and_tag as usize, hash_and_tag) && index > self.start_index) - .then(|| index - self.start_index) + (index > self.start_index).then(|| index - self.start_index) } } @@ -933,19 +995,16 @@ impl Finder { let mut next_step = ip0 + K_STEP_INCR; let mut curr; - // One turn per position. Ends with the place of a match, or with `None` once a repeat is stored. - let place = loop { + // One turn per position. Ends with a match, or with `None` once a repeat is stored. + let found = loop { let Some(bytes1) = near.read64(ip1) else { break 'block; }; curr = base_index.wrapping_add(ip0); let match_index = hashed_swap(hash_table, hasher.hash(bytes0), curr); - if window.read32_before(ip0 + 1, rep_offset1) == Some((bytes0 >> 8) as u32) { - ip0 += 1; - let rep_place = window.place_of(ip0).wrapping_sub(rep_offset1); - let end = ip0 + 4 + window.count(ip0 + 4, rep_place + 4); - sink.store::<32>(ip0, end, REPCODE1_TO_OFFBASE); + if let Some(end) = window.repeat_end(ip0 + 1, rep_offset1, (bytes0 >> 8) as u32) { + sink.store::<32>(ip0 + 1, end, REPCODE1_TO_OFFBASE); ip0 = end; break None; } @@ -955,11 +1014,11 @@ impl Finder { && window.older.read32(place) == Some(bytes0 as u32) && match_index <= base_index { - break Some(place); + break Some(window.found_older(ip0, sink.anchor, place, 4)); } let match_pos = match_index.wrapping_sub(base_index); if near.has::<0xFFFF_FFFF>(match_pos, bytes0) { - break Some(window.place_of(match_pos)); + break Some(window.found_near(ip0, sink.anchor, match_pos, 4)); } if ip1 >= next_step { @@ -972,13 +1031,11 @@ impl Finder { bytes0 = bytes1; }; - if let Some(place) = place { - let end = ip0 + 4 + window.count(ip0 + 4, place + 4); - let back = window.count_backwards_if_any(ip0, sink.anchor, place); + if let Some(found) = found { rep_offset2 = rep_offset1; - rep_offset1 = window.place_of(ip0) - place; - sink.store::<32>(ip0 - back, end, offset_to_offbase(rep_offset1)); - ip0 = end; + rep_offset1 = found.offset; + sink.store::<32>(ip0 - found.back, found.end, offset_to_offbase(rep_offset1)); + ip0 = found.end; } // `ip0` is not past the last position that can be looked at. @@ -995,9 +1052,7 @@ impl Finder { base_index.wrapping_add(ip0 - 2), ); - while window.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) { - let rep_place = window.place_of(ip0).wrapping_sub(rep_offset2); - let end = ip0 + 4 + window.count(ip0 + 4, rep_place + 4); + while let Some(end) = window.repeat_end(ip0, rep_offset2, bytes0 as u32) { core::mem::swap(&mut rep_offset1, &mut rep_offset2); hashed_put( hash_table, From 7dc6d2926c58a2ca33a27b9495ee2e588274840a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 23:11:47 +0000 Subject: [PATCH 124/199] zstd: streams in tests report that they were closed, too --- test/js/bun/util/zstd.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 42ccb0d1d879..ffbfb50712ac 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1318,6 +1318,7 @@ describe("the output is that of zstd 1.5.7", () => { stream.on("data", chunk => chunks.push(chunk)); const { promise, resolve, reject } = Promise.withResolvers(); stream.on("end", resolve).on("error", reject); + stream.on("close", () => reject(new Error("closed before the end"))); for (let at = 0, n = 1; at < corpus.length; at += n, n = (n * 7) % 90_001) { stream.write(corpus.subarray(at, at + n)); if (flushes && n % 3 === 0) stream.flush(); @@ -1365,6 +1366,7 @@ describe("the output is that of zstd 1.5.7", () => { stream.on("data", chunk => chunks.push(chunk)); const { promise, resolve, reject } = Promise.withResolvers(); stream.on("end", resolve).on("error", reject); + stream.on("close", () => reject(new Error("closed before the end"))); let at = 0; for (const size of sizes) { stream.write(corpus.subarray(at, at + size)); @@ -1477,7 +1479,7 @@ describe("bugs of zstd 1.5.7", () => { params: { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 1 << 20 }, }); const { promise, resolve } = Promise.withResolvers(); - stream.on("error", resolve).on("end", resolve).resume(); + stream.on("error", resolve).on("end", resolve).on("close", resolve).resume(); stream.write(input.subarray(0, 1000)); stream.end(input.subarray(1000)); expect(await promise).toMatchObject({ code: "ZSTD_error_srcSize_wrong" }); @@ -1723,6 +1725,7 @@ describe("bugs that zstd has had", () => { const stream = zlib.createZstdDecompress(); const { promise, resolve, reject } = Promise.withResolvers(); stream.once("data", chunk => resolve(chunk.length)).on("error", reject); + stream.on("close", () => reject(new Error("closed before any data"))); stream.write(frame.subarray(0, 5000), () => stream.flush()); expect(await promise).toBeWithin(4000, 5000); stream.destroy(); From a9fef23d8f681c56bbd977a9599d3256e850185b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 22:30:33 +0000 Subject: [PATCH 125/199] zstd: an output that is allocated for its frames has room to copy in chunks up to its end, and a decoder clears what it used of its own --- src/zstd/decompress/block.rs | 23 ++++++++++++++++++----- src/zstd/lib.rs | 5 ++++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 6d90594080b4..007890e5c495 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -232,6 +232,8 @@ struct Scratch { batch: [RawSequence; 3 + 128], /// Where a block that is all of a small output is decoded: with room to copy in chunks up to its end. small_output: [u8; Self::MAX_SMALL_OUTPUT + WILDCOPY_OVERLENGTH], + /// All of `small_output` from here on is zeros. + small_output_used: usize, } thread_local! { @@ -263,7 +265,10 @@ impl Drop for BlockDecoder { fn drop(&mut self) { if let Some(mut scratch) = self.scratch.take() { scratch.lit_buffer.clear(); - scratch.small_output.fill(0); + let used = core::mem::take(&mut scratch.small_output_used); + if let Some(used) = scratch.small_output.get_mut(..used) { + used.fill(0); + } if scratch.lit_buffer.capacity() > Scratch::MAX_SPARE_LIT_BUFFER { scratch.lit_buffer = Vec::new(); } @@ -297,6 +302,7 @@ impl BlockDecoder { lit_buffer: Vec::new(), batch: [RawSequence::EMPTY; _], small_output: [0; _], + small_output_used: 0, }) }) }) @@ -612,6 +618,7 @@ impl BlockDecoder { lit_buffer, batch, small_output, + small_output_used, } = &mut **scratch; let dict_tables = dict_entropy.map(|dict| &dict.seq); let is_history_long = total_history_size > (1 << 24); @@ -648,15 +655,21 @@ impl BlockDecoder { } else { literals.size }; + // A block that has that room where it goes needs no other place. let is_small = start == prefix_start && capacity <= Scratch::MAX_SMALL_OUTPUT + && capacity < block_size_max + WILDCOPY_OVERLENGTH && literals.location == LitLocation::NotInDst && matches!(ext_dict, ExtDict::Dict(_)); let (out, copy_to) = match buf { - OutputBuffer::Whole(bytes) if is_small => ( - OutputBuffer::Whole(small_output.sub_mut(..capacity + WILDCOPY_OVERLENGTH)?), - Some(bytes.span_mut(start, capacity)?), - ), + OutputBuffer::Whole(bytes) if is_small => { + let used = capacity + WILDCOPY_OVERLENGTH; + *small_output_used = used.max(*small_output_used); + ( + OutputBuffer::Whole(small_output.sub_mut(..used)?), + Some(bytes.span_mut(start, capacity)?), + ) + } OutputBuffer::Whole(bytes) => (OutputBuffer::Whole(bytes.sub_mut(..oend)?), None), used => (used, None), }; diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 530eeab2aea0..51f7e8b397ea 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -153,6 +153,9 @@ pub fn compress_alloc(src: &[u8], level: i32) -> Result, ErrorCode> { Ok(out) } +/// With that much room after its output, a decoder copies in chunks up to the end. +const OUTPUT_SLACK: usize = common::WILDCOPY_OVERLENGTH; + /// `len` zeros for a decoder to write to. fn output_buffer(len: usize) -> Result, ZstdError> { util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) @@ -190,7 +193,7 @@ pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { } // Fast path: size is known and within reasonable limits - let mut output = output_buffer(size as usize)?; + let mut output = output_buffer(size as usize + OUTPUT_SLACK)?; let written = decompress(&mut output, src) .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; output.truncate(written); From aaa12363f128209d482e66d0c8826951850b0c2f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 22:53:48 +0000 Subject: [PATCH 126/199] zstd: a branch on additional bits is worth it up to one length in seven --- src/zstd/decompress/seq_table.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs index cfdb803b89e0..cd103ce656a9 100644 --- a/src/zstd/decompress/seq_table.rs +++ b/src/zstd/decompress/seq_table.rs @@ -543,9 +543,9 @@ pub(crate) struct SeqTableRefs<'a> { } impl SeqTableRefs<'_> { - /// Often enough for a branch on that to cost more than it saves: one that goes wrong costs about 10 times that. + /// Often enough for a branch on that to cost more than it saves: one that goes wrong costs many times that. pub(crate) fn lengths_often_have_more_bits(&self) -> bool { - self.ll.with_more_bits + self.ml.with_more_bits > 512 / 10 + self.ll.with_more_bits + self.ml.with_more_bits > 512 / 7 } pub(crate) fn matches_are_long(&self) -> bool { From 15a9f11c2a688b22daa45b47142af7ed7e6304dc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 22:39:01 +0000 Subject: [PATCH 127/199] zstd: the copy of the start of a stream is made where it stays --- src/zstd/common/entropy.rs | 8 ++++--- src/zstd/decompress/block.rs | 45 +++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/zstd/common/entropy.rs b/src/zstd/common/entropy.rs index 433c5ec890ae..82ad6b5ee52c 100644 --- a/src/zstd/common/entropy.rs +++ b/src/zstd/common/entropy.rs @@ -3,7 +3,7 @@ use super::fse_decompress::{Spread, fse_decompress_weights}; use super::{FSE_MIN_TABLELOG, FSE_TABLELOG_ABSOLUTE_MAX, HUF_TABLELOG_MAX}; use crate::error::{ErrorCode, Result}; -use crate::util::{SliceExt, highbit32, read_le32}; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; /// What [`read_ncount`] found besides the counts. pub(crate) struct NCount { @@ -331,11 +331,13 @@ pub(crate) fn huf_read_stats_wksp( *rank = counts0.at(weight)? + counts1.at(weight)? + counts2.at(weight)? + counts3.at(weight)?; } - let (valid, too_large) = ranks.split_at(rank_stats.len()); + let Some((valid, too_large)) = ranks.split_first_chunk() else { + return Err(BUG); + }; if all_bits & 0xF0F0_F0F0 != 0 || too_large.iter().any(|&rank| rank != 0) { return Err(ErrorCode::CorruptionDetected); } - rank_stats.copy_from_slice(valid); + *rank_stats = *valid; // Those that complete the last four. if let Some(rank) = rank_stats.first_mut() { *rank -= (4 - others.len()) as u32; diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 007890e5c495..eb0c52d7fdd6 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -5,7 +5,7 @@ use super::seq_table::{ Cell, Codes, LL_CODES, ML_CODES, OF_CODES, OffsetTable, SeqTable, SeqTableRefs, SeqTableSources, SeqTables, TableSource, }; -use core::cell::{self, OnceCell}; +use core::cell; use core::hint::{Locality, prefetch_read}; use core::ops::Range; @@ -266,7 +266,9 @@ impl Drop for BlockDecoder { if let Some(mut scratch) = self.scratch.take() { scratch.lit_buffer.clear(); let used = core::mem::take(&mut scratch.small_output_used); - if let Some(used) = scratch.small_output.get_mut(..used) { + if used != 0 + && let Some(used) = scratch.small_output.get_mut(..used) + { used.fill(0); } if scratch.lit_buffer.capacity() > Scratch::MAX_SPARE_LIT_BUFFER { @@ -828,33 +830,31 @@ impl History { /// A copy of the start of a stream after zeroes, where it can be read as if its start were far. That it was not shows /// afterwards. -struct PaddedStart(OnceCell<[u8; Self::PADDING + Self::MAX_SIZE]>); +struct PaddedStart([u8; Self::PADDING + Self::MAX_SIZE]); impl PaddedStart { const PADDING: usize = 32; const MAX_SIZE: usize = 224; const fn new() -> Self { - Self(OnceCell::new()) + Self([0; _]) + } + + /// Whether little enough is left of `bit_d`. + fn fits(bit_d: &BitDStream<'_>) -> bool { + bit_d.bytes().len() <= Self::MAX_SIZE && bit_d.bits_consumed <= 64 } - /// `bit_d` on the copy, if little enough is left of it. - fn set(&self, bit_d: &BitDStream<'_>) -> Option> { + /// `bit_d` on the copy, which is new. It fits. + fn set(&mut self, bit_d: &BitDStream<'_>) -> Option> { let src = bit_d.bytes(); - if src.len() > Self::MAX_SIZE || bit_d.bits_consumed > 64 { - return None; - } - let copy = self.0.get_or_init(|| { - let mut copy = [0; _]; - if let Some(start) = copy.get_mut(Self::PADDING..Self::PADDING + src.len()) { - start.copy_from_slice(src); - } - copy - }); + self.0 + .get_mut(Self::PADDING..Self::PADDING + src.len())? + .copy_from_slice(src); // A stream of less than 8 bytes counts what it lacks as consumed. let lacking = 8usize.saturating_sub(src.len()) as u32 * 8; let pos = Self::PADDING + src.len() - 8; - BitDStream::resume(copy, pos, bit_d.bits_consumed.checked_sub(lacking)?).ok() + BitDStream::resume(&self.0, pos, bit_d.bits_consumed.checked_sub(lacking)?).ok() } /// How many bits are left to read of the copy and of the zeroes. @@ -1630,7 +1630,9 @@ impl<'a> Executor<'_, 'a> { // executes the last ones. let mut ahead = [Sequence::default(); 8]; let ahead = ahead.sub_mut(..if LONG { nb_seq.min(8) } else { 0 })?; - let start = PaddedStart::new(); + let mut start = None; + // There until decoding goes on from it. + let mut start = (!LONG).then_some(&mut start); let mut seq_state: SeqState<'_> = *seq_state; let more_bits = seq_state.tables.lengths_often_have_more_bits(); @@ -1651,12 +1653,13 @@ impl<'a> Executor<'_, 'a> { } // Where decoding went on from a copy of the start of the stream, if it did. let mut left_at = None; - let mut can_leave = !LONG; while remaining > ahead.len() { let mut count = 0; if !self.split { - if can_leave && let Some(bit_d) = start.set(&seq_state.bit_d) { - can_leave = false; + if PaddedStart::fits(&seq_state.bit_d) + && let Some(start) = start.take() + && let Some(bit_d) = start.insert(PaddedStart::new()).set(&seq_state.bit_d) + { left_at = Some((seq_state, remaining)); seq_state.bit_d = bit_d; } From fad8f8fabc75b0ac3ce0b557625766761f08828c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 23:04:29 +0000 Subject: [PATCH 128/199] zstd: a tiny output is allocated without room after it --- src/zstd/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 51f7e8b397ea..e3fd2f49a4b5 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -193,7 +193,13 @@ pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { } // Fast path: size is known and within reasonable limits - let mut output = output_buffer(size as usize + OUTPUT_SLACK)?; + // Not where it would be most of the buffer: to give that back can take a copy. + let slack = if size as usize >= 2 * OUTPUT_SLACK { + OUTPUT_SLACK + } else { + 0 + }; + let mut output = output_buffer(size as usize + slack)?; let written = decompress(&mut output, src) .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; output.truncate(written); From dc8cf4b9f0533885e45731ae2b4dd5f680028a17 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 00:18:03 +0000 Subject: [PATCH 129/199] node:zlib: zstd no longer hands back C strings of its own --- src/runtime/node/node_zlib_binding.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/node/node_zlib_binding.rs b/src/runtime/node/node_zlib_binding.rs index 22d3102affde..cdd9b3413f84 100644 --- a/src/runtime/node/node_zlib_binding.rs +++ b/src/runtime/node/node_zlib_binding.rs @@ -33,7 +33,7 @@ impl Drop for CountedKeepAlive { /// `msg` / `code` are nullable NUL-terminated C strings. /// Kept as raw `*const c_char` (not `&'static str`) because zlib (`z_stream.msg`) -/// and zstd (`ZSTD_getErrorString`) hand back runtime C pointers. +/// hands back a runtime C pointer. #[derive(Clone, Copy)] pub(crate) struct Error { pub(crate) msg: *const c_char, From ea2600552c71c0d1e73090c4019130262d94cd36 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 00:53:25 +0000 Subject: [PATCH 130/199] Tests no longer name zstd as a C dependency --- test/aot/sanitizers/thread-sanitizer.ts | 2 +- test/internal/build-local-webkit.test.ts | 36 ++++++++++++------------ test/js/bun/util/zstd.test.ts | 9 +++--- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/test/aot/sanitizers/thread-sanitizer.ts b/test/aot/sanitizers/thread-sanitizer.ts index 9724408e99af..84fb085f0d95 100644 --- a/test/aot/sanitizers/thread-sanitizer.ts +++ b/test/aot/sanitizers/thread-sanitizer.ts @@ -10,7 +10,7 @@ compile step (IR to image). No compiled program is run. build-engine [--ninja ] [--jobs ] [option of scripts/build.ts ..] the jsc shell, in build/thread-sanitizer build-front-end [--control] bun-sema, in build/thread-sanitizer-front-end; --codegen : the codegen folder of - a build of Bun of the same sources, with zstd's objects in ../obj/vendor/zstd + a build of Bun of the same sources --control: every task of the checker adds to one plain word: a data race that must be reported compile [--largest ] [file..] each program through both; the default: the 100 largest of test/aot/programs --jobs at a time, --threads for the engine's compiler, --only front-end | engine diff --git a/test/internal/build-local-webkit.test.ts b/test/internal/build-local-webkit.test.ts index 813be511cb68..b260d4382266 100644 --- a/test/internal/build-local-webkit.test.ts +++ b/test/internal/build-local-webkit.test.ts @@ -708,10 +708,10 @@ describe("DirectBuild groups and steps", () => { }); const root = String(dir); const buildDir = join(root, "build"); - const cfg = linux({ buildDir, asan: false, localDeps: `zstd=${join(root, "src")}` }); - const out = join(buildDir, "deps", "zstd"); + const cfg = linux({ buildDir, asan: false, localDeps: `zlib=${join(root, "src")}` }); + const out = join(buildDir, "deps", "zlib"); const dep: Dependency = { - name: "zstd", + name: "zlib", source: () => ({ kind: "github-archive", repo: "x/y", commit: "0" }), build: () => ({ kind: "direct", @@ -751,9 +751,9 @@ describe("DirectBuild groups and steps", () => { const rel = (p: string) => relative(buildDir, p); // The tool's object is not bun's; the generated header is order-only for consumers. - expect(resolved.objects.map(rel)).toEqual(["obj/vendor/zstd/a.cpp.o", "obj/vendor/zstd/b.c.o"]); + expect(resolved.objects.map(rel)).toEqual(["obj/vendor/zlib/a.cpp.o", "obj/vendor/zlib/b.c.o"]); expect(resolved.libs).toEqual(["/outside/libicu.a"]); - expect(resolved.generatedHeaders.map(rel)).toEqual(["deps/zstd/gen.h"]); + expect(resolved.generatedHeaders.map(rel)).toEqual(["deps/zlib/gen.h"]); expect(resolved.outputs).toEqual([]); const lines = n @@ -767,17 +767,17 @@ describe("DirectBuild groups and steps", () => { for (let i = at + 1; lines[i]?.startsWith(" "); i++) vars.push(lines[i]!.trim()); return { line: lines[at]!.replace(/ \| \S+(?=: )/, ""), vars: vars.join("\n") }; }; - expect(edge("deps/zstd/gen.h").line).toBe("build deps/zstd/gen.h: dep_gen deps/zstd/bin/tool"); - expect(edge("deps/zstd/gen.h").vars).toContain(`opts = --cwd=${out} --env=K=V --stdout=${join(out, "gen.h")}`); - expect(edge("deps/zstd/gen.h").vars).toContain("cmd = ruby gen.rb"); - expect(edge("deps/zstd/bin/tool").line).toStartWith("build deps/zstd/bin/tool: link obj/vendor/zstd/tool.cpp.o |"); - expect(edge("deps/zstd/.lib-ready").line).toBe("build deps/zstd/.lib-ready: phony deps/zstd/gen.h"); - expect(edge("pch/prefix.h.hxx.pch").line).toEndWith("|| pch/.dir deps/zstd/.lib-ready"); - expect(edge("obj/vendor/zstd/a.cpp.o").line).toContain(": cxx_pch "); - expect(edge("obj/vendor/zstd/a.cpp.o").line).toEndWith("|| obj/.dir deps/zstd/.lib-ready"); - expect(edge("obj/vendor/zstd/a.cpp.o").vars).toContain("-DLIB -std=c++23"); - expect(edge("obj/vendor/zstd/b.c.o").vars).toStartWith("cflags = -x c++ "); - expect(edge("lib").line).toBe("build lib: phony obj/vendor/zstd/a.cpp.o obj/vendor/zstd/b.c.o"); + expect(edge("deps/zlib/gen.h").line).toBe("build deps/zlib/gen.h: dep_gen deps/zlib/bin/tool"); + expect(edge("deps/zlib/gen.h").vars).toContain(`opts = --cwd=${out} --env=K=V --stdout=${join(out, "gen.h")}`); + expect(edge("deps/zlib/gen.h").vars).toContain("cmd = ruby gen.rb"); + expect(edge("deps/zlib/bin/tool").line).toStartWith("build deps/zlib/bin/tool: link obj/vendor/zlib/tool.cpp.o |"); + expect(edge("deps/zlib/.lib-ready").line).toBe("build deps/zlib/.lib-ready: phony deps/zlib/gen.h"); + expect(edge("pch/prefix.h.hxx.pch").line).toEndWith("|| pch/.dir deps/zlib/.lib-ready"); + expect(edge("obj/vendor/zlib/a.cpp.o").line).toContain(": cxx_pch "); + expect(edge("obj/vendor/zlib/a.cpp.o").line).toEndWith("|| obj/.dir deps/zlib/.lib-ready"); + expect(edge("obj/vendor/zlib/a.cpp.o").vars).toContain("-DLIB -std=c++23"); + expect(edge("obj/vendor/zlib/b.c.o").vars).toStartWith("cflags = -x c++ "); + expect(edge("lib").line).toBe("build lib: phony obj/vendor/zlib/a.cpp.o obj/vendor/zlib/b.c.o"); }); }); @@ -805,13 +805,13 @@ describe("Provides.files", () => { test("are only for a prebuilt source", () => { using g = graph(); const dep: Dependency = { - name: "zstd", + name: "zlib", source: () => ({ kind: "github-archive", repo: "x/y", commit: "0" }), build: () => ({ kind: "direct", sources: [] }), provides: () => ({ libs: [], includes: [], files: ["COPYING"] }), }; expect(() => resolveDep(g.n, g.cfg, dep, new Map())).toThrow( - "zstd: Provides.files is for a prebuilt source", + "zlib: Provides.files is for a prebuilt source", ); }); }); diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index ffbfb50712ac..750a7184ae57 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -777,15 +777,14 @@ describe("sync compression argument handling", () => { }, 60_000); }); -// x64 builds target nehalem, so zstd picks its BMI2 kernels at run time. It used to ask CPUID in -// every CCtx and DCtx init. CPUID is a VM exit under a hypervisor, about 2 us each, two per init. -// arch_prctl(ARCH_SET_CPUID, 0) makes CPUID raise SIGSEGV on the calling thread, so a context that -// still probes the CPU kills the child. The first context of each kind is allowed to probe. +// CPUID is a VM exit under a hypervisor, about 2 us each. The C library once asked it twice in every +// CCtx and DCtx init. arch_prctl(ARCH_SET_CPUID, 0) makes CPUID raise SIGSEGV on the calling thread, +// so a context that probes the CPU kills the child. The first round trip loads what else may probe. describe("zstd context setup", () => { const canFaultOnCpuid = isLinux && process.arch === "x64" && /^flags\s*:.*\bcpuid_fault\b/m.test(readFileSync("/proc/cpuinfo", "utf8")); - it.skipIf(!canFaultOnCpuid)("asks CPUID once, not for each context", async () => { + it.skipIf(!canFaultOnCpuid)("does not ask CPUID", async () => { const script = ` const { dlopen } = require("bun:ffi"); const zlib = require("node:zlib"); From 55c98c0efbeafab3d4d37f5385d219bd6ac6df62 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 01:29:49 +0000 Subject: [PATCH 131/199] zstd: a growing output is zeroed at most 64 steps ahead It was zeroed by as much again as was written, so up to twice the output of a large one-call decompression was touched and stayed resident with the result. --- src/zstd/lib.rs | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index e3fd2f49a4b5..a9da37f53683 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -50,7 +50,7 @@ pub struct OutBuffer<'a> { pub pos: usize, } -/// Output at the end of a `Vec`. The codecs write to initialized memory only: this zeroes about as much as gets +/// Output at the end of a `Vec`. The codecs write to initialized memory only: this zeroes little more than gets /// written, however many calls that takes and however much capacity the `Vec` has to spare. pub struct VecOutput<'a> { vec: &'a mut Vec, @@ -60,6 +60,9 @@ pub struct VecOutput<'a> { } impl<'a> VecOutput<'a> { + /// Memory that is zeroed and not written stays resident with the output. + const MOST_STEPS_AT_ONCE: usize = 64; + pub fn new(vec: &'a mut Vec) -> Self { let filled = vec.len(); Self { @@ -74,13 +77,14 @@ impl<'a> VecOutput<'a> { self.filled } - /// Room for one call of a codec: `step` bytes or more, but never past `limit`. [`advance`](Self::advance) by - /// what it wrote. + /// Room for one call of a codec: `step` bytes, or more of them after much output, but never past `limit`. + /// [`advance`](Self::advance) by what it wrote. pub fn buffer(&mut self, step: usize, limit: usize) -> Result, ErrorCode> { let room = self.vec.len() - self.filled; let budget = limit.saturating_sub(self.filled); if room < step.min(budget) { - let more = step.max(self.filled - self.start).min(budget) - room; + let most = step.saturating_mul(Self::MOST_STEPS_AT_ONCE); + let more = (self.filled - self.start).clamp(step, most).min(budget) - room; self.vec .try_reserve(more) .map_err(|_| ErrorCode::MemoryAllocation)?; @@ -461,3 +465,23 @@ pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { inflated.push(0); inflated.into_boxed_slice() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn little_more_is_zeroed_than_is_written() { + const STEP: usize = 16; + const MOST: usize = STEP * VecOutput::MOST_STEPS_AT_ONCE; + let mut vec = Vec::new(); + let mut out = VecOutput::new(&mut vec); + while out.filled() < 4 * MOST { + let room = out + .buffer(STEP, usize::MAX) + .map_or(0, |buffer| buffer.dst.len()); + assert!((STEP..=MOST).contains(&room)); + out.advance(room); + } + } +} From 708e9b4567e242648315ca9b6f4e5caea5c26dfc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 01:29:49 +0000 Subject: [PATCH 132/199] zstd tests: level -2 instead of -1, which node:zlib takes for unset --- test/js/bun/util/zstd.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 750a7184ae57..983c65fe63f6 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1120,15 +1120,16 @@ describe("the output is that of zstd 1.5.7", () => { const data = level < 19 ? large : large.subarray(0, 140_000); results[level] = `${digest(zstdCompressSync(data, { level }))}, ${digest(zstdCompressSync(small, { level }))}`; } - for (const level of [-1, -7, -100]) { + // Not -1: node:zlib takes that for a parameter that is not set. + for (const level of [-2, -7, -100]) { const params = { [c.ZSTD_c_compressionLevel]: level }; results[level] = `${digest(zlib.zstdCompressSync(large, { params }))}, ${digest(zlib.zstdCompressSync(small, { params }))}`; } expect(results).toMatchInlineSnapshot(` { - "-1": "44935 b2f48ca4f89477e9, 810 e35cec5c3500c856", "-100": "167967 45693a3bc89d2838, 2112 a1f04bdd2c75d853", + "-2": "55329 a7cd8da33a4cce8b, 954 cb3a52b86b5193c7", "-7": "65641 25bb487794d2f45f, 1081 6841140ea417de55", "1": "46828 2a61d18b13a9df88, 824 3a3b2b799ed70f7b", "10": "42353 d560f70047c278d, 807 bc69857972f7737c", From 61bc26a6a6bc33d4491d098d01a8cc21cb69537b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:48:05 +0000 Subject: [PATCH 133/199] zstd: btlazy2 searches a short segment in the loop of the parser, without hints --- src/zstd/compress/internal.rs | 8 +++++++- src/zstd/compress/lazy.rs | 29 +++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 0c72f413768e..d4ce0852c426 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1184,10 +1184,16 @@ impl<'a> TreeInput<'a> { near, dict_limit, dict, - far: near.len() > 1 << 16, + far: Self::is_far(near), }) } + /// What `far` is for a current segment. + #[inline(always)] + pub(crate) const fn is_far(near: Near<'_>) -> bool { + near.len() > 1 << 16 + } + /// `ZSTD_count_2segments`, for the position `ip` of `near` and a match that starts in another segment. #[inline(always)] pub(crate) fn count_2segments( diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index ea805d342ed6..4f0ac754fa7b 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -56,6 +56,7 @@ struct Dubt<'a> { } impl<'a> Dubt<'a> { + #[inline(always)] fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, search_dms: bool) -> Result { let MatchState { hash_table, @@ -243,7 +244,7 @@ impl<'a> Dubt<'a> { impl Dubt<'_> { /// `ZSTD_BtFindBestMatch`: updates the trees up to the position `ip` of `input.near` and returns the best match there. /// Its length is 0 if there is none. - #[inline(never)] + #[inline(always)] fn search(&mut self, ip: u32) -> Result { let mut off_base = 999_999_999u32; let curr = ip.wrapping_add(self.input.dict_limit); @@ -276,6 +277,12 @@ impl Dubt<'_> { }) } + /// [`Self::search`], where the time goes to waiting for memory. + #[inline(never)] + fn search_far(&mut self, ip: u32) -> Result { + self.search(ip) + } + /// [`DictTree::root`] for the position `ip` of `input.near`. #[inline(always)] fn dict_root(&self, dms: &DictMatchState<'_>, ip: u32) -> Result { @@ -757,14 +764,17 @@ trait Finder<'a>: Sized { // ── binary trees, for the parser ────────────────────────────────────────── -struct TreeFinder<'a, const DICT_MODE: u8>(Dubt<'a>); +/// `FAR` is [`TreeInput::far`]. +struct TreeFinder<'a, const DICT_MODE: u8, const FAR: bool>(Dubt<'a>); -impl<'a, const DICT_MODE: u8> Finder<'a> for TreeFinder<'a, DICT_MODE> { +impl<'a, const DICT_MODE: u8, const FAR: bool> Finder<'a> for TreeFinder<'a, DICT_MODE, FAR> { const MARGIN: u32 = 8; #[inline(always)] fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, _block: LazyBlock<'a>) -> Result { - Ok(Self(Dubt::new(ms, src, DICT_MODE == DICT_MATCH_STATE)?)) + let mut dubt = Dubt::new(ms, src, DICT_MODE == DICT_MATCH_STATE)?; + dubt.input.far = FAR; + Ok(Self(dubt)) } #[inline(always)] @@ -774,7 +784,11 @@ impl<'a, const DICT_MODE: u8> Finder<'a> for TreeFinder<'a, DICT_MODE> { wanted: Wanted, _skipping: bool, ) -> Result> { - let found = self.0.search(ip)?; + let found = if FAR { + self.0.search_far(ip)? + } else { + self.0.search(ip)? + }; Ok(Some(found).filter(|found| found.len > wanted.longer_than)) } @@ -2207,8 +2221,11 @@ fn compress_block_lazy_any_strategy( Strategy::Greedy => compress_block_lazy_hashed::<0, DICT_MODE>(args, use_row), Strategy::Lazy => compress_block_lazy_hashed::<1, DICT_MODE>(args, use_row), Strategy::Lazy2 => compress_block_lazy_hashed::<2, DICT_MODE>(args, use_row), + Strategy::BtLazy2 if !use_row && TreeInput::is_far(args.block.near) => { + compress_block_lazy_generic::, 2, DICT_MODE>(args) + } Strategy::BtLazy2 if !use_row => { - compress_block_lazy_generic::, 2, DICT_MODE>(args) + compress_block_lazy_generic::, 2, DICT_MODE>(args) } _ => Err(BUG), } From c1cb938120faa624afeb2b7a4f56927402eceb96 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:17:12 +0000 Subject: [PATCH 134/199] zstd: btlazy2 compares candidates by their first 16 bytes, without waiting for what they are known to share --- src/zstd/compress/internal.rs | 33 ++++++++++++++++++++++++++++----- src/zstd/compress/lazy.rs | 4 ++-- src/zstd/compress/opt.rs | 2 +- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index d4ce0852c426..f19d146357be 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -904,6 +904,24 @@ impl<'a> Near<'a> { Err(len) } + /// [`Self::compare`] of a position that starts with `head`, if these bytes are enough to tell. What it reads does not + /// depend on `from`, so it does not wait for the comparison that `from` comes from. + #[inline(always)] + pub(crate) fn compare_head(self, head: &[u8; 16], match_pos: u32, from: u32) -> Option { + let match_bytes = self.chunk16(match_pos)?; + let mut differ = 0u32; + for (n, (byte, match_byte)) in head.iter().zip(match_bytes).enumerate() { + differ |= u32::from(byte != match_byte) << n; + } + let first = differ.trailing_zeros(); + if first >= 16 || first < from { + return None; + } + let byte = head.get(first as usize & 15).copied().unwrap_or(0); + let match_byte = match_bytes.get(first as usize & 15).copied().unwrap_or(0); + Some(Common::followed_by(first, match_byte, byte)) + } + /// A hint that `pos`, which may be anything, is about to be read. #[inline(always)] pub(crate) fn prefetch(self, pos: u32) { @@ -1286,9 +1304,9 @@ pub(crate) fn bt_look_ahead( /// Goes down the tree whose root is `root` for the position `ip` of `input.near`, which `descent` is for and which /// becomes the root. `visit` is called with each position that is met on the way, which are all at least `low` and at /// most `nb_compares`, and what it has in common with `ip`, and returns whether to go on. Returns how many more positions -/// could have been compared. +/// could have been compared. `BY_HEAD` is whether to try [`Near::compare_head`], which is for ways down that are short. #[inline(always)] -pub(crate) fn bt_walk( +pub(crate) fn bt_walk( bt: &mut Nodes, input: &TreeInput<'_>, ip: u32, @@ -1303,6 +1321,7 @@ pub(crate) fn bt_walk( } let near = input.near; let dict_limit = input.dict_limit; + let head = near.chunk16(ip).filter(|_| BY_HEAD); let mut match_index = root; while nb_compares != 0 && match_index >= low { let children = bt_children(bt, match_index); @@ -1314,9 +1333,13 @@ pub(crate) fn bt_walk( near.prefetch(child.wrapping_sub(dict_limit).wrapping_add(from)); } } - let common = match near.compare_chunks(ip, match_index.wrapping_sub(dict_limit), from) { - Ok(common) => common, - Err(from) => input.compare_rest(ip, match_index, from)?, + let match_pos = match_index.wrapping_sub(dict_limit); + let common = match head.and_then(|head| near.compare_head(head, match_pos, from)) { + Some(common) => common, + None => match near.compare_chunks(ip, match_pos, from) { + Ok(common) => common, + Err(from) => input.compare_rest(ip, match_index, from)?, + }, }; if !visit(match_index, common)? { break; diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 4f0ac754fa7b..5cf43afe1f3d 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -202,7 +202,7 @@ impl<'a> Dubt<'a> { let mut search_dict = true; match_index = table_get(self.hash_table, h); table_put(self.hash_table, h, curr); - let nb_compares = bt_walk( + let nb_compares = bt_walk::( self.bt, &self.input, ip, @@ -390,7 +390,7 @@ fn sort_dubt1( ) -> Result<()> { // The next sorted candidate is in the first slot. The other one has the previous unsorted candidate. let [next_sorted, _] = bt_children(bt, curr); - bt_walk( + bt_walk::( bt, input, curr.wrapping_sub(input.dict_limit), diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index d627397e3888..df4f3e55b3a5 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -569,7 +569,7 @@ impl<'a> Trees<'a> { bt_look_ahead::<4, 2>(self.hash_table, self.bt, &self.input, self.hasher, ip); let descent = Descent::new(self.bt, curr); - bt_walk( + bt_walk::( self.bt, &self.input, ip, From 98633202c48825a6173bb8397c9567c48ded0d68 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:26:14 +0000 Subject: [PATCH 135/199] zstd: the comparison of 16 bytes is done on vectors with overflow checks too --- src/zstd/compress/internal.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index f19d146357be..adbca36d6f50 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -890,8 +890,8 @@ impl<'a> Near<'a> { ) { // A comparison of vectors. let mut differ = 0u32; - for (n, (byte, match_byte)) in bytes.iter().zip(match_bytes).enumerate() { - differ |= u32::from(byte != match_byte) << n; + for (n, (byte, match_byte)) in (0..16).zip(bytes.iter().zip(match_bytes)) { + differ |= u32::from(byte != match_byte).wrapping_shl(n); } if differ != 0 { let first = differ.trailing_zeros(); @@ -910,8 +910,8 @@ impl<'a> Near<'a> { pub(crate) fn compare_head(self, head: &[u8; 16], match_pos: u32, from: u32) -> Option { let match_bytes = self.chunk16(match_pos)?; let mut differ = 0u32; - for (n, (byte, match_byte)) in head.iter().zip(match_bytes).enumerate() { - differ |= u32::from(byte != match_byte) << n; + for (n, (byte, match_byte)) in (0..16).zip(head.iter().zip(match_bytes)) { + differ |= u32::from(byte != match_byte).wrapping_shl(n); } let first = differ.trailing_zeros(); if first >= 16 || first < from { From 8990d637e6e5934fc5ceed63f157ca8bf4bb8722 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:33:22 +0000 Subject: [PATCH 136/199] zstd: btlazy2 does not go down a tree that has nothing --- src/zstd/compress/lazy.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 5cf43afe1f3d..02f3c6c38487 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -166,6 +166,18 @@ impl<'a> Dubt<'a> { let mut match_index = table_get(self.hash_table, h); bt_look_ahead::<2, 1>(self.hash_table, self.bt, &self.input, self.hasher, ip); + let low = window_low.saturating_add(1); + if match_index < low { + // There is no candidate: the position is the whole of its tree. + table_put(self.hash_table, h, curr); + bt_set_children(self.bt, curr, [0, 0]); + *self.next_to_update = curr.wrapping_add(1); + return Ok(DubtFound { + best_length: 0, + nb_compares: self.nb_compares, + }); + } + let unsort_limit = bt_low(self.bt, curr).max(window_low); let mut nb_candidates = self.nb_compares; let mut previous_candidate = 0u32; @@ -207,7 +219,7 @@ impl<'a> Dubt<'a> { &self.input, ip, match_index, - window_low.saturating_add(1), + low, self.nb_compares, Descent::new(self.bt, curr), |match_index, common| { From 65d2eea2d8eb67320154f388b87c1b133918d540 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 02:06:26 +0000 Subject: [PATCH 137/199] zstd: the errors are in error.rs, as Error and Result ErrorCode is now Error, and ZstdError moves next to it. Both derive thiserror::Error. --- Cargo.lock | 1 + src/runtime/api/BunObject.rs | 2 +- src/runtime/node/zlib/NativeZstd.rs | 60 ++++++------ src/runtime/webcore/CompressionStreamCoder.rs | 4 +- src/zstd/Cargo.toml | 1 + src/zstd/common/bitstream.rs | 10 +- src/zstd/common/entropy.rs | 30 +++--- src/zstd/common/fse_decompress.rs | 16 ++-- src/zstd/compress/block_entropy.rs | 6 +- src/zstd/compress/cctx.rs | 26 +++--- src/zstd/compress/fse.rs | 24 ++--- src/zstd/compress/hist.rs | 6 +- src/zstd/compress/huf.rs | 28 +++--- src/zstd/compress/internal.rs | 10 +- src/zstd/compress/ldm.rs | 4 +- src/zstd/compress/literals.rs | 6 +- src/zstd/compress/mt.rs | 31 +++---- src/zstd/compress/params.rs | 8 +- src/zstd/compress/sequences.rs | 14 +-- src/zstd/compress/stream.rs | 24 ++--- src/zstd/compress/superblock.rs | 8 +- src/zstd/decompress/block.rs | 88 +++++++++--------- src/zstd/decompress/huf.rs | 10 +- src/zstd/decompress/huf_exact.rs | 40 ++++---- src/zstd/decompress/mod.rs | 93 +++++++++---------- src/zstd/error.rs | 51 +++++++++- src/zstd/lib.rs | 38 +------- src/zstd/util.rs | 12 +-- 28 files changed, 333 insertions(+), 318 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2fce01f31476..bc4ea21acffd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2238,6 +2238,7 @@ version = "0.0.0" dependencies = [ "bun_pod", "strum", + "thiserror", ] [[package]] diff --git a/src/runtime/api/BunObject.rs b/src/runtime/api/BunObject.rs index e986ab846960..9b69900258e5 100644 --- a/src/runtime/api/BunObject.rs +++ b/src/runtime/api/BunObject.rs @@ -2004,7 +2004,7 @@ pub(crate) mod JSZstd { fn compress_to_box(input: &[u8], level: i32) -> Result, Failure> { match bun_zstd::compress_alloc(input, level) { Ok(output) => Ok(output.into_boxed_slice()), - Err(bun_zstd::ErrorCode::MemoryAllocation) => Err(Failure::OutOfMemory), + Err(bun_zstd::Error::MemoryAllocation) => Err(Failure::OutOfMemory), Err(err) => Err(Failure::Compression(err.message().to_bytes())), } } diff --git a/src/runtime/node/zlib/NativeZstd.rs b/src/runtime/node/zlib/NativeZstd.rs index 110dcbb82d46..e38b1e3ca5b6 100644 --- a/src/runtime/node/zlib/NativeZstd.rs +++ b/src/runtime/node/zlib/NativeZstd.rs @@ -5,7 +5,7 @@ mod constants { use core::ffi::{CStr, c_char}; use bun_zstd::{ - EndDirective, ErrorCode, Strategy, ZSTD_CLEVEL_DEFAULT, ZSTD_D_WINDOW_LOG_MAX, c_param, + EndDirective, Error, Strategy, ZSTD_CLEVEL_DEFAULT, ZSTD_D_WINDOW_LOG_MAX, c_param, }; struct Constant { @@ -17,7 +17,7 @@ mod constants { Constant { name, value } } - const fn error(code: ErrorCode) -> Constant { + const fn error(code: Error) -> Constant { named(code.name(), code as i32) } @@ -59,33 +59,33 @@ mod constants { named(c"ZSTD_d_windowLogMax", ZSTD_D_WINDOW_LOG_MAX), named(c"ZSTD_CLEVEL_DEFAULT", ZSTD_CLEVEL_DEFAULT), named(c"ZSTD_error_no_error", 0), - error(ErrorCode::Generic), - error(ErrorCode::PrefixUnknown), - error(ErrorCode::VersionUnsupported), - error(ErrorCode::FrameParameterUnsupported), - error(ErrorCode::FrameParameterWindowTooLarge), - error(ErrorCode::CorruptionDetected), - error(ErrorCode::ChecksumWrong), - error(ErrorCode::LiteralsHeaderWrong), - error(ErrorCode::DictionaryCorrupted), - error(ErrorCode::DictionaryWrong), - error(ErrorCode::DictionaryCreationFailed), - error(ErrorCode::ParameterUnsupported), - error(ErrorCode::ParameterCombinationUnsupported), - error(ErrorCode::ParameterOutOfBound), - error(ErrorCode::TableLogTooLarge), - error(ErrorCode::MaxSymbolValueTooLarge), - error(ErrorCode::MaxSymbolValueTooSmall), - error(ErrorCode::StabilityConditionNotRespected), - error(ErrorCode::StageWrong), - error(ErrorCode::InitMissing), - error(ErrorCode::MemoryAllocation), - error(ErrorCode::WorkSpaceTooSmall), - error(ErrorCode::DstSizeTooSmall), - error(ErrorCode::SrcSizeWrong), - error(ErrorCode::DstBufferNull), - error(ErrorCode::NoForwardProgressDestFull), - error(ErrorCode::NoForwardProgressInputEmpty), + error(Error::Generic), + error(Error::PrefixUnknown), + error(Error::VersionUnsupported), + error(Error::FrameParameterUnsupported), + error(Error::FrameParameterWindowTooLarge), + error(Error::CorruptionDetected), + error(Error::ChecksumWrong), + error(Error::LiteralsHeaderWrong), + error(Error::DictionaryCorrupted), + error(Error::DictionaryWrong), + error(Error::DictionaryCreationFailed), + error(Error::ParameterUnsupported), + error(Error::ParameterCombinationUnsupported), + error(Error::ParameterOutOfBound), + error(Error::TableLogTooLarge), + error(Error::MaxSymbolValueTooLarge), + error(Error::MaxSymbolValueTooSmall), + error(Error::StabilityConditionNotRespected), + error(Error::StageWrong), + error(Error::InitMissing), + error(Error::MemoryAllocation), + error(Error::WorkSpaceTooSmall), + error(Error::DstSizeTooSmall), + error(Error::SrcSizeWrong), + error(Error::DstBufferNull), + error(Error::NoForwardProgressDestFull), + error(Error::NoForwardProgressInputEmpty), ]; /// The name of the constant at `index`, whose value goes to `value`, or null past the last one. @@ -364,7 +364,7 @@ mod _impl { output: *mut [u8], output_pos: usize, pledged_src_size: u64, - error: Option, + error: Option, } impl Default for Context { diff --git a/src/runtime/webcore/CompressionStreamCoder.rs b/src/runtime/webcore/CompressionStreamCoder.rs index 82f7d500b562..518a5d301e56 100644 --- a/src/runtime/webcore/CompressionStreamCoder.rs +++ b/src/runtime/webcore/CompressionStreamCoder.rs @@ -74,8 +74,8 @@ fn spare(out: &mut Vec, cap: usize) -> Result<&mut [core::mem::MaybeUninit CodecError { - if code == zstd::ErrorCode::MemoryAllocation { +fn zstd_error(code: zstd::Error, message: &'static str) -> CodecError { + if code == zstd::Error::MemoryAllocation { CodecError::OutOfMemory } else { CodecError::Message(message) diff --git a/src/zstd/Cargo.toml b/src/zstd/Cargo.toml index 31e702c51bec..07d0d2c27dba 100644 --- a/src/zstd/Cargo.toml +++ b/src/zstd/Cargo.toml @@ -11,4 +11,5 @@ workspace = true [dependencies] strum.workspace = true +thiserror.workspace = true bun_pod = { workspace = true, features = ["alloc"] } diff --git a/src/zstd/common/bitstream.rs b/src/zstd/common/bitstream.rs index 2576cab18fed..bbc8bce19479 100644 --- a/src/zstd/common/bitstream.rs +++ b/src/zstd/common/bitstream.rs @@ -1,6 +1,6 @@ //! `bitstream.h`: a bit stream is written forward and read backward. -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{SliceExt, highbit32, read_le64}; const CONTAINER_BYTES: usize = size_of::(); @@ -21,7 +21,7 @@ impl<'a> BitCStream<'a> { #[inline] pub(crate) fn new(dst: &'a mut [u8]) -> Result { if dst.len() <= CONTAINER_BYTES { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } let end = dst.len() - CONTAINER_BYTES; Ok(Self { @@ -107,12 +107,12 @@ impl<'a> BitDStream<'a> { #[inline] pub(crate) fn new(src: &'a [u8]) -> Result { let Some(&last_byte) = src.last() else { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); }; let padding = 8 - highbit32(u32::from(last_byte)); if let Some(tail) = src.last_chunk::() { if last_byte == 0 { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } return Ok(Self { src, @@ -121,7 +121,7 @@ impl<'a> BitDStream<'a> { }); } if last_byte == 0 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let mut bytes = [0u8; CONTAINER_BYTES]; bytes.write_at(0, src)?; diff --git a/src/zstd/common/entropy.rs b/src/zstd/common/entropy.rs index 82ad6b5ee52c..f69c82de1212 100644 --- a/src/zstd/common/entropy.rs +++ b/src/zstd/common/entropy.rs @@ -2,7 +2,7 @@ use super::fse_decompress::{Spread, fse_decompress_weights}; use super::{FSE_MIN_TABLELOG, FSE_TABLELOG_ABSOLUTE_MAX, HUF_TABLELOG_MAX}; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{BUG, SliceExt, highbit32, read_le32}; /// What [`read_ncount`] found besides the counts. @@ -124,7 +124,7 @@ fn read_any_ncount( buffer.write_at(0, header)?; let ncount = read_any_ncount(normalized_counter, max_symbol_value, &buffer)?; if ncount.size > header.len() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } return Ok(ncount); } @@ -142,7 +142,7 @@ fn read_any_ncount( let mut bit_stream = read_le32(header, 0)?; let mut nb_bits = ((bit_stream & 0xF) + FSE_MIN_TABLELOG) as i32; if nb_bits > FSE_TABLELOG_ABSOLUTE_MAX as i32 { - return Err(ErrorCode::TableLogTooLarge); + return Err(Error::TableLogTooLarge); } bit_stream >>= 4; let mut bit_count: i32 = 4; @@ -237,13 +237,13 @@ fn read_any_ncount( } } if remaining != 1 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } if charnum > max_sv1 { - return Err(ErrorCode::MaxSymbolValueTooSmall); + return Err(Error::MaxSymbolValueTooSmall); } if bit_count > 32 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } ip += ((bit_count + 7) >> 3) as isize; Ok(NCount { @@ -287,7 +287,7 @@ pub(crate) fn huf_read_stats_wksp( spread: &mut Spread, ) -> Result { let Some((&first, rest)) = src.split_first() else { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); }; let mut i_size = usize::from(first); let o_size; @@ -297,10 +297,10 @@ pub(crate) fn huf_read_stats_wksp( o_size = i_size - 127; i_size = o_size.div_ceil(2); if i_size + 1 > src.len() { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } if o_size >= huff_weight.len() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let (pairs, _) = huff_weight.as_chunks_mut::<2>(); for (pair, &byte) in pairs.iter_mut().zip(rest.sub(..i_size)?) { @@ -308,7 +308,7 @@ pub(crate) fn huf_read_stats_wksp( } } else { if i_size + 1 > src.len() { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } o_size = fse_decompress_weights(huff_weight.sub_mut(..255)?, rest.sub(..i_size)?, spread)?; } @@ -335,7 +335,7 @@ pub(crate) fn huf_read_stats_wksp( return Err(BUG); }; if all_bits & 0xF0F0_F0F0 != 0 || too_large.iter().any(|&rank| rank != 0) { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } *rank_stats = *valid; // Those that complete the last four. @@ -346,27 +346,27 @@ pub(crate) fn huf_read_stats_wksp( .map(|(weight, &rank)| rank << (weight - 1)) .sum::(); if weight_total == 0 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } // The last symbol's weight is not stored: it completes the total to a power of 2. let table_log = highbit32(weight_total) + 1; if table_log > HUF_TABLELOG_MAX { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let total = 1u32 << table_log; let rest = total - weight_total; let verif = 1u32 << highbit32(rest); let last_weight = highbit32(rest) + 1; if verif != rest { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } huff_weight.put(o_size, last_weight as u8)?; *rank_stats.at_mut(last_weight as usize)? += 1; let [_, rank1, ..] = *rank_stats; if rank1 < 2 || rank1 & 1 != 0 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } Ok(HufStats { diff --git a/src/zstd/common/fse_decompress.rs b/src/zstd/common/fse_decompress.rs index cc9da08043d0..9c7c0212826a 100644 --- a/src/zstd/common/fse_decompress.rs +++ b/src/zstd/common/fse_decompress.rs @@ -3,7 +3,7 @@ use super::bitstream::BitDStream; use super::entropy::read_ncount; use super::{FSE_MAX_SYMBOL_VALUE, FSE_MAX_TABLELOG, fse_table_step}; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{SliceExt, write_le64}; /// The largest table log of the weights of a Huffman table. @@ -244,13 +244,13 @@ fn build_dtable( wksp_size: usize, ) -> Result<()> { if build_dtable_wksp_size(table_log, max_symbol_value) > wksp_size { - return Err(ErrorCode::MaxSymbolValueTooLarge); + return Err(Error::MaxSymbolValueTooLarge); } if max_symbol_value > FSE_MAX_SYMBOL_VALUE { - return Err(ErrorCode::MaxSymbolValueTooLarge); + return Err(Error::MaxSymbolValueTooLarge); } if table_log > FSE_MAX_TABLELOG { - return Err(ErrorCode::TableLogTooLarge); + return Err(Error::TableLogTooLarge); } dt.table_log = table_log; let mut states = spread.states(max_symbol_value as usize + 1, table_log)?; @@ -354,7 +354,7 @@ fn decompress_using_dtable(dst: &mut [u8], src: &[u8], dt: &FseDTable) -> Result decoder.read_state(dt.table_log), ]; if decoder.is_past_start() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } // All but the last symbol leave room for one more. @@ -381,7 +381,7 @@ fn decompress_using_dtable(dst: &mut [u8], src: &[u8], dt: &FseDTable) -> Result } loop { if op + 2 > omax { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } dst.put(op, decoder.decode_one())?; op += 1; @@ -400,12 +400,12 @@ pub(crate) fn fse_decompress_weights( ) -> Result { let header = read_ncount(&mut spread.counts, FSE_MAX_SYMBOL_VALUE, src)?; if header.table_log > MAX_LOG { - return Err(ErrorCode::TableLogTooLarge); + return Err(Error::TableLogTooLarge); } let body = src.sub(header.size..)?; if decompress_wksp_size(header.table_log, header.max_symbol_value) > WKSP_SIZE { - return Err(ErrorCode::TableLogTooLarge); + return Err(Error::TableLogTooLarge); } let wksp_size = WKSP_SIZE - NCOUNT_SIZE - dtable_size_u32(header.table_log) * size_of::(); diff --git a/src/zstd/compress/block_entropy.rs b/src/zstd/compress/block_entropy.rs index 30f0cbb2f908..096b65aa3152 100644 --- a/src/zstd/compress/block_entropy.rs +++ b/src/zstd/compress/block_entropy.rs @@ -18,7 +18,7 @@ use crate::common::{ MINMATCH, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, SymbolEncodingType, ZSTD_BLOCKHEADERSIZE, }; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{SliceExt, write_le16}; /// `ZSTD_literalsCompressionIsDisabled` @@ -219,7 +219,7 @@ fn entropy_compress_seq_store_internal( } if dst.len() - op < 3 + 1 { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } op += write_nb_seq(dst.sub_mut(op..)?, nb_seq)?; if nb_seq == 0 { @@ -285,7 +285,7 @@ pub(crate) fn entropy_compress_seq_store( cctx_params, ) { // There is room for the block as it is. - Err(ErrorCode::DstSizeTooSmall) if block_size <= dst.len() => return Ok(0), + Err(Error::DstSizeTooSmall) if block_size <= dst.len() => return Ok(0), c_size => c_size?, }; let max_c_size = block_size.wrapping_sub(min_gain(block_size, cctx_params.c_params.strat())); diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 9ddfc11d4908..14210d0c2a12 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -34,7 +34,7 @@ use crate::common::{ ZSTD_FRAMEHEADERSIZE_MAX, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGICNUMBER, ZSTD_REP_NUM, ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, }; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{ SliceExt, highbit32, read_le32, try_zeroed, write_le16, write_le24, write_le32, write_le64, }; @@ -394,7 +394,7 @@ fn dict_ncount_repeat( /// `ZSTD_loadCEntropy`: reads the tables of the dictionary `dict` into `bs`. Returns where its content starts. fn load_c_entropy(bs: &mut CompressedBlockState, dict: &[u8]) -> Result { - const CORRUPTED: ErrorCode = ErrorCode::DictionaryCorrupted; + const CORRUPTED: Error = Error::DictionaryCorrupted; let mut offcode_ncount = [0i16; MAX_OFF as usize + 1]; let mut dict_ptr = 8usize; bs.entropy.huf.repeat_mode = HufRepeat::Check; @@ -721,7 +721,7 @@ fn write_frame_header( let mut pos = 0usize; if dst.len() < ZSTD_FRAMEHEADERSIZE_MAX { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } if params.format == Format::Zstd1 { write_le32(dst, 0, ZSTD_MAGICNUMBER)?; @@ -774,7 +774,7 @@ fn write_frame_header( /// `ZSTD_writeLastEmptyBlock` pub(crate) fn write_last_empty_block(dst: &mut [u8]) -> Result { if dst.len() < ZSTD_BLOCKHEADERSIZE { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } write_le24(dst, 0, 1 + ((BlockType::Raw as u32) << 1))?; Ok(ZSTD_BLOCKHEADERSIZE) @@ -1342,7 +1342,7 @@ impl FrameCompressor { } if dst.len() < ZSTD_BLOCKHEADERSIZE { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } let mut c_seqs_size = entropy_compress_seq_store( &mut self.seq_store, @@ -1603,7 +1603,7 @@ impl FrameCompressor { src, last_block, ) { - Err(ErrorCode::DstSizeTooSmall) => {} + Err(Error::DstSizeTooSmall) => {} c_size => { let c_size = c_size?; let max_c_size = src @@ -1665,7 +1665,7 @@ impl FrameCompressor { let out = dst.sub_mut(op..)?; if out.len() < ZSTD_BLOCKHEADERSIZE + MIN_CBLOCK_SIZE + 1 { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } let ms = &mut self.match_state; @@ -1725,7 +1725,7 @@ impl FrameCompressor { ) -> Result { let mut fh_size = 0usize; if self.stage == CompressionStage::Created { - return Err(ErrorCode::StageWrong); + return Err(Error::StageWrong); } if self.stage == CompressionStage::Init { @@ -1763,7 +1763,7 @@ impl FrameCompressor { if self.pledged_src_size_plus_one != 0 && self.consumed_src_size + 1 > self.pledged_src_size_plus_one { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } Ok(c_size + fh_size) } @@ -1782,7 +1782,7 @@ impl FrameCompressor { fn write_epilogue(&mut self, dst: &mut [u8]) -> Result { let mut op = 0usize; if self.stage == CompressionStage::Created { - return Err(ErrorCode::StageWrong); + return Err(Error::StageWrong); } // An empty frame. @@ -1794,7 +1794,7 @@ impl FrameCompressor { if self.stage != CompressionStage::Ending { // One last empty block, to mark the end of the frame. if dst.len() - op < 3 { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } write_le24(dst, op, 1 + ((BlockType::Raw as u32) << 1))?; op += ZSTD_BLOCKHEADERSIZE; @@ -1803,7 +1803,7 @@ impl FrameCompressor { if self.applied_params.f_params.checksum_flag { let checksum = self.xxh_state.digest() as u32; if dst.len() - op < 4 { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } write_le32(dst, op, checksum)?; op += 4; @@ -1825,7 +1825,7 @@ impl FrameCompressor { if self.pledged_src_size_plus_one != 0 && self.pledged_src_size_plus_one != self.consumed_src_size + 1 { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } Ok(c_size + end_result) } diff --git a/src/zstd/compress/fse.rs b/src/zstd/compress/fse.rs index 5d969fef0ea7..6d536fdf0c6a 100644 --- a/src/zstd/compress/fse.rs +++ b/src/zstd/compress/fse.rs @@ -2,7 +2,7 @@ use crate::common::bitstream::BitCStream; use crate::common::{FSE_MAX_TABLELOG, FSE_MIN_TABLELOG, MAX_SEQ, ML_FSE_LOG, fse_table_step}; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{BUG, SliceExt, highbit32, write_le16}; pub(crate) const FSE_NCOUNTBOUND: usize = 512; @@ -91,7 +91,7 @@ impl FseCTable { } /// `FSE_buildCTable_wksp`: `normalized_counter` adds up to `1 << table_log`, a count of -1 standing for 1. - /// [`ErrorCode::TableLogTooLarge`] if the table has no room for it. + /// [`Error::TableLogTooLarge`] if the table has no room for it. pub(crate) fn build( &mut self, normalized_counter: &[i16], @@ -99,7 +99,7 @@ impl FseCTable { table_log: u32, ) -> Result<()> { if table_log > ML_FSE_LOG || max_symbol_value > MAX_SEQ { - return Err(ErrorCode::TableLogTooLarge); + return Err(Error::TableLogTooLarge); } let table_size = 1usize << table_log; let table_mask = table_size - 1; @@ -338,7 +338,7 @@ fn write_ncount_generic( // The lowest 16 bits of the stream leave it. let mut write16 = |bit_stream: &mut u32, out: &mut usize| -> Result<()> { if !write_is_safe && *out + 2 > header.len() { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } write_le16(header, *out, *bit_stream as u16)?; *out += 2; @@ -394,7 +394,7 @@ fn write_ncount_generic( bit_count -= i32::from(count < max); previous_is_0 = count == 1; if remaining < 1 { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } while remaining < threshold { nb_bits -= 1; @@ -408,7 +408,7 @@ fn write_ncount_generic( } if remaining != 1 { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } let last = out; @@ -424,10 +424,10 @@ pub(crate) fn write_ncount( table_log: u32, ) -> Result { if table_log > FSE_MAX_TABLELOG { - return Err(ErrorCode::TableLogTooLarge); + return Err(Error::TableLogTooLarge); } if table_log < FSE_MIN_TABLELOG { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } let write_is_safe = buffer.len() >= ncount_write_bound(max_symbol_value, table_log); write_ncount_generic( @@ -574,7 +574,7 @@ fn normalize_m2( let s_end = (end >> v_step_log) as u32; let weight = s_end.wrapping_sub(s_start); if weight < 1 { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } *norm = weight as i16; tmp_total = end; @@ -597,13 +597,13 @@ pub(crate) fn normalize_count( table_log = FSE_DEFAULT_TABLELOG; } if table_log < FSE_MIN_TABLELOG { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } if table_log > FSE_MAX_TABLELOG { - return Err(ErrorCode::TableLogTooLarge); + return Err(Error::TableLogTooLarge); } if table_log < min_table_log(total, max_symbol_value) { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } const RTB_TABLE: [u32; 8] = [0, 473195, 504333, 520860, 550000, 700000, 750000, 830000]; diff --git a/src/zstd/compress/hist.rs b/src/zstd/compress/hist.rs index 745fc94e641f..9ea10424765c 100644 --- a/src/zstd/compress/hist.rs +++ b/src/zstd/compress/hist.rs @@ -1,6 +1,6 @@ //! `hist.c`: how many times each byte value occurs. -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{BUG, SliceExt}; /// `HIST_add`: `count` has room for every value in `src`. @@ -98,7 +98,7 @@ fn count_parallel( let max_symbol = counting1.iter().rposition(|&c| c != 0).ok_or(BUG)? as u32; if check && max_symbol > *max_symbol_value { - return Err(ErrorCode::MaxSymbolValueTooSmall); + return Err(Error::MaxSymbolValueTooSmall); } *max_symbol_value = max_symbol; count.write_at(0, counting1.sub(..count_size)?)?; @@ -118,7 +118,7 @@ pub(crate) fn count_fast_wksp( } /// `HIST_count_wksp`: as [`count_fast_wksp`], but a value above `max_symbol_value` in `src` is -/// [`ErrorCode::MaxSymbolValueTooSmall`]. +/// [`Error::MaxSymbolValueTooSmall`]. pub(crate) fn count_wksp( count: &mut [u32], max_symbol_value: &mut u32, diff --git a/src/zstd/compress/huf.rs b/src/zstd/compress/huf.rs index 73d985e53945..7426d0152d30 100644 --- a/src/zstd/compress/huf.rs +++ b/src/zstd/compress/huf.rs @@ -5,7 +5,7 @@ use super::hist; use super::internal::Strategy; use crate::common::entropy::huf_read_stats; use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{BUG, SliceExt, highbit32, write_le16}; pub(crate) const HUF_BLOCKSIZE_MAX: usize = 128 * 1024; @@ -188,7 +188,7 @@ pub(crate) fn write_ctable( let mut huff_weight = [0u8; HUF_SYMBOLVALUE_MAX as usize]; if max_symbol_value > HUF_SYMBOLVALUE_MAX { - return Err(ErrorCode::MaxSymbolValueTooLarge); + return Err(Error::MaxSymbolValueTooLarge); } // The weight of the last symbol is not stored. let nb_weights = max_symbol_value as usize; @@ -203,7 +203,7 @@ pub(crate) fn write_ctable( let max_dst_size = dst.len(); let Some((first, rest)) = dst.split_first_mut() else { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); }; let h_size = compress_weights(rest, weights)?; if h_size > 1 && h_size < nb_weights / 2 { @@ -213,11 +213,11 @@ pub(crate) fn write_ctable( // Not compressed: 4 bits to a weight. if max_symbol_value > 256 - 128 { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } let raw_size = nb_weights.div_ceil(2); if raw_size + 1 > max_dst_size { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } *first = 128u32.wrapping_add(max_symbol_value.wrapping_sub(1)) as u8; // An odd number of weights is completed with a 0, which `huff_weight` still has there. @@ -237,7 +237,7 @@ pub(crate) struct HufCTableStats { pub(crate) has_zero_weights: bool, } -/// `HUF_readCTable`: [`ErrorCode::MaxSymbolValueTooSmall`] if the table has symbols above `max_symbol_value`. +/// `HUF_readCTable`: [`Error::MaxSymbolValueTooSmall`] if the table has symbols above `max_symbol_value`. pub(crate) fn read_ctable( ctable: &mut HufCTable, max_symbol_value: u32, @@ -251,10 +251,10 @@ pub(crate) fn read_ctable( let [zero_weights, ..] = rank_val; if table_log > HUF_TABLELOG_MAX { - return Err(ErrorCode::TableLogTooLarge); + return Err(Error::TableLogTooLarge); } if stats.nb_symbols > max_symbol_value.wrapping_add(1) { - return Err(ErrorCode::MaxSymbolValueTooSmall); + return Err(Error::MaxSymbolValueTooSmall); } let max_symbol_value = stats.nb_symbols - 1; ctable.write_header(table_log, max_symbol_value); @@ -682,7 +682,7 @@ pub(crate) fn build_ctable( max_nb_bits = HUF_TABLELOG_DEFAULT; } if max_symbol_value > HUF_SYMBOLVALUE_MAX { - return Err(ErrorCode::MaxSymbolValueTooLarge); + return Err(Error::MaxSymbolValueTooLarge); } let mut huff_node = HuffNodes([NodeElt::default(); _]); @@ -691,7 +691,7 @@ pub(crate) fn build_ctable( build_tree(huff_node.0.sub_mut(..present.len)?); max_nb_bits = set_max_height(&mut huff_node, non_null_rank as u32, max_nb_bits)?; if max_nb_bits > HUF_TABLELOG_MAX { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } build_ctable_from_tree( ctable, @@ -765,7 +765,7 @@ impl<'a> HufCStream<'a> { #[inline] fn new(dst: &'a mut [u8]) -> Result { if dst.len() <= size_of::() { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } let end = dst.len() - size_of::(); Ok(Self { @@ -1039,13 +1039,13 @@ fn compress_internal( return Ok(0); } if src_size > HUF_BLOCKSIZE_MAX { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } if huff_log > HUF_TABLELOG_MAX { - return Err(ErrorCode::TableLogTooLarge); + return Err(Error::TableLogTooLarge); } if max_symbol_value > HUF_SYMBOLVALUE_MAX { - return Err(ErrorCode::MaxSymbolValueTooLarge); + return Err(Error::MaxSymbolValueTooLarge); } if max_symbol_value == 0 { max_symbol_value = HUF_SYMBOLVALUE_MAX; diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index adbca36d6f50..556e32393cd4 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -10,7 +10,7 @@ use crate::common::{ BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64, try_zeroed, write_le24}; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; @@ -374,7 +374,7 @@ impl SeqStore { // What it holds is not worth a copy. *vec = Vec::new(); vec.try_reserve_exact(len) - .map_err(|_| ErrorCode::MemoryAllocation) + .map_err(|_| Error::MemoryAllocation) } self.reset(); let block_size = block_size.min(self.max_nb_lit); @@ -398,7 +398,7 @@ impl SeqStore { if self.codes.capacity() < len { self.codes = Vec::new(); (self.codes.try_reserve_exact(self.sequences.capacity())) - .map_err(|_| ErrorCode::MemoryAllocation)?; + .map_err(|_| Error::MemoryAllocation)?; } if self.codes.len() < len { self.codes.resize(len, SeqCodes::default()); @@ -653,7 +653,7 @@ pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> let c_block_header24 = u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } write_le24(dst, 0, c_block_header24)?; dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; @@ -670,7 +670,7 @@ pub(crate) fn rle_compress_block( let c_block_header = u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); if dst.len() < 4 { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } write_le24(dst, 0, c_block_header)?; dst.put(3, byte)?; diff --git a/src/zstd/compress/ldm.rs b/src/zstd/compress/ldm.rs index 1b1cb9b0d3a8..6959f6dbcb50 100644 --- a/src/zstd/compress/ldm.rs +++ b/src/zstd/compress/ldm.rs @@ -9,7 +9,7 @@ use super::params::{ZSTD_HASHLOG_MAX, ZSTD_HASHLOG_MIN, ZSTD_LDM_BUCKETSIZELOG_M use super::{compress_block, double_fast, fast}; use crate::common::ZSTD_REP_NUM; use crate::common::xxhash::Xxh64; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{BUG, SliceExt}; const LDM_BUCKET_SIZE_LOG: u32 = 4; @@ -467,7 +467,7 @@ fn generate_sequences_internal( let offset = new_entry.offset.wrapping_sub(best_entry.offset); let m_length = forward_match_length + backward_match_length; if raw_seq_store.size == raw_seq_store.capacity { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } raw_seq_store.seq.put( raw_seq_store.size, diff --git a/src/zstd/compress/literals.rs b/src/zstd/compress/literals.rs index c37e8c0c26fe..14450d85c865 100644 --- a/src/zstd/compress/literals.rs +++ b/src/zstd/compress/literals.rs @@ -3,7 +3,7 @@ use super::huf::{self, HUF_OPTIMAL_DEPTH_THRESHOLD, HufFlags, HufRepeat}; use super::internal::{HufCTables, Strategy, min_gain}; use crate::common::{HUF_SYMBOLVALUE_MAX, LIT_HUF_LOG, SymbolEncodingType}; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{SliceExt, write_le16, write_le24, write_le32}; /// The header of a section of raw or run-length encoded literals. Returns its size. @@ -23,7 +23,7 @@ fn write_short_header(dst: &mut [u8], ty: SymbolEncodingType, src_size: usize) - pub(crate) fn no_compress_literals(dst: &mut [u8], src: &[u8]) -> Result { let fl_size = 1 + usize::from(src.len() > 31) + usize::from(src.len() > 4095); if src.len() + fl_size > dst.len() { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } write_short_header(dst, SymbolEncodingType::Basic, src.len())?; dst.write_at(fl_size, src)?; @@ -72,7 +72,7 @@ pub(crate) fn compress_literals( } if dst.len() < lh_size + 1 { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } let c_lit_size = { let mut repeat = prev_huf.repeat_mode; diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 9f2ce5fa0499..0d6c32699cce 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -26,7 +26,7 @@ use super::params::{ use super::stream::EndDirective; use crate::common::ZSTD_BLOCKSIZE_MAX; use crate::common::xxhash::Xxh64; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{BUG, SliceExt, try_copy}; use crate::{InBuffer, OutBuffer}; @@ -74,7 +74,7 @@ impl SerialState { self.ldm_state.loaded_dict_end = 0; self.round_buff.clear(); (self.round_buff.try_reserve_exact(round_buff_capacity)) - .map_err(|_| ErrorCode::MemoryAllocation)?; + .map_err(|_| Error::MemoryAllocation)?; } else { self.ldm_state = LdmState::new(); self.round_buff = Vec::new(); @@ -95,7 +95,7 @@ impl SerialState { let written = self.round_buff.get_mut(round_pos..).ok_or(BUG)?; let (over, beyond) = src.split_at(written.len().min(src.len())); written.write_at(0, over)?; - (self.round_buff.try_reserve(beyond.len())).map_err(|_| ErrorCode::MemoryAllocation)?; + (self.round_buff.try_reserve(beyond.len())).map_err(|_| Error::MemoryAllocation)?; self.round_buff.extend_from_slice(beyond); (self.ldm_state.window).update( BufId::InBuff.start() + round_pos, @@ -128,7 +128,7 @@ enum Progress { /// Some more output, and how much of the input is consumed by now. Output { data: Vec, consumed: usize }, /// The job stops here. - Failed(ErrorCode), + Failed(Error), } /// `ZSTDMT_jobDescription`, as far as the worker is concerned. @@ -165,8 +165,7 @@ const CHUNK_SIZE: usize = 4 * ZSTD_BLOCKSIZE_MAX; /// Makes `seq` `len` long, in the allocation that it has if that is large enough. fn resize_in_place(seq: &mut Vec, len: usize) -> Result<()> { seq.clear(); - seq.try_reserve(len) - .map_err(|_| ErrorCode::MemoryAllocation)?; + seq.try_reserve(len).map_err(|_| Error::MemoryAllocation)?; seq.resize(len, RawSeq::default()); Ok(()) } @@ -185,9 +184,9 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - size: 0, capacity: job.max_nb_seq, }; - let mut serial = job.serial_in.recv().map_err(|_| ErrorCode::Generic)?; + let mut serial = job.serial_in.recv().map_err(|_| Error::Generic)?; let generated = if cancelled.load(Ordering::SeqCst) { - Err(ErrorCode::Generic) + Err(Error::Generic) } else { resize_in_place(&mut raw_seq_store.seq, job.nb_seq_bound) .and_then(|()| serial.gen_sequences(&mut raw_seq_store, src, job.round_pos)) @@ -270,7 +269,7 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - }; for chunk_nb in 1..nb_chunks { if cancelled.load(Ordering::SeqCst) { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } let chunk = Span { buf: BufId::User, @@ -323,7 +322,7 @@ struct PendingJob { consumed: usize, src_size: usize, frame_checksum_needed: bool, - error: Option, + error: Option, } impl PendingJob { @@ -370,7 +369,7 @@ impl PendingJob { } } // The worker is gone. - None => self.take(Progress::Failed(ErrorCode::Generic)), + None => self.take(Progress::Failed(Error::Generic)), } } } @@ -488,7 +487,7 @@ impl MtCtx { .name("zstd".into()) .stack_size(2 << 20) .spawn(move || worker_main(&queue_end, &cancelled)) - .map_err(|_| ErrorCode::MemoryAllocation)?; + .map_err(|_| Error::MemoryAllocation)?; self.nb_workers += 1; } self.params.nb_workers = nb_workers as i32; @@ -639,7 +638,7 @@ impl MtCtx { let (progress, progress_receiver) = channel(); let (serial_out, serial_next) = channel(); - let serial_in = self.serial.replace(serial_next).ok_or(ErrorCode::Generic)?; + let serial_in = self.serial.replace(serial_next).ok_or(Error::Generic)?; pending.progress = Some(progress_receiver); let mut ldm_params = params.ldm_params; if ldm_params.enable_ldm == ParamSwitch::Enable { @@ -660,7 +659,7 @@ impl MtCtx { serial_out, progress, }; - self.queue.send(job).map_err(|_| ErrorCode::Generic)?; + self.queue.send(job).map_err(|_| Error::Generic)?; self.jobs.push_back(pending); self.next_job_id += 1; Ok(()) @@ -763,7 +762,7 @@ impl MtCtx { .min(self.in_buff_prefix_size + self.target_section_size); self.in_buff .try_reserve_exact(capacity - self.in_buff.len()) - .map_err(|_| ErrorCode::MemoryAllocation) + .map_err(|_| Error::MemoryAllocation) } /// `ZSTDMT_compressStream_generic` @@ -777,7 +776,7 @@ impl MtCtx { if self.frame_ended && end_op == EndDirective::Continue { // The frame is complete and still to be flushed. - return Err(ErrorCode::StageWrong); + return Err(Error::StageWrong); } if input.src.len() > input.pos && (self.in_buff_ready || self.try_get_input_range()?) { diff --git a/src/zstd/compress/params.rs b/src/zstd/compress/params.rs index e003c587677b..7c2c81e2c5e8 100644 --- a/src/zstd/compress/params.rs +++ b/src/zstd/compress/params.rs @@ -13,7 +13,7 @@ use crate::common::{ ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WINDOWLOG_LIMIT_DEFAULT, ZSTD_WINDOWLOG_MAX, ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, }; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::highbit32; pub const ZSTD_CLEVEL_DEFAULT: i32 = 3; @@ -169,7 +169,7 @@ fn bounds(param: i32) -> Result> { c_param::LDM_HASH_RATE_LOG => ZSTD_LDM_HASHRATELOG_MIN..=ZSTD_LDM_HASHRATELOG_MAX, c_param::FORMAT => 0..=1, c_param::TARGET_C_BLOCK_SIZE => ZSTD_TARGETCBLOCKSIZE_MIN..=ZSTD_TARGETCBLOCKSIZE_MAX, - _ => return Err(ErrorCode::ParameterUnsupported), + _ => return Err(Error::ParameterUnsupported), }) } @@ -182,7 +182,7 @@ fn clamp_bounds(param: i32, value: i32) -> Result { /// `BOUNDCHECK` fn bound_check(param: i32, value: i32) -> Result<()> { if !bounds(param)?.contains(&value) { - return Err(ErrorCode::ParameterOutOfBound); + return Err(Error::ParameterOutOfBound); } Ok(()) } @@ -346,7 +346,7 @@ impl CCtxParams { self.target_c_block_size = bound_check_unless_0(param, value)? as usize; Ok(self.target_c_block_size) } - _ => Err(ErrorCode::ParameterUnsupported), + _ => Err(Error::ParameterUnsupported), } } } diff --git a/src/zstd/compress/sequences.rs b/src/zstd/compress/sequences.rs index 960e32bdd833..adb489da53e2 100644 --- a/src/zstd/compress/sequences.rs +++ b/src/zstd/compress/sequences.rs @@ -11,7 +11,7 @@ use crate::common::{ MAX_SEQ, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, OFF_FSE_LOG, SymbolEncodingType, }; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{SliceExt, highbit32}; /// `-log2(x / 256)` in units of 1/256 of a bit, for `x` from 1 to 255. The entry for 0 is not used. @@ -81,7 +81,7 @@ pub(crate) fn fse_bit_cost(ctable: &FseCTable, count: &[u32], max: u32) -> Resul const K_ACCURACY_LOG: u32 = 8; let mut cost: usize = 0; if ctable.max_symbol_value < max { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } let bad_cost = (ctable.table_log + 1) << K_ACCURACY_LOG; for (s, &c) in count.sub(..=max as usize)?.iter().enumerate() { @@ -90,7 +90,7 @@ pub(crate) fn fse_bit_cost(ctable: &FseCTable, count: &[u32], max: u32) -> Resul } let bit_cost = ctable.bit_cost(s as u32, K_ACCURACY_LOG); if bit_cost >= bad_cost { - return Err(ErrorCode::Generic); + return Err(Error::Generic); } cost += c as usize * bit_cost as usize; } @@ -156,12 +156,12 @@ pub(crate) fn select_encoding_type( let basic_cost = cost_or_error(if is_default_allowed { cross_entropy_cost(default_norm, default_norm_log, count, max) } else { - Err(ErrorCode::Generic) + Err(Error::Generic) }); let repeat_cost = cost_or_error(if *repeat_mode != FseRepeat::None { fse_bit_cost(prev_ctable, count, max) } else { - Err(ErrorCode::Generic) + Err(Error::Generic) }); let ncount_cost = cost_or_error(ncount_cost(count, max, nb_seq, fse_log)); let compressed_cost = (ncount_cost << 3).wrapping_add(entropy_cost(count, max, nb_seq)?); @@ -296,7 +296,7 @@ pub(crate) fn build_ctable( SymbolEncodingType::Rle => { next_ctable.build_rle(max as u8)?; let Some(first) = dst.first_mut() else { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); }; *first = max as u8; Ok(1) @@ -816,5 +816,5 @@ pub(crate) fn encode_sequences( // The stream has to end before the last position that a container can be written to. .filter(|nb_bits| (nb_bits >> 3) + size_of::() < capacity) .map(|nb_bits| nb_bits.div_ceil(8)) - .ok_or(ErrorCode::DstSizeTooSmall) + .ok_or(Error::DstSizeTooSmall) } diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs index 79e7a13ff5b5..914930c5db48 100644 --- a/src/zstd/compress/stream.rs +++ b/src/zstd/compress/stream.rs @@ -10,7 +10,7 @@ use super::params::{ get_c_params_from_cctx_params, get_params_internal, resolve_block_splitter_mode, resolve_enable_ldm, resolve_row_match_finder_mode, }; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{SliceExt, try_copy}; use crate::{InBuffer, OutBuffer}; @@ -80,7 +80,7 @@ fn resize_buffer(buffer: &mut Vec, size: &mut usize, new_size: usize) -> Res *size = 0; buffer .try_reserve_exact(new_size) - .map_err(|_| ErrorCode::MemoryAllocation)?; + .map_err(|_| Error::MemoryAllocation)?; *size = new_size; } Ok(()) @@ -124,7 +124,7 @@ impl CCtx { /// `ZSTD_CCtx_setParameter`: `param` is one of [`super::params::c_param`]. pub fn set_parameter(&mut self, param: i32, value: i32) -> Result { if self.stream_stage != StreamStage::Init { - return Err(ErrorCode::StageWrong); + return Err(Error::StageWrong); } self.requested_params.set_parameter(param, value) } @@ -132,7 +132,7 @@ impl CCtx { /// `ZSTD_CCtx_setPledgedSrcSize` pub fn set_pledged_src_size(&mut self, pledged_src_size: u64) -> Result<()> { if self.stream_stage != StreamStage::Init { - return Err(ErrorCode::StageWrong); + return Err(Error::StageWrong); } self.frame.pledged_src_size_plus_one = pledged_src_size.wrapping_add(1); Ok(()) @@ -142,7 +142,7 @@ impl CCtx { /// with no dictionary. pub fn load_dictionary(&mut self, dict: &[u8]) -> Result<()> { if self.stream_stage != StreamStage::Init { - return Err(ErrorCode::StageWrong); + return Err(Error::StageWrong); } self.local_dict = None; self.cdict = None; @@ -168,7 +168,7 @@ impl CCtx { } // Whatever is wrong with the dictionary, the C code only knows that it got no `ZSTD_CDict`. let cdict = CDict::new(Arc::clone(dict), &self.requested_params) - .map_err(|_| ErrorCode::MemoryAllocation)?; + .map_err(|_| Error::MemoryAllocation)?; self.cdict = Some(Arc::new(cdict)); Ok(()) } @@ -212,7 +212,7 @@ impl CCtx { if self.mtctx.is_none() { self.mtctx = Some(Box::new(MtCtx::new(params.nb_workers as u32)?)); } - let mtctx = self.mtctx.as_mut().ok_or(ErrorCode::MemoryAllocation)?; + let mtctx = self.mtctx.as_mut().ok_or(Error::MemoryAllocation)?; mtctx.init_c_stream_internal(self.cdict.clone(), params, pledged_src_size)?; self.frame.dict_id = cdict.map_or(0, CDict::dict_id); self.frame.dict_content_size = cdict.map_or(0, |cdict| cdict.dict_content().len()); @@ -285,7 +285,7 @@ impl CCtx { loop { match self.stream_stage { - StreamStage::Init => return Err(ErrorCode::InitMissing), + StreamStage::Init => return Err(Error::InitMissing), StreamStage::Load => { if flush_mode == EndDirective::End @@ -425,10 +425,10 @@ impl CCtx { end_op: EndDirective, ) -> Result { if output.pos > output.dst.len() { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } if input.pos > input.src.len() { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } if self.stream_stage == StreamStage::Init { @@ -436,7 +436,7 @@ impl CCtx { } if self.frame.applied_params.nb_workers > 0 { - let mtctx = self.mtctx.as_mut().ok_or(ErrorCode::Generic)?; + let mtctx = self.mtctx.as_mut().ok_or(Error::Generic)?; loop { let ipos = input.pos; let opos = output.pos; @@ -448,7 +448,7 @@ impl CCtx { let consumed = self.frame.consumed_src_size + 1; let is_complete = end_op == EndDirective::End && flush_min == Ok(0); if pledged != 0 && (consumed > pledged || (is_complete && consumed != pledged)) { - flush_min = flush_min.and(Err(ErrorCode::SrcSizeWrong)); + flush_min = flush_min.and(Err(Error::SrcSizeWrong)); } if flush_min.is_err() || (end_op == EndDirective::End && flush_min == Ok(0)) { // `reset_session`, but `mtctx` is borrowed. diff --git a/src/zstd/compress/superblock.rs b/src/zstd/compress/superblock.rs index c6bb28bab0b5..452677b8fe2b 100644 --- a/src/zstd/compress/superblock.rs +++ b/src/zstd/compress/superblock.rs @@ -18,7 +18,7 @@ use crate::common::{ MAX_LL, MAX_ML, MINMATCH, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, SymbolEncodingType, ZSTD_BLOCKHEADERSIZE, }; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{SliceExt, write_le24, write_le32}; const ZSTD_TARGETCBLOCKSIZE_MIN: usize = 1340; @@ -131,7 +131,7 @@ fn compress_sub_block_sequences( ) -> Result<(usize, bool)> { let nb_seq = sequences.len(); if dst.len() < 3 + 1 { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } let mut op = write_nb_seq(dst, nb_seq)?; if nb_seq == 0 { @@ -283,7 +283,7 @@ fn estimate_sub_block_size_symbol_type( SymbolEncodingType::Basic if max <= default_max => { cross_entropy_cost(default_norm, default_norm_log, count, max) } - SymbolEncodingType::Basic => Err(ErrorCode::Generic), + SymbolEncodingType::Basic => Err(Error::Generic), SymbolEncodingType::Rle => Ok(0), SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { fse_bit_cost(fse_c_table, count, max) @@ -373,7 +373,7 @@ fn size_block_sequences( avg_seq_cost: usize, first_sub_block: bool, ) -> Result { - let (first, rest) = sp.split_first().ok_or(ErrorCode::Generic)?; + let (first, rest) = sp.split_first().ok_or(Error::Generic)?; // The first sub-block also has the tables. let header_size = usize::from(first_sub_block) * 120 * BYTESCALE; let mut budget = header_size; diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index eb0c52d7fdd6..655f652e1a6d 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -17,7 +17,7 @@ use crate::common::{ MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, MINMATCH, ML_FSE_LOG, OFF_FSE_LOG, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{BUG, SliceExt, read_le16, read_le24, read_le32}; /// `blockProperties_t` @@ -30,7 +30,7 @@ pub(crate) struct BlockProperties { /// `ZSTD_getcBlockSize`: reads a block header, and returns how many bytes of input the block takes after it. pub(crate) fn get_c_block_size(src: &[u8]) -> Result<(usize, BlockProperties)> { if src.len() < ZSTD_BLOCKHEADERSIZE { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } let c_block_header = read_le24(src, 0)?; let c_size = c_block_header >> 3; @@ -47,7 +47,7 @@ pub(crate) fn get_c_block_size(src: &[u8]) -> Result<(usize, BlockProperties)> { }; match block_type { BlockType::Rle => Ok((1, properties)), - BlockType::Reserved => Err(ErrorCode::CorruptionDetected), + BlockType::Reserved => Err(Error::CorruptionDetected), _ => Ok((c_size as usize, properties)), } } @@ -252,7 +252,7 @@ impl Scratch { if self.lit_buffer.len() < needed { self.lit_buffer .try_reserve(needed - self.lit_buffer.len()) - .map_err(|_| ErrorCode::MemoryAllocation)?; + .map_err(|_| Error::MemoryAllocation)?; self.lit_buffer.resize(needed, 0); } // Literals are copied by more than there are: this much can be read along with the last ones. @@ -320,7 +320,7 @@ impl BlockDecoder { streaming: Streaming, ) -> Result { if src.len() < MIN_CBLOCK_SIZE { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let first = src.at(0)?; let lit_enc_type = SymbolEncodingType::from_bits(first); @@ -330,10 +330,10 @@ impl BlockDecoder { match lit_enc_type { SymbolEncodingType::Repeat | SymbolEncodingType::Compressed => { if lit_enc_type == SymbolEncodingType::Repeat && !self.lit_entropy { - return Err(ErrorCode::DictionaryCorrupted); + return Err(Error::DictionaryCorrupted); } if src.len() < 5 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let lhc = read_le32(src, 0)? as usize; let (single_stream, lh_size, lit_size, lit_c_size) = match lhl_code { @@ -347,16 +347,16 @@ impl BlockDecoder { _ => (lhl_code == 0, 3, (lhc >> 4) & 0x3FF, (lhc >> 14) & 0x3FF), }; if lit_size > block_size_max { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } if !single_stream && lit_size < MIN_LITERALS_FOR_4_STREAMS { - return Err(ErrorCode::LiteralsHeaderWrong); + return Err(Error::LiteralsHeaderWrong); } if lit_c_size + lh_size > src.len() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } if expected_write_size < lit_size { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } let location = lit_location(block_size_max, dst_capacity, lit_size, streaming); let compressed = src.span(lh_size, lit_c_size)?; @@ -399,7 +399,7 @@ impl BlockDecoder { ) }; if huf_result.is_err() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } self.lit_entropy = true; @@ -419,21 +419,21 @@ impl BlockDecoder { 1 => (2, usize::from(read_le16(src, 0)? >> 4)), 3 => { if src.len() < 3 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } (3, (read_le24(src, 0)? >> 4) as usize) } _ => (1, usize::from(first >> 3)), }; if lit_size > block_size_max { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } if expected_write_size < lit_size { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } if lh_size + lit_size + WILDCOPY_OVERLENGTH > src.len() { if lit_size + lh_size > src.len() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } self.scratch() .lit_buffer(lit_size)? @@ -457,23 +457,23 @@ impl BlockDecoder { let (lh_size, lit_size) = match lhl_code { 1 => { if src.len() < 3 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } (2, usize::from(read_le16(src, 0)? >> 4)) } 3 => { if src.len() < 4 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } (3, (read_le24(src, 0)? >> 4) as usize) } _ => (1, usize::from(first >> 3)), }; if lit_size > block_size_max { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } if expected_write_size < lit_size { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } self.scratch().lit_buffer(lit_size)?.fill(src.at(lh_size)?); Ok(Literals { @@ -489,7 +489,7 @@ impl BlockDecoder { /// `ZSTD_decodeSeqHeaders`: returns how many sequences there are, and the size of the header. fn decode_seq_headers(&mut self, src: &[u8]) -> Result<(usize, usize)> { if src.len() < MIN_SEQUENCES_SIZE { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } let mut ip = 0usize; let mut nb_seq = usize::from(src.at(ip)?); @@ -497,13 +497,13 @@ impl BlockDecoder { if nb_seq > 0x7F { if nb_seq == 0xFF { if ip + 2 > src.len() { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } nb_seq = usize::from(read_le16(src, ip)?) + LONGNBSEQ; ip += 2; } else { if ip >= src.len() { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } nb_seq = ((nb_seq - 0x80) << 8) + usize::from(src.at(ip)?); ip += 1; @@ -512,17 +512,17 @@ impl BlockDecoder { if nb_seq == 0 { if ip != src.len() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } return Ok((0, ip)); } if ip + 1 > src.len() { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } let modes = src.at(ip)?; if modes & 3 != 0 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } ip += 1; @@ -543,7 +543,7 @@ impl BlockDecoder { workspace, fse_entropy, ) - .map_err(|_| ErrorCode::CorruptionDetected)?; + .map_err(|_| Error::CorruptionDetected)?; ip += build_seq_table( &mut seq.of, &mut self.seq_sources.of, @@ -555,7 +555,7 @@ impl BlockDecoder { workspace, fse_entropy, ) - .map_err(|_| ErrorCode::CorruptionDetected)?; + .map_err(|_| Error::CorruptionDetected)?; ip += build_seq_table( &mut seq.ml, &mut self.seq_sources.ml, @@ -567,7 +567,7 @@ impl BlockDecoder { workspace, fse_entropy, ) - .map_err(|_| ErrorCode::CorruptionDetected)?; + .map_err(|_| Error::CorruptionDetected)?; Ok((nb_seq, ip)) } @@ -589,7 +589,7 @@ impl BlockDecoder { ext_dict, } = out; if src.len() > block_size_max { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } let literals = @@ -603,7 +603,7 @@ impl BlockDecoder { let sequences = sequences.sub(seq_h_size..)?; if capacity == 0 && nb_seq > 0 { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } self.scratch(); @@ -726,10 +726,10 @@ fn build_seq_table( match encoding_type { SymbolEncodingType::Rle => { let Some(&symbol) = src.first() else { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); }; if u32::from(symbol) > max { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } table.build_rle(codes.at(usize::from(symbol))?); *source = TableSource::Local; @@ -741,15 +741,15 @@ fn build_seq_table( } SymbolEncodingType::Repeat => { if !flag_repeat_table { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } Ok(0) } SymbolEncodingType::Compressed => { let header = read_ncount(&mut workspace.counts, max, src) - .map_err(|_| ErrorCode::CorruptionDetected)?; + .map_err(|_| Error::CorruptionDetected)?; if header.table_log > max_log { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } table.build( header.max_symbol_value as usize + 1, @@ -887,7 +887,7 @@ struct SeqState<'a> { impl<'a> SeqState<'a> { fn new(src: &'a [u8], tables: SeqTableRefs<'a>, rep: &[u32; ZSTD_REP_NUM]) -> Result { - let mut bit_d = BitDStream::new(src).map_err(|_| ErrorCode::CorruptionDetected)?; + let mut bit_d = BitDStream::new(src).map_err(|_| Error::CorruptionDetected)?; // `ZSTD_initFseState` let mut init = |table_log: u32| { let state = bit_d.read_bits(table_log) as usize; @@ -1251,17 +1251,17 @@ impl<'a> Executor<'_, 'a> { let mut op = self.op; if lit_length + match_length > self.oend - op { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } self.out .extend_to(op + lit_length + match_length, self.oend); if lit_length > self.lits_left() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } if self.split { let lit_ptr = self.lit_end_in_dst.wrapping_sub(self.lits_left()); if op > lit_ptr && op < lit_ptr.wrapping_add(lit_length) { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } } @@ -1276,7 +1276,7 @@ impl<'a> Executor<'_, 'a> { let mut match_pos = op.wrapping_sub(offset); if offset > in_prefix { if offset > self.ext_dict.reach(in_prefix) { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let back = offset - in_prefix; if back >= match_length { @@ -1419,7 +1419,7 @@ impl<'a> Executor<'_, 'a> { let leftover_lit = self.lits_left(); if leftover_lit > 0 { if leftover_lit > self.oend - self.op { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } self.out.extend_to(self.op + leftover_lit, self.oend); self.out @@ -1719,7 +1719,7 @@ impl<'a> Executor<'_, 'a> { None => seq_state.bit_d.is_at_end(), }; if !is_at_end { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } for &sequence in &*ahead { self.exec_sequence_maybe_split(sequence)?; @@ -1778,7 +1778,7 @@ impl<'a> Executor<'_, 'a> { let parts = parts.map(Option::unwrap_or_default); let size = parts.iter().map(|part| part.len()).sum::(); if size > self.oend - self.op { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } self.out.extend_to(self.op + size, self.op + size); for part in parts { diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index 27348ee7df98..ccda8954d847 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -7,7 +7,7 @@ use super::huf_exact::{self, TableType, select_decoder_x2}; use crate::common::entropy::huf_read_stats_wksp; use crate::common::fse_decompress::Spread; use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{BUG, SliceExt, read_le16}; const NB_SYMBOLS: usize = HUF_SYMBOLVALUE_MAX as usize + 1; @@ -868,7 +868,7 @@ pub(crate) fn decompress_1x1( ) -> Result { let h_size = dtable.read(src, TableType::X1, dst.len(), spread)?; if h_size >= src.len() { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } decompress_1x_using_dtable(dst, src.sub(h_size..)?, dtable) } @@ -893,10 +893,10 @@ pub(crate) fn decompress_4x_huf_only( spread: &mut Spread, ) -> Result { if dst.is_empty() { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } if src.is_empty() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let table_type = if select_decoder_x2(dst.len(), src.len())? { TableType::X2 @@ -905,7 +905,7 @@ pub(crate) fn decompress_4x_huf_only( }; let h_size = dtable.read(src, table_type, dst.len(), spread)?; if h_size >= src.len() { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } decompress_4x_using_dtable(dst, src.sub(h_size..)?, dtable) } diff --git a/src/zstd/decompress/huf_exact.rs b/src/zstd/decompress/huf_exact.rs index b5772355e3f4..1effda19066f 100644 --- a/src/zstd/decompress/huf_exact.rs +++ b/src/zstd/decompress/huf_exact.rs @@ -4,7 +4,7 @@ use super::huf::Code; use crate::common::bitstream::{BitDStream, DStreamStatus}; use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{BUG, SliceExt, highbit32, read_le16, read_le64}; /// Tables of this log are decoded by the loops that keep four streams in registers. @@ -118,7 +118,7 @@ impl FastArgs { return Ok(None); } if src.len() < 10 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } if table_log != HUF_DECODER_FAST_TABLELOG { return Ok(None); @@ -129,7 +129,7 @@ impl FastArgs { } let [_, _, _, length4] = streams.length; if length4 > src.len() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let [_, start2, start3, start4] = streams.start; let ip = [start2 - 8, start3 - 8, start4 - 8, src.len() - 8]; @@ -160,11 +160,11 @@ impl FastArgs { segment_end: usize, ) -> Result> { if self.op.at(stream)? > segment_end { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let ip = self.ip.at(stream)?; if ip + 8 < self.iend.at(stream)? { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } BitDStream::resume(src, ip, self.bits.at(stream)?.trailing_zeros()) } @@ -246,7 +246,7 @@ fn build_dtable_x1(dtable: &mut ExactTable, stats: &Code) -> Result<()> { let target_table_log = max_table_log.min(HUF_DECODER_FAST_TABLELOG); let table_log = rescale_stats(weights, &mut rank_val, stats.table_log, target_table_log)?; if table_log > max_table_log { - return Err(ErrorCode::TableLogTooLarge); + return Err(Error::TableLogTooLarge); } dtable.table_type = TableType::X1; dtable.table_log = table_log; @@ -326,7 +326,7 @@ fn decompress_1x1_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) let mut bit_d = BitDStream::new(src)?; decode_stream_x1(dst, &mut bit_d, &dtable.x1, dtable.table_log); if !bit_d.is_at_end() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } Ok(dst.len()) } @@ -334,10 +334,10 @@ fn decompress_1x1_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) /// `HUF_decompress4X1_usingDTable_internal_body` fn decompress_4x1_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { if src.len() < 10 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } if dst.len() < 6 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let dst_size = dst.len(); let dt = &dtable.x1; @@ -345,11 +345,11 @@ fn decompress_4x1_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Resul let streams = Streams::new(src)?; let [_, _, _, length4] = streams.length; if length4 > src.len() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let segment_size = dst_size.div_ceil(4); if 3 * segment_size > dst_size { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let [mut bit_d1, mut bit_d2, mut bit_d3, mut bit_d4] = streams.open(src)?; @@ -385,7 +385,7 @@ fn decompress_4x1_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Resul decode_stream_x1(out4.sub_mut(done..)?, &mut bit_d4, dt, dt_log); if !(bit_d1.is_at_end() & bit_d2.is_at_end() & bit_d3.is_at_end() & bit_d4.is_at_end()) { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } Ok(dst_size) } @@ -611,7 +611,7 @@ fn build_dtable_x2(dtable: &mut ExactTable, stats: &Code) -> Result<()> { let table_log = stats.table_log; if table_log > max_table_log { - return Err(ErrorCode::TableLogTooLarge); + return Err(Error::TableLogTooLarge); } if table_log <= HUF_DECODER_FAST_TABLELOG && max_table_log > HUF_DECODER_FAST_TABLELOG { max_table_log = HUF_DECODER_FAST_TABLELOG; @@ -764,7 +764,7 @@ fn decompress_1x2_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) let mut bit_d = BitDStream::new(src)?; decode_stream_x2(dst, &mut bit_d, &dtable.x2, dtable.table_log)?; if !bit_d.is_at_end() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } Ok(dst.len()) } @@ -772,10 +772,10 @@ fn decompress_1x2_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) /// `HUF_decompress4X2_usingDTable_internal_body` fn decompress_4x2_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { if src.len() < 10 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } if dst.len() < 6 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let oend = dst.len(); let dt = &dtable.x2; @@ -783,14 +783,14 @@ fn decompress_4x2_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Resul let streams = Streams::new(src)?; let [_, _, _, length4] = streams.length; if length4 > src.len() { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let segment_size = oend.div_ceil(4); let op_start2 = segment_size; let op_start3 = 2 * segment_size; let op_start4 = 3 * segment_size; if op_start4 > oend { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } let [mut bit_d1, mut bit_d2, mut bit_d3, mut bit_d4] = streams.open(src)?; let (mut op1, mut op2, mut op3, mut op4) = (0, op_start2, op_start3, op_start4); @@ -819,7 +819,7 @@ fn decompress_4x2_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Resul } if op1 > op_start2 || op2 > op_start3 || op3 > op_start4 { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } decode_stream_x2(dst.sub_mut(op1..op_start2)?, &mut bit_d1, dt, dt_log)?; @@ -828,7 +828,7 @@ fn decompress_4x2_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Resul decode_stream_x2(dst.sub_mut(op4..oend)?, &mut bit_d4, dt, dt_log)?; if !(bit_d1.is_at_end() & bit_d2.is_at_end() & bit_d3.is_at_end() & bit_d4.is_at_end()) { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } Ok(oend) } diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 32ba78b676f9..8c7239c271a0 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -23,7 +23,7 @@ use crate::common::{ ZSTD_WINDOWLOG_LIMIT_DEFAULT, ZSTD_WINDOWLOG_MAX, ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, }; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; use crate::util::{SliceExt, read_le16, read_le32, read_le64, try_copy}; use crate::{InBuffer, OutBuffer}; @@ -84,7 +84,7 @@ fn is_skippable_magic(magic: u32) -> bool { /// `ZSTD_frameHeaderSize_internal` fn frame_header_size(src: &[u8]) -> Result { if src.len() < FRAMEHEADERSIZE_PREFIX { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } let fhd = src.at(FRAMEHEADERSIZE_PREFIX - 1)?; let dict_id = usize::from(fhd & 3); @@ -111,7 +111,7 @@ fn get_frame_header(header: &mut FrameHeader, src: &[u8]) -> Result Result Result Result> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; if window_log > ZSTD_WINDOWLOG_MAX { - return Err(ErrorCode::FrameParameterWindowTooLarge); + return Err(Error::FrameParameterWindowTooLarge); } window_size = 1u64 << window_log; window_size += (window_size >> 3) * u64::from(wl_byte & 7); @@ -221,14 +221,14 @@ pub fn get_frame_content_size(src: &[u8]) -> u64 { /// The size of the skippable frame whose header is at the start of `src`. fn skippable_frame_size(src: &[u8]) -> Result { if src.len() < ZSTD_SKIPPABLEHEADERSIZE { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } let size_u32 = read_le32(src, ZSTD_FRAMEIDSIZE)?; if size_u32 .checked_add(ZSTD_SKIPPABLEHEADERSIZE as u32) .is_none() { - return Err(ErrorCode::FrameParameterUnsupported); + return Err(Error::FrameParameterUnsupported); } Ok(ZSTD_SKIPPABLEHEADERSIZE + size_u32 as usize) } @@ -237,7 +237,7 @@ fn skippable_frame_size(src: &[u8]) -> Result { fn read_skippable_frame_size(src: &[u8]) -> Result { let skippable_size = skippable_frame_size(src)?; if skippable_size > src.len() { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } Ok(skippable_size) } @@ -292,7 +292,7 @@ pub fn frame_extent(src: &[u8]) -> Result { fn find_frame_compressed_size(src: &[u8]) -> Result { match frame_extent(src)? { FrameExtent::Whole(size) => Ok(size), - FrameExtent::AtLeast(_) => Err(ErrorCode::SrcSizeWrong), + FrameExtent::AtLeast(_) => Err(Error::SrcSizeWrong), } } @@ -329,7 +329,7 @@ pub(crate) fn find_decompressed_size(mut src: &[u8]) -> u64 { /// `ZSTD_loadDEntropy`: reads the tables at the start of a dictionary, and returns the last offsets that it sets. fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result<[u32; ZSTD_REP_NUM]> { - const CORRUPTED: ErrorCode = ErrorCode::DictionaryCorrupted; + const CORRUPTED: Error = Error::DictionaryCorrupted; if dict.len() <= 8 { return Err(CORRUPTED); } @@ -438,8 +438,7 @@ impl<'a> DDict<'a> { } ddict.dict_id = read_le32(dict, ZSTD_FRAMEIDSIZE)?; let mut entropy = Box::new(EntropyDTables::new()); - ddict.rep = - load_d_entropy(&mut entropy, dict).map_err(|_| ErrorCode::DictionaryCorrupted)?; + ddict.rep = load_d_entropy(&mut entropy, dict).map_err(|_| Error::DictionaryCorrupted)?; ddict.entropy = Some(entropy); Ok(ddict) } @@ -514,7 +513,7 @@ impl Buffer { let mut bytes = Vec::new(); bytes .try_reserve_exact(size) - .map_err(|_| ErrorCode::MemoryAllocation)?; + .map_err(|_| Error::MemoryAllocation)?; Ok(Self { bytes, size }) } @@ -641,13 +640,13 @@ impl DCtx { /// `ZSTD_DCtx_loadDictionary`: `dict` is copied, and used for every frame from now on. None if it is empty. pub fn load_dictionary(&mut self, dict: &[u8]) -> Result<()> { if self.state.stream_stage != DStreamStage::Init { - return Err(ErrorCode::StageWrong); + return Err(Error::StageWrong); } self.ddict = None; if !dict.is_empty() { // The C function has nothing else to report a dictionary that it cannot make sense of as. let ddict = DDict::new(Cow::Owned(try_copy(dict)?), DictContentType::Auto) - .map_err(|_| ErrorCode::MemoryAllocation)?; + .map_err(|_| Error::MemoryAllocation)?; self.ddict = Some(Box::new(ddict)); } Ok(()) @@ -656,16 +655,16 @@ impl DCtx { /// `ZSTD_DCtx_setParameter` pub fn set_parameter(&mut self, param: i32, mut value: i32) -> Result<()> { if self.state.stream_stage != DStreamStage::Init { - return Err(ErrorCode::StageWrong); + return Err(Error::StageWrong); } if param != ZSTD_D_WINDOW_LOG_MAX { - return Err(ErrorCode::ParameterUnsupported); + return Err(Error::ParameterUnsupported); } if value == 0 { value = ZSTD_WINDOWLOG_LIMIT_DEFAULT as i32; } if value < ZSTD_WINDOWLOG_ABSOLUTEMIN as i32 || value > ZSTD_WINDOWLOG_MAX as i32 { - return Err(ErrorCode::ParameterOutOfBound); + return Err(Error::ParameterOutOfBound); } self.state.max_window_size = 1usize << value; Ok(()) @@ -698,7 +697,7 @@ pub(crate) fn decompress(dst: &mut [u8], src: &[u8]) -> Result { /// `ZSTD_copyRawBlock` fn copy_raw_block(dst: &mut [u8], src: &[u8]) -> Result { let Some(dst) = dst.get_mut(..src.len()) else { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); }; dst.copy_from_slice(src); Ok(src.len()) @@ -707,7 +706,7 @@ fn copy_raw_block(dst: &mut [u8], src: &[u8]) -> Result { /// `ZSTD_setRleBlock` fn set_rle_block(dst: &mut [u8], byte: u8, regen_size: usize) -> Result { let Some(dst) = dst.get_mut(..regen_size) else { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); }; dst.fill(byte); Ok(regen_size) @@ -774,10 +773,10 @@ impl DState { /// `ZSTD_decodeFrameHeader`: `src` is all of a frame header. fn decode_frame_header(&mut self, src: &[u8]) -> Result<()> { if let HeaderStatus::Needs(_) = get_frame_header(&mut self.f_params, src)? { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } if self.f_params.dict_id != 0 && self.dict_id != self.f_params.dict_id { - return Err(ErrorCode::DictionaryWrong); + return Err(Error::DictionaryWrong); } self.validate_checksum = self.f_params.checksum_flag; if self.validate_checksum { @@ -832,12 +831,12 @@ impl DState { let mut op = ostart; if src.len() < FRAMEHEADERSIZE_MIN + ZSTD_BLOCKHEADERSIZE { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } { let frame_header_size = frame_header_size(src.sub(..FRAMEHEADERSIZE_PREFIX)?)?; if src.len() < frame_header_size + ZSTD_BLOCKHEADERSIZE { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } self.decode_frame_header(src.sub(..frame_header_size)?)?; ip += frame_header_size; @@ -847,7 +846,7 @@ impl DState { let (c_block_size, properties) = get_c_block_size(src.sub(ip..)?)?; ip += ZSTD_BLOCKHEADERSIZE; if c_block_size > src.len() - ip { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } let block = src.span(ip, c_block_size)?; @@ -864,7 +863,7 @@ impl DState { )?; // Only a streaming decoder checks that in 1.5.7, where such a block can overwrite its own literals. if decoded_size > self.f_params.block_size_max { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } decoded_size } @@ -874,7 +873,7 @@ impl DState { block.at(0)?, properties.orig_size as usize, )?, - BlockType::Reserved => return Err(ErrorCode::CorruptionDetected), + BlockType::Reserved => return Err(Error::CorruptionDetected), }; if self.validate_checksum { self.xxh_state.update(dst.span(op, decoded_size)?); @@ -889,14 +888,14 @@ impl DState { if self.f_params.frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN && (op - ostart) as u64 != self.f_params.frame_content_size { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } if self.f_params.checksum_flag { if src.len() - ip < 4 { - return Err(ErrorCode::ChecksumWrong); + return Err(Error::ChecksumWrong); } if read_le32(src, ip)? != self.xxh_state.digest() as u32 { - return Err(ErrorCode::ChecksumWrong); + return Err(Error::ChecksumWrong); } ip += 4; } @@ -925,8 +924,8 @@ impl DState { match self.decompress_frame(dst, op, src, ddict) { // More likely `src` is longer than its frames than damaged where a magic number is expected. - Err(ErrorCode::PrefixUnknown) if more_than_1_frame => { - return Err(ErrorCode::SrcSizeWrong); + Err(Error::PrefixUnknown) if more_than_1_frame => { + return Err(Error::SrcSizeWrong); } Err(err) => return Err(err), Ok((written, read)) => { @@ -938,7 +937,7 @@ impl DState { } if !src.is_empty() { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } Ok(op - dst_start) } @@ -966,7 +965,7 @@ impl DState { ddict: Option<&DDict<'_>>, ) -> Result { if src.len() != self.next_src_size_to_decompress_with_input_size(src.len()) { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } let dst = self.out_start; self.check_continuity(dst, dst_capacity); @@ -975,7 +974,7 @@ impl DState { DStage::DecodeBlockHeader => { let (c_block_size, bp) = get_c_block_size(src)?; if c_block_size > self.f_params.block_size_max { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } self.expected = c_block_size; self.b_type = bp.block_type; @@ -1022,12 +1021,12 @@ impl DState { .span_mut(dst, dst_capacity.min(self.rle_size)) .and_then(|dst| set_rle_block(dst, src.at(0)?, self.rle_size)) } - BlockType::Reserved => Err(ErrorCode::CorruptionDetected), + BlockType::Reserved => Err(Error::CorruptionDetected), }; self.out_buff = out_buff; let r_size = r_size?; if r_size > self.f_params.block_size_max { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } self.decoded_size += r_size as u64; if self.validate_checksum { @@ -1045,7 +1044,7 @@ impl DState { if self.f_params.frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN && self.decoded_size != self.f_params.frame_content_size { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } self.end_of_blocks(); } else { @@ -1057,7 +1056,7 @@ impl DState { DStage::CheckChecksum => { if self.validate_checksum && read_le32(src, 0)? != self.xxh_state.digest() as u32 { - return Err(ErrorCode::ChecksumWrong); + return Err(Error::ChecksumWrong); } self.expected = 0; self.stage = DStage::GetFrameHeaderSize; @@ -1071,7 +1070,7 @@ impl DState { } // `ZSTD_decompressStream` reads frame headers itself, and is done when it gets back here. - DStage::GetFrameHeaderSize => Err(ErrorCode::Generic), + DStage::GetFrameHeaderSize => Err(Error::Generic), } } @@ -1146,10 +1145,10 @@ impl DState { ddict: Option<&DDict<'_>>, ) -> Result { if input.pos > input.src.len() { - return Err(ErrorCode::SrcSizeWrong); + return Err(Error::SrcSizeWrong); } if output.pos > output.dst.len() { - return Err(ErrorCode::DstSizeTooSmall); + return Err(Error::DstSizeTooSmall); } let src = input.src; let istart = input.pos; @@ -1233,7 +1232,7 @@ impl DState { .window_size .max(1 << ZSTD_WINDOWLOG_ABSOLUTEMIN); if self.f_params.window_size > self.max_window_size as u64 { - return Err(ErrorCode::FrameParameterWindowTooLarge); + return Err(Error::FrameParameterWindowTooLarge); } self.resize_buffers()?; @@ -1265,7 +1264,7 @@ impl DState { let loaded_size = to_load.min(iend - ip); if self.stage != DStage::SkipFrame { if to_load > self.in_buff.size - self.in_pos { - return Err(ErrorCode::CorruptionDetected); + return Err(Error::CorruptionDetected); } self.in_buff.bytes.truncate(self.in_pos); self.in_buff @@ -1321,10 +1320,10 @@ impl DState { self.no_forward_progress += 1; if self.no_forward_progress >= ZSTD_NO_FORWARD_PROGRESS_MAX { if op == oend { - return Err(ErrorCode::NoForwardProgressDestFull); + return Err(Error::NoForwardProgressDestFull); } if ip == iend { - return Err(ErrorCode::NoForwardProgressInputEmpty); + return Err(Error::NoForwardProgressInputEmpty); } } } else { diff --git a/src/zstd/error.rs b/src/zstd/error.rs index 2992bd86808b..d216d0bf82c4 100644 --- a/src/zstd/error.rs +++ b/src/zstd/error.rs @@ -4,8 +4,9 @@ use core::ffi::CStr; /// `ZSTD_ErrorCode`, without `ZSTD_error_no_error`. The values are part of `node:zlib`'s constants. #[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ErrorCode { +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("{}", self.message().to_str().unwrap_or_default())] +pub enum Error { Generic = 1, PrefixUnknown = 10, VersionUnsupported = 12, @@ -36,9 +37,9 @@ pub enum ErrorCode { NoForwardProgressInputEmpty = 82, } -pub(crate) type Result = core::result::Result; +pub type Result = core::result::Result; -impl ErrorCode { +impl Error { /// `ZSTD_getErrorString`. pub const fn message(self) -> &'static CStr { match self { @@ -120,3 +121,45 @@ impl ErrorCode { } } } + +/// What the functions that allocate their output fail with. Users get to see the names. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error, strum::IntoStaticStr)] +pub enum ZstdError { + /// The output, or the decoder state the frame's window size dictates, could not be allocated. + #[error("OutOfMemory")] + OutOfMemory, + #[error("InvalidZstdData")] + InvalidZstdData, + #[error("DecompressionFailed")] + DecompressionFailed, + #[error("ZstdDecompressionError")] + ZstdDecompressionError, + #[error("ShortRead")] + ShortRead, +} + +impl ZstdError { + /// The error for a failed decompression call; `other` is the non-allocation failure. + pub(crate) fn for_decompression(code: Error, other: ZstdError) -> ZstdError { + if code == Error::MemoryAllocation { + ZstdError::OutOfMemory + } else { + other + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn errors_show_what_c_and_users_know_them_by() { + assert_eq!( + Error::ChecksumWrong.to_string(), + "Restored data doesn't match checksum" + ); + assert_eq!(ZstdError::ShortRead.to_string(), "ShortRead"); + assert_eq!(<&'static str>::from(ZstdError::ShortRead), "ShortRead"); + } +} diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index a9da37f53683..d548d4e39f16 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -27,7 +27,7 @@ pub use decompress::{ DCtx, DDict, FrameExtent, ZSTD_D_WINDOW_LOG_MAX, frame_extent, get_frame_content_size, }; use decompress::{decompress, find_decompressed_size}; -pub use error::ErrorCode; +pub use error::{Error, Result, ZstdError}; // Callers share contexts between threads behind their own `unsafe impl`s, which should have nothing to vouch for here. const _: () = { @@ -79,7 +79,7 @@ impl<'a> VecOutput<'a> { /// Room for one call of a codec: `step` bytes, or more of them after much output, but never past `limit`. /// [`advance`](Self::advance) by what it wrote. - pub fn buffer(&mut self, step: usize, limit: usize) -> Result, ErrorCode> { + pub fn buffer(&mut self, step: usize, limit: usize) -> Result> { let room = self.vec.len() - self.filled; let budget = limit.saturating_sub(self.filled); if room < step.min(budget) { @@ -87,7 +87,7 @@ impl<'a> VecOutput<'a> { let more = (self.filled - self.start).clamp(step, most).min(budget) - room; self.vec .try_reserve(more) - .map_err(|_| ErrorCode::MemoryAllocation)?; + .map_err(|_| Error::MemoryAllocation)?; self.vec.resize(self.vec.len() + more, 0); } let end = self.vec.len().min(limit); @@ -110,37 +110,9 @@ impl Drop for VecOutput<'_> { // ----------------------------------- -#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] -pub enum ZstdError { - /// The output, or the decoder state the frame's window size dictates, could not be allocated. - OutOfMemory, - InvalidZstdData, - DecompressionFailed, - ZstdDecompressionError, - ShortRead, -} - -impl core::fmt::Display for ZstdError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.write_str(<&'static str>::from(self)) - } -} -impl core::error::Error for ZstdError {} - -impl ZstdError { - /// The error for a failed decompression call; `other` is the non-allocation failure. - fn for_decompression(code: ErrorCode, other: ZstdError) -> ZstdError { - if code == ErrorCode::MemoryAllocation { - ZstdError::OutOfMemory - } else { - other - } - } -} - /// [`compress`] to the end of `out`, within its capacity: [`compress_bound`] more than its length is always enough. /// Returns how many bytes are added. -pub fn compress_append(out: &mut Vec, src: &[u8], level: i32) -> Result { +pub fn compress_append(out: &mut Vec, src: &[u8], level: i32) -> Result { let start = out.len(); let room = (out.capacity() - start).min(compress_bound(src.len())); out.resize(start + room, 0); @@ -150,7 +122,7 @@ pub fn compress_append(out: &mut Vec, src: &[u8], level: i32) -> Result Result, ErrorCode> { +pub fn compress_alloc(src: &[u8], level: i32) -> Result> { let mut out = util::try_zeroed(compress_bound(src.len()))?; let size = compress(&mut out, src, level)?; out.truncate(size); diff --git a/src/zstd/util.rs b/src/zstd/util.rs index 145a299ca936..c534f1663002 100644 --- a/src/zstd/util.rs +++ b/src/zstd/util.rs @@ -2,10 +2,10 @@ use core::slice::SliceIndex; -use crate::error::{ErrorCode, Result}; +use crate::error::{Error, Result}; /// An index or a length that the algorithm keeps in range was out of range. No input should be able to cause it. -pub(crate) const BUG: ErrorCode = ErrorCode::Generic; +pub(crate) const BUG: Error = Error::Generic; pub(crate) trait SliceExt { fn at(&self, index: usize) -> Result @@ -161,16 +161,16 @@ pub(crate) const fn highbit32(value: u32) -> u32 { 31 - (value | 1).leading_zeros() } -/// `len` zeros, or [`ErrorCode::MemoryAllocation`]. An allocator that reuses memory has to clear all of it. +/// `len` zeros, or [`Error::MemoryAllocation`]. An allocator that reuses memory has to clear all of it. pub(crate) fn try_zeroed(len: usize) -> Result> { - bun_pod::try_zeroed_vec(len).ok_or(ErrorCode::MemoryAllocation) + bun_pod::try_zeroed_vec(len).ok_or(Error::MemoryAllocation) } -/// A copy of `bytes`, or [`ErrorCode::MemoryAllocation`]. +/// A copy of `bytes`, or [`Error::MemoryAllocation`]. pub(crate) fn try_copy(bytes: &[u8]) -> Result> { let mut vec = Vec::new(); vec.try_reserve_exact(bytes.len()) - .map_err(|_| ErrorCode::MemoryAllocation)?; + .map_err(|_| Error::MemoryAllocation)?; vec.extend_from_slice(bytes); Ok(vec) } From 0b92c0c35bcb56c9c1efa55253d29bd74bb1d695 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 02:59:42 +0000 Subject: [PATCH 138/199] zstd: workers give up a frame within a block or 4096 positions of a prefix A worker looked whether its frame was given up before each 512 KB, and not while it indexed the prefix of its job, which can take seconds. 30 streams at level 19 with 4 workers that were dropped in the middle of a frame left 93 threads with 3.5 GB behind for 24 s; now 8 threads with 380 MB for 0.04 s. Also, from a review of the last changes: - A context that is not a worker's lets go of its tables when the books say that the C code would replace its workspace, instead of by a rule of its own. - If the buffers of a stream cannot be allocated, the next frame starts from nothing, as they are part of the workspace in the C code. - decompress_alloc_with_prefix leaves room after its output to copy in chunks up to the end, as decompress_alloc does. - Hash chains use Hasher, which is Hash8 without a salt. - One function compares 16 bytes, for the three that did. - TaggedTable keeps no factor that it does not use. - The names of ZstdError are written once. --- src/zstd/compress/cctx.rs | 38 +++++++++----- src/zstd/compress/double_fast.rs | 2 - src/zstd/compress/fast.rs | 14 +++--- src/zstd/compress/internal.rs | 49 +++++++++--------- src/zstd/compress/lazy.rs | 22 ++++----- src/zstd/compress/mt.rs | 85 +++++++++++++++++++++++++++++--- src/zstd/compress/opt.rs | 27 +++++++--- src/zstd/compress/stream.rs | 16 +++--- src/zstd/decompress/block.rs | 4 +- src/zstd/error.rs | 6 +-- src/zstd/lib.rs | 20 ++++---- 11 files changed, 194 insertions(+), 89 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 14210d0c2a12..93b9c78e564a 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -2,6 +2,7 @@ //! and what they need. use std::sync::Arc; +use std::sync::atomic::AtomicBool; use super::block_entropy::{ build_entropy_statistics_and_estimate_sub_block_size, count_seq_store_literals_bytes, @@ -812,7 +813,7 @@ pub(crate) struct FrameCompressor { pub(crate) dict_content_size: usize, /// `None` for a worker, see [`Self::for_worker`]. workspace: Option, - /// For how many frames in a row the tables have been much larger than it takes. + /// For a worker: for how many frames in a row the tables have been much larger than it takes. oversized_duration: usize, pub(crate) block_size_max: usize, pub(crate) pledged_src_size_plus_one: u64, @@ -880,11 +881,18 @@ impl FrameCompressor { /// For the jobs of one worker. Which context a job goes to is a matter of timing in the C code, so what its contexts /// have been through cannot show in the output. - pub(crate) fn for_worker() -> Self { - Self { + pub(crate) fn for_worker(cancelled: Arc) -> Self { + let mut worker = Self { workspace: None, ..Self::new() - } + }; + worker.match_state.cancelled = Some(cancelled); + worker + } + + /// The next frame starts from nothing, as after a reset that failed. + pub(crate) fn forget(&mut self) { + self.initialized = false; } /// `ZSTD_resetCCtx_internal` @@ -947,14 +955,21 @@ impl FrameCompressor { IndexResetPolicy::Continue }; - // `ZSTD_cwksp_check_wasteful`: a context does not hold on for long to much more than it uses. - let needed = tables_size(¶ms.c_params, params.use_row_match_finder); - if tables_capacity(&self.match_state) < needed.saturating_mul(ZSTD_WORKSPACETOOLARGE_FACTOR) - { - self.oversized_duration = 0; - } else if self.oversized_duration < ZSTD_WORKSPACETOOLARGE_MAXDURATION { - self.oversized_duration += 1; + // A context does not hold on for long to much more than it uses. + let new_tables = if self.workspace.is_some() { + new_workspace } else { + let needed = tables_size(¶ms.c_params, params.use_row_match_finder); + let oversized = tables_capacity(&self.match_state) + >= needed.saturating_mul(ZSTD_WORKSPACETOOLARGE_FACTOR); + self.oversized_duration = if oversized { + self.oversized_duration + 1 + } else { + 0 + }; + self.oversized_duration > ZSTD_WORKSPACETOOLARGE_MAXDURATION + }; + if new_tables { self.oversized_duration = 0; let ms = &mut self.match_state; ms.hash_table = Table::new(); @@ -1654,6 +1669,7 @@ impl FrameCompressor { } while ip < src.end { + self.match_state.check_cancelled()?; let remaining = src.end - ip; let block_size = self.optimal_block_size(bytes.sub(ip - src.start..)?, savings)?; let last_block = last_frame_chunk && block_size == remaining; diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index fae0f1a607d5..f9391c640ba1 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -464,13 +464,11 @@ pub(crate) fn compress_block_double_fast_dict_match_state( long: TaggedTable::new( &dms.ms.hash_table, dms.ms.tagged_hash_log(), - 8, block.dict_start_index, )?, short: TaggedTable::new( &dms.ms.chain_table, dms.ms.tagged_chain_log(), - ms.c_params.min_match, block.dict_start_index, )?, }; diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 90541cb6fd8a..45cf4aab4fd2 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -74,6 +74,7 @@ pub(super) fn restore_rep( /// What `fast` and `dfast` index a dictionary by: every third position, with the 10 bytes there, which the 8 bytes at /// it and at the two positions in between are among. +/// `array_windows().step_by(3)` gives the same, and indexing a dictionary then takes 1.4 to 1.6 times as long. #[derive(Clone)] pub(super) struct ThirdPositions<'a> { /// From the first position that is left up to the end of the 10 bytes at the last one. @@ -155,6 +156,7 @@ pub(super) fn put_tagged_in_between( hasher: Hasher, positions: ThirdPositions<'_>, ) { + debug_assert!(table.iter().all(|&cell| cell == 0)); for at in positions.rev() { put_tagged(table, hasher.hash(le64_at::<2, _>(at.bytes)), at.index + 2); put_tagged(table, hasher.hash(le64_at::<1, _>(at.bytes)), at.index + 1); @@ -929,18 +931,17 @@ impl<'a> DictBlock<'a> { #[derive(Clone, Copy)] pub(super) struct TaggedTable<'a> { table: &'a [u32], - /// To a hash and a tag. - hasher: Hasher, + /// How many bits a hash and a tag have. + h_bits: u32, /// The index that the first place has in the table. start_index: u32, } impl<'a> TaggedTable<'a> { - /// `mls` is how many bytes are hashed. - pub(super) fn new(table: &'a [u32], h_bits: u32, mls: u32, start_index: u32) -> Result { + pub(super) fn new(table: &'a [u32], h_bits: u32, start_index: u32) -> Result { Ok(Self { table: table.sub(..1 << (h_bits - ZSTD_SHORT_CACHE_TAG_BITS))?, - hasher: Hasher::new(h_bits, mls), + h_bits, start_index, }) } @@ -949,7 +950,7 @@ impl<'a> TaggedTable<'a> { /// tells that it is for something else, or if it is the first place, which stands for none. #[inline(always)] pub(super) fn get(self, bytes: u64) -> Option { - let hash_and_tag = self.hasher.with_mls(MLS).hash(bytes); + let hash_and_tag = Hasher::new(self.h_bits, MLS).hash(bytes); let index_and_tag = hashed_get(self.table, hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS); if !compare_packed_tags(index_and_tag as usize, hash_and_tag) { return None; @@ -1086,7 +1087,6 @@ pub(crate) fn compress_block_fast_dict_match_state( let dict_table = TaggedTable::new( &dms.ms.hash_table, dms.ms.tagged_hash_log(), - ms.c_params.min_match, block.dict_start_index, )?; let hlog = ms.c_params.hash_log; diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 556e32393cd4..bce75ec9a0b3 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1,7 +1,8 @@ //! `zstd_compress_internal.h`: what the parts of the compressor share. use core::hint::{Locality, prefetch_read}; -use core::sync::atomic::{Ordering, compiler_fence}; +use core::sync::atomic::{AtomicBool, Ordering, compiler_fence}; +use std::sync::Arc; use super::fse::{FseCTable, FseRepeat}; use super::huf::{HufCTable, HufRepeat}; @@ -717,6 +718,16 @@ pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { #[derive(Clone, Copy)] pub(crate) struct Near<'a>(&'a [u8]); +/// A bit for each of the 16 bytes that is not the same in both. A comparison of vectors. +#[inline(always)] +fn differences(bytes: &[u8; 16], match_bytes: &[u8; 16]) -> u32 { + let mut differ = 0u32; + for (n, (byte, match_byte)) in (0..16).zip(bytes.iter().zip(match_bytes)) { + differ |= u32::from(byte != match_byte).wrapping_shl(n); + } + differ +} + impl<'a> Near<'a> { #[inline] pub(crate) fn new(bytes: &'a [u8]) -> Result { @@ -888,11 +899,7 @@ impl<'a> Near<'a> { self.chunk16(pos.wrapping_add(len)), other.chunk16(match_pos.wrapping_add(len)), ) { - // A comparison of vectors. - let mut differ = 0u32; - for (n, (byte, match_byte)) in (0..16).zip(bytes.iter().zip(match_bytes)) { - differ |= u32::from(byte != match_byte).wrapping_shl(n); - } + let differ = differences(bytes, match_bytes); if differ != 0 { let first = differ.trailing_zeros(); let byte = bytes.get(first as usize & 15).copied().unwrap_or(0); @@ -909,11 +916,7 @@ impl<'a> Near<'a> { #[inline(always)] pub(crate) fn compare_head(self, head: &[u8; 16], match_pos: u32, from: u32) -> Option { let match_bytes = self.chunk16(match_pos)?; - let mut differ = 0u32; - for (n, (byte, match_byte)) in (0..16).zip(head.iter().zip(match_bytes)) { - differ |= u32::from(byte != match_byte).wrapping_shl(n); - } - let first = differ.trailing_zeros(); + let first = differences(head, match_bytes).trailing_zeros(); if first >= 16 || first < from { return None; } @@ -931,11 +934,8 @@ impl<'a> Near<'a> { /// How many of the 16 bytes at `match_pos` are the same as `bytes`, from the first on. It takes no branch. #[inline(always)] fn count16(self, bytes: &[u8; 16], match_pos: u32) -> Option { - let mut same = 0u16; - for (i, (byte, match_byte)) in bytes.iter().zip(self.chunk16(match_pos)?).enumerate() { - same |= u16::from(byte == match_byte) << i; - } - Some((!u32::from(same)).trailing_zeros()) + let differ = differences(bytes, self.chunk16(match_pos)?); + Some((differ | 1 << 16).trailing_zeros()) } /// [`Self::count`], of which only the comparison of the first 16 bytes is inlined. It takes few registers of a loop @@ -1711,12 +1711,6 @@ impl Hash8 { pub(crate) const fn of(self, bytes: u64) -> u32 { (bytes.wrapping_mul(self.mul) ^ self.salt).wrapping_shr(self.shift) as u32 } - - /// [`Self::of`], when the salt is known to be 0. - #[inline(always)] - pub(crate) const fn of_unsalted(self, bytes: u64) -> u32 { - bytes.wrapping_mul(self.mul).wrapping_shr(self.shift) as u32 - } } // ── the window ──────────────────────────────────────────────────────────── @@ -2199,6 +2193,8 @@ pub(crate) struct MatchState { /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. pub(crate) has_dict_match_state: bool, pub(crate) c_params: CompressionParameters, + /// For a worker: whether its frame is given up. + pub(crate) cancelled: Option>, } impl MatchState { @@ -2228,6 +2224,15 @@ impl MatchState { target_length: 0, strategy: None, }, + cancelled: None, + } + } + + /// Fails if there is no use in going on. + pub(crate) fn check_cancelled(&self) -> Result<()> { + match &self.cancelled { + Some(cancelled) if cancelled.load(Ordering::SeqCst) => Err(Error::Generic), + _ => Ok(()), } } diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 02f3c6c38487..817c3e978e2e 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -846,7 +846,7 @@ pub(crate) fn insert_and_find_first_index( struct ChainDict<'a> { hash_table: &'a [u32], chain_table: &'a [u32], - hasher: Hash8, + hasher: Hasher, /// The index, among its own, of its first byte. lowest_index: u32, /// Below this, an index is no longer in the chain table. @@ -858,7 +858,7 @@ impl<'a> ChainDict<'a> { Self { hash_table: &dms.ms.hash_table, chain_table: &dms.ms.chain_table, - hasher: Hash8::new(mls, dms.ms.c_params.hash_log, 0), + hasher: Hasher::new(dms.ms.c_params.hash_log, mls), lowest_index: dms.ms.window.dict_limit, min_chain: dms_end_index(dms).saturating_sub(1 << dms.ms.c_params.chain_log), } @@ -874,8 +874,8 @@ struct Chains<'a> { impl Chains<'_> { /// Adds the position that `bytes` is the 8 bytes of. #[inline(always)] - fn insert(&mut self, hasher: Hash8, bytes: u64, index: u32) { - let hash = hasher.of(bytes) as usize & self.hash_table.len().wrapping_sub(1); + fn insert(&mut self, hasher: Hasher, bytes: u64, index: u32) { + let hash = hasher.hash(bytes) & self.hash_table.len().wrapping_sub(1); let in_chain = index as usize & self.chain_table.len().wrapping_sub(1); if let Some(first) = self.hash_table.get_mut(hash) && let Some(next) = self.chain_table.get_mut(in_chain) @@ -886,7 +886,7 @@ impl Chains<'_> { /// Adds every position that has 8 bytes in `positions`. The first one has the index `index`. #[inline(never)] - fn insert_all(self, hasher: Hash8, positions: &[u8], index: u32) { + fn insert_all(self, hasher: Hasher, positions: &[u8], index: u32) { let (Some(hash_mask), Some(chain_mask)) = ( self.hash_table.len().checked_sub(1), self.chain_table.len().checked_sub(1), @@ -895,7 +895,7 @@ impl Chains<'_> { }; for (n, bytes) in positions.array_windows().enumerate() { let index = index as usize + n; - let hash = hasher.of_unsalted(u64::from_le_bytes(*bytes)) as usize; + let hash = hasher.hash(u64::from_le_bytes(*bytes)); if let Some(head) = self.hash_table.get_mut(hash & hash_mask) && let Some(next) = self.chain_table.get_mut(index & chain_mask) { @@ -919,7 +919,7 @@ struct Candidate { struct ChainFinder<'a, const DICT_MODE: u8, const SIMPLE: bool> { block: LazyBlock<'a>, chains: Chains<'a>, - hasher: Hash8, + hasher: Hasher, mls: u32, nb_attempts: u32, /// `ms.next_to_update`, as a position. @@ -970,7 +970,7 @@ impl ChainFinder<'_, DICT_MODE, SIMPLE> } } self.next_to_update = ip; - let hash = self.hasher.of(near.read64(ip).ok_or(BUG)?) as usize; + let hash = self.hasher.hash(near.read64(ip).ok_or(BUG)?); let hash = hash & self.chains.hash_table.len().wrapping_sub(1); let head = self.chains.hash_table.get(hash).copied().unwrap_or(0); self.next_hash = hash; @@ -994,7 +994,7 @@ impl<'a, const DICT_MODE: u8, const SIMPLE: bool> Finder<'a> hash_table: &mut ms.hash_table, chain_table: &mut ms.chain_table, }, - hasher: Hash8::new(mls, ms.c_params.hash_log, 0), + hasher: Hasher::new(ms.c_params.hash_log, mls), mls, nb_attempts: 1 << ms.c_params.search_log, next_to_update: ms.next_to_update.wrapping_sub(block.base_index), @@ -1058,7 +1058,7 @@ impl<'a, const DICT_MODE: u8, const SIMPLE: bool> Finder<'a> && low_limit >= self.dense_from && let Some(end) = (best.len + 1).checked_sub(self.mls) && let Some(bytes) = block.near.read64(ip + end) - && table_get(self.chains.hash_table, self.hasher.of(bytes) as usize) < low_limit + && table_get(self.chains.hash_table, self.hasher.hash(bytes)) < low_limit { return Ok(None); } @@ -1083,7 +1083,7 @@ impl<'a, const DICT_MODE: u8, const SIMPLE: bool> Finder<'a> if DICT_MODE == DICT_MATCH_STATE { let dms = self.dms.ok_or(BUG)?; - let hash = dms.hasher.of(block.near.read64(ip).ok_or(BUG)?) as usize; + let hash = dms.hasher.hash(block.near.read64(ip).ok_or(BUG)?); match_index = dms.hash_table.at(hash)?; while match_index >= dms.lowest_index && nb_attempts > 0 { diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 0d6c32699cce..1fabeeb5543e 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -268,9 +268,6 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - Ok(()) }; for chunk_nb in 1..nb_chunks { - if cancelled.load(Ordering::SeqCst) { - return Err(Error::Generic); - } let chunk = Span { buf: BufId::User, start: ip, @@ -296,9 +293,9 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - send(state.dst.sub(..last_c_block_size)?, src_size) } -fn worker_main(jobs: &Receiver, cancelled: &AtomicBool) { +fn worker_main(jobs: &Receiver, cancelled: &Arc) { let mut state = WorkerState { - cctx: FrameCompressor::for_worker(), + cctx: FrameCompressor::for_worker(Arc::clone(cancelled)), seq: Vec::new(), dst: Vec::new(), }; @@ -377,7 +374,7 @@ impl PendingJob { /// `ZSTDMT_CCtx` pub(crate) struct MtCtx { - /// They are on their own: they share nothing, and leave once there is no queue or the frame is given up. + /// They are on their own: they share nothing, and leave once nothing can be sent to their queue. nb_workers: usize, /// Whichever worker is free takes the next job, so that a job does not wait for one thread to be scheduled. queue: Sender, @@ -490,7 +487,6 @@ impl MtCtx { .map_err(|_| Error::MemoryAllocation)?; self.nb_workers += 1; } - self.params.nb_workers = nb_workers as i32; Ok(()) } @@ -817,3 +813,78 @@ impl Drop for MtCtx { self.cancelled.store(true, Ordering::SeqCst); } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::compress::internal::{CompressionParameters, Strategy}; + use crate::compress::params::{FrameParameters, Parameters}; + + #[test] + fn a_worker_stops_at_a_prefix_or_a_block_of_a_frame_that_is_given_up() { + let params = CCtxParams::from_params( + &Parameters { + c_params: CompressionParameters { + window_log: 10, + chain_log: 6, + hash_log: 6, + search_log: 1, + min_match: 4, + target_length: 0, + strategy: Some(Strategy::BtOpt), + }, + f_params: FrameParameters { + content_size_flag: false, + checksum_flag: false, + no_dict_id_flag: false, + }, + }, + 0, + ); + let data = [0u8; 64]; + let input = Input { + bufs: Buffers { + in_buff: &[], + user: &data, + dict: &[], + }, + cdict: None, + }; + let span = |start, end| Span { + buf: BufId::User, + start, + end, + }; + let cancelled = Arc::new(AtomicBool::new(false)); + let mut cctx = FrameCompressor::for_worker(Arc::clone(&cancelled)); + let begin = |cctx: &mut FrameCompressor| { + cctx.begin( + &input, + Some(span(0, 32)), + DictContentType::RawContent, + DictTableLoadMethod::Fast, + ¶ms, + 32, + false, + ) + .map(|_| ()) + }; + let mut dst = [0u8; 128]; + + cancelled.store(true, Ordering::SeqCst); + assert_eq!(begin(&mut cctx), Err(Error::Generic)); + + cancelled.store(false, Ordering::SeqCst); + assert_eq!(begin(&mut cctx), Ok(())); + cancelled.store(true, Ordering::SeqCst); + let given_up = cctx.compress_continue(&mut dst, &input, span(32, 64)); + assert_eq!(given_up, Err(Error::Generic)); + + cancelled.store(false, Ordering::SeqCst); + assert_eq!(begin(&mut cctx), Ok(())); + assert!( + cctx.compress_continue(&mut dst, &input, span(32, 64)) + .is_ok() + ); + } +} diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index df4f3e55b3a5..a5a384972c55 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -613,13 +613,21 @@ impl<'a> Trees<'a> { } /// `ZSTD_updateTree_internal`: adds the positions from `next_to_update` up to `target` to the trees. - #[inline(never)] + #[inline(always)] fn update_tree(&mut self, target: u32) -> Result<()> { + self.update_tree_in_part(target, target)?; + self.next_to_update = target; + Ok(()) + } + + /// The part of [`Self::update_tree`] that gets to `up_to`, or somewhat beyond. + #[inline(never)] + fn update_tree_in_part(&mut self, up_to: u32, target: u32) -> Result<()> { let mut idx = self.next_to_update; - while idx < target { + while idx < up_to { idx = idx.wrapping_add(self.insert_bt1(idx, target)?); } - self.next_to_update = target; + self.next_to_update = idx; Ok(()) } @@ -877,9 +885,16 @@ pub(crate) fn update_tree( ) -> Result<()> { let no_dict = Prefix { buf: &[], base: 0 }; let mls = ms.c_params.min_match; - let mut trees = Trees::new(ms, prefix, no_dict, None, iend, mls)?; - trees.update_tree(prefix.index_of(ip))?; - ms.next_to_update = prefix.index_of(ip); + let target = prefix.index_of(ip); + // The prefix of a job can take seconds. + while ms.next_to_update < target { + ms.check_cancelled()?; + let up_to = ms.next_to_update.saturating_add(1 << 12).min(target); + let mut trees = Trees::new(ms, prefix, no_dict, None, iend, mls)?; + trees.update_tree_in_part(up_to, target)?; + ms.next_to_update = trees.next_to_update; + } + ms.next_to_update = target; Ok(()) } diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs index 914930c5db48..e76febfb69b1 100644 --- a/src/zstd/compress/stream.rs +++ b/src/zstd/compress/stream.rs @@ -238,16 +238,20 @@ impl CCtx { pledged_src_size, true, )?; + // They are part of the workspace in the C code. resize_buffer( &mut self.in_buff, &mut self.in_buff_size, sizes.buff_in_size, - )?; - resize_buffer( - &mut self.out_buff, - &mut self.out_buff_size, - sizes.buff_out_size, - )?; + ) + .and_then(|()| { + resize_buffer( + &mut self.out_buff, + &mut self.out_buff_size, + sizes.buff_out_size, + ) + }) + .inspect_err(|_| self.frame.forget())?; self.in_to_compress = 0; self.in_buff_pos = 0; // One more byte, to tell an input of exactly one block from a longer one. diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 655f652e1a6d..6a7844987de0 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -1192,9 +1192,9 @@ fn resolve_offset(sequences: &mut [RawSequence], index: usize) -> u32 { /// Turns sequences into output. struct Executor<'buf, 'a> { - /// Ends no further than `oend`. + /// What is beyond `oend` is room for copies in chunks. out: OutputBuffer<'buf>, - /// Where the block has to stop writing. + /// Where the output of the block has to end. oend: usize, op: usize, prefix_start: usize, diff --git a/src/zstd/error.rs b/src/zstd/error.rs index d216d0bf82c4..82d11c4e9c6a 100644 --- a/src/zstd/error.rs +++ b/src/zstd/error.rs @@ -124,17 +124,13 @@ impl Error { /// What the functions that allocate their output fail with. Users get to see the names. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error, strum::IntoStaticStr)] +#[error("{}", <&'static str>::from(self))] pub enum ZstdError { /// The output, or the decoder state the frame's window size dictates, could not be allocated. - #[error("OutOfMemory")] OutOfMemory, - #[error("InvalidZstdData")] InvalidZstdData, - #[error("DecompressionFailed")] DecompressionFailed, - #[error("ZstdDecompressionError")] ZstdDecompressionError, - #[error("ShortRead")] ShortRead, } diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index d548d4e39f16..e3bcffbcf058 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -132,9 +132,15 @@ pub fn compress_alloc(src: &[u8], level: i32) -> Result> { /// With that much room after its output, a decoder copies in chunks up to the end. const OUTPUT_SLACK: usize = common::WILDCOPY_OVERLENGTH; -/// `len` zeros for a decoder to write to. -fn output_buffer(len: usize) -> Result, ZstdError> { - util::try_zeroed(len).map_err(|_| ZstdError::OutOfMemory) +/// Zeros for a decoder to write `size` bytes to. +fn output_buffer(size: usize) -> Result, ZstdError> { + // Not where it would be most of the buffer: to give that back can take a copy. + let slack = if size >= 2 * OUTPUT_SLACK { + OUTPUT_SLACK + } else { + 0 + }; + util::try_zeroed(size + slack).map_err(|_| ZstdError::OutOfMemory) } /// Decompress data, automatically allocating the output buffer. @@ -169,13 +175,7 @@ pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { } // Fast path: size is known and within reasonable limits - // Not where it would be most of the buffer: to give that back can take a copy. - let slack = if size as usize >= 2 * OUTPUT_SLACK { - OUTPUT_SLACK - } else { - 0 - }; - let mut output = output_buffer(size as usize + slack)?; + let mut output = output_buffer(size as usize)?; let written = decompress(&mut output, src) .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; output.truncate(written); From 60b35fefdb33ba45fb35a29c1e60644bc115d7ee Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 03:33:45 +0000 Subject: [PATCH 139/199] zstd: tests for four fixes to the optimal parser that nothing noticed the lack of Each input gives other bytes, or an error, once the check it is for is taken out of the crate: a repeat offset beyond the window after the dictionary is given up, the byte after the block, the last position of the price table, and the price of 128 KB of literals. The expected values are those of C 1.5.7. --- test/js/bun/util/zstd.test.ts | 189 ++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 983c65fe63f6..6a17618bb87b 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -2074,4 +2074,193 @@ describe("bugs that zstd has had", () => { expect(zlib.zstdDecompressSync(frame, { dictionary }).equals(input)).toBe(true); }); }); + + // Each input is one of many from its seed that gives other bytes once the check that it is for is taken out. + describe("the optimal parser", () => { + /** xorshift32 */ + class Input { + bytes: number[] = []; + #x: number; + constructor(seed: number) { + this.#x = Math.imul(seed, 2654435761) >>> 0; + } + next() { + this.#x ^= this.#x << 13; + this.#x ^= this.#x >>> 17; + this.#x ^= this.#x << 5; + return (this.#x >>>= 0); + } + range(from: number, to: number) { + return from + (this.next() % (to - from + 1)); + } + pick(values: number[]) { + return values[this.next() % values.length]; + } + some(length: number) { + return Array.from({ length }, () => this.next() & 255); + } + noise(length: number) { + return this.add(this.some(length)); + } + copy(offset: number, length: number) { + for (let i = 0; i < length; i++) this.bytes.push(this.bytes[this.bytes.length - offset]); + return this; + } + add(bytes: number[]) { + for (const byte of bytes) this.bytes.push(byte); + return this; + } + /** Many of these make one literal cheaper than none. */ + shortMatchAndLiteral() { + return this.copy(this.range(50, Math.min(2000, this.bytes.length)), this.range(5, 14)).noise(1); + } + } + const digest = (frame: Buffer) => `${frame.length} ${Bun.hash.xxHash32(frame).toString(16)}`; + + // The first block ends with a match in the dictionary, a little further back than the window is large. For the + // second block the dictionary is given up, and the same distance leads to the right bytes again. + it.each([ + [26, false, "1520 a9ba4dd5"], + [10, false, "2316 30a8306d"], + [26, true, "9125 e0ee4a07"], + ])( + "a repeat offset beyond the window is not used once the dictionary is given up: %i, %p", + async (seed, known, expected) => { + const input = new Input(seed); + const windowLog = input.range(10, 12); + const window = 1 << windowLog; + const offset = window + input.range(1, 200); + const dictionarySize = input.range(offset + 50, 4 * window); + input.noise(dictionarySize); + const first = input.next() % 2 < 1 ? input.range(window >> 2, window - 50) : window; + const [length, after] = [input.range(8, 40), input.range(0, 5)]; + input + .noise(first - length - after) + .copy(offset, length) + .noise(after); + for (let i = input.range(1, 6); i > 0; i--) input.noise(input.range(1, 250)).copy(offset, input.range(4, 40)); + // From 8 KB on the tables of the dictionary are copied, and it is the older segment. Else it is attached. + const more = known ? 8300 + input.range(0, 2999) : input.range(1, 3000); + while (input.bytes.length - dictionarySize < first + more) { + if (input.next() % 3 < 1) input.copy(offset, input.range(4, 40)); + else input.noise(input.range(1, 400)); + } + const dictionary = Buffer.from(input.bytes).subarray(0, dictionarySize); + const source = Buffer.from(input.bytes).subarray(dictionarySize); + const params = { + [c.ZSTD_c_strategy]: input.range(c.ZSTD_btopt, c.ZSTD_btultra2), + [c.ZSTD_c_windowLog]: windowLog, + [c.ZSTD_c_minMatch]: input.range(3, 5), + [c.ZSTD_c_searchLog]: input.range(2, 6), + }; + const stream = zlib.createZstdCompress({ + params, + dictionary, + pledgedSrcSize: known ? source.length : undefined, + }); + stream.write(source.subarray(0, first)); + stream.flush(); + const frame = (await through(stream, [source.subarray(first)])) as Buffer; + expect(digest(frame)).toBe(expected); + expect(zlib.zstdDecompressSync(frame, { dictionary }).equals(source)).toBe(true); + }, + ); + + // facebook/zstd b0e8580dc. The input ends with `q 6 x 1`. `q 5 x 2` is a little before, so that the repeat offset + // gives `x` and not what follows, and `x 1` is far before. + it("does not look at the byte after the block for a literal that replaces the end of a match", () => { + const input = new Input(68); + const x = input.some(input.range(8, 40)); + input + .noise(200) + .add(x) + .add([1]) + .noise(input.pick([300, 5000, 40000])); + for (let i = input.range(30, 600); i > 0; i--) input.shortMatchAndLiteral(); + const q = input.some(input.range(6, 20)); + for (let i = input.range(0, 29); i > 0; i--) { + const [a, b] = [input.some(input.range(6, 20)), input.some(input.range(6, 20))]; + for (const w of [3, 4]) + input + .add(a) + .add([w]) + .add(b) + .add([w + 10]); + } + input.add(q).add([5]).add(x).add([2]); + for (let i = input.range(0, 2); i > 0; i--) input.shortMatchAndLiteral(); + input.add(q).add([6]).add(x).add([1]); + const params = { + [c.ZSTD_c_strategy]: input.range(c.ZSTD_btopt, c.ZSTD_btultra2), + [c.ZSTD_c_windowLog]: input.range(19, 21), + [c.ZSTD_c_searchLog]: input.range(3, 7), + [c.ZSTD_c_minMatch]: input.range(3, 5), + [c.ZSTD_c_targetLength]: input.pick([48, 64, 200, 999]), + }; + expect(digest(roundTrip(Buffer.from(input.bytes), params))).toBe("41368 2e959c78"); + }); + + // facebook/zstd 22574d848. Matches for `text` are short and overlap, as both copies of it are spoilt every so often, + // at other places. Its last byte is only in a third, short copy. + it("has room for a series that goes on to the last position a match can lead to, and a literal", () => { + const input = new Input(255); + const size = input.pick([4095, 4095, 4095, 4095, 4094, 4096, 4093, 4097]); + const text = input.some(size); + const step = input.range(40, 300); + const spoilt = (first: number) => + text.map((byte, i) => (i === size - 1 || (i >= first && (i - first) % step === 0) ? byte ^ 0x55 : byte)); + const end = text.slice(size - input.range(5, 40)); + input.noise(100); + const far = input.next() % 2 < 1; + if (far) input.add(end).noise(input.pick([10, 1000, 20000])); + input + .add(spoilt(step >> 1)) + .noise(20) + .add(spoilt(step - 1)) + .noise(20); + for (let i = input.range(100, 900); i > 0; i--) input.shortMatchAndLiteral(); + if (!far) input.add(end); + input.noise(input.range(12, 60)).add(text).noise(input.range(20, 400)); + const params = { + [c.ZSTD_c_strategy]: input.range(c.ZSTD_btopt, c.ZSTD_btultra2), + [c.ZSTD_c_windowLog]: 20, + [c.ZSTD_c_searchLog]: input.range(3, 7), + [c.ZSTD_c_minMatch]: input.range(3, 5), + [c.ZSTD_c_targetLength]: input.pick([4095, 4096, 131072]), + }; + expect(digest(roundTrip(Buffer.from(input.bytes), params))).toBe("5917 9ffc7ec4"); + }); + + // facebook/zstd 4d8a2132d. No 4 bytes come twice before the last 9, which are a copy from far away that the small + // tables have lost by the position after its first, and half of the block is one byte, so that literals are cheap. + it("has a price for 128 KB of literals", () => { + const input = new Input(4); + const size = 128 << 10; + const last = [0, 0, 0, input.range(1, 254), 0, 0, 0, input.range(1, 254)]; + last.push(input.next() % 3 < 1 ? 0 : input.range(1, 254)); + const from = 4 * input.range(1, (size - 70000) >> 2); + const pairs = Array.from({ length: 254 * 254 }, (_, i) => [1 + Math.floor(i / 254), 1 + (i % 254)]); + for (let i = pairs.length - 1; i > 0; i--) { + const j = input.next() % (i + 1); + [pairs[i], pairs[j]] = [pairs[j], pairs[i]]; + } + for (const pair of pairs) { + if (input.bytes.length === from) input.add(last); + if (input.add([0, 0]).add(pair).bytes.length >= size) break; + } + input.bytes.length = size - last.length; + input.bytes[input.bytes.length - 1] ||= 7; + input.add(last); + const params = { + [c.ZSTD_c_strategy]: input.range(c.ZSTD_btopt, c.ZSTD_btultra2), + [c.ZSTD_c_minMatch]: input.range(4, 6), + [c.ZSTD_c_windowLog]: 18, + [c.ZSTD_c_targetLength]: input.pick([48, 64, 256, 999]), + [c.ZSTD_c_hashLog]: input.range(14, 19), + [c.ZSTD_c_chainLog]: input.range(6, 8), + [c.ZSTD_c_searchLog]: 1, + }; + expect(digest(roundTrip(Buffer.from(input.bytes), params))).toBe("81886 6893fa8c"); + }); + }); }); From 0a40fdcf1c292f722d40cf2cad823b85513c981c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 04:32:40 +0000 Subject: [PATCH 140/199] zstd: rows of 64 entries stay a comparison of vectors in a release build A release build leaves the crates' code generation to the link's ThinLTO. There the masks of four times 16 tags, put together as a | b << 16 | c << 32 | d << 48, came out as 281 instructions on x64 instead of 31, and levels 11 and 12 took 1.1 to 1.2 times as long as with the C library. Folded from the top they are the same code with and without LTO (aarch64: the same either way). --- src/zstd/compress/lazy.rs | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 817c3e978e2e..047ab789a188 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -1413,29 +1413,22 @@ pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> /// Bit `n` is set if the entry `n` has the tag `tag`. This becomes a comparison of vectors. #[inline(always)] -fn tags_match_mask(tags: &[u8; 16], tag: u8) -> u32 { - let mut matches = 0u16; +fn tags_match_mask(tags: &[u8; 16], tag: u8) -> u16 { + let mut matches = 0; for (n, &entry) in tags.iter().enumerate() { matches |= u16::from(entry == tag) << n; } - u32::from(matches) + matches } -/// Bit `n` is set if the entry `n` has the tag `tag`. +/// Bit `n` is set if the entry `n` has the tag `tag`. As `a | b << 16 | ..`, LTO makes 10 times the instructions of it. #[inline(always)] fn row_tags_match_mask(tags: &[u8; ROW_ENTRIES], tag: u8) -> u64 { - match *tags.as_chunks::<16>().0 { - [ref tags] => u64::from(tags_match_mask(tags, tag)), - [ref low, ref high] => { - u64::from(tags_match_mask(low, tag) | tags_match_mask(high, tag) << 16) - } - [ref tags_0, ref tags_1, ref tags_2, ref tags_3] => { - let low = tags_match_mask(tags_0, tag) | tags_match_mask(tags_1, tag) << 16; - let high = tags_match_mask(tags_2, tag) | tags_match_mask(tags_3, tag) << 16; - u64::from(low) | u64::from(high) << 32 - } - _ => 0, - } + const { assert!(matches!(ROW_ENTRIES, 16 | 32 | 64)) } + let (chunks, _) = tags.as_chunks::<16>(); + chunks.iter().rev().fold(0, |matches, tags| { + matches << 16 | u64::from(tags_match_mask(tags, tag)) + }) } /// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` has the tag `tag`. @@ -1453,7 +1446,7 @@ fn row_get_match_mask( } } -/// [`row_get_match_mask`]. On its own, it stays a comparison of vectors. +/// [`row_get_match_mask`]. Inlined, levels 11 and 12 take a tenth longer for text. #[inline(never)] fn row_get_second_match_mask( tags: &[u8; ROW_ENTRIES], From 8e0cb8528743ab4d2ec13d3c48c506c95f1fd8ea Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 04:32:41 +0000 Subject: [PATCH 141/199] zstd: remove two parameters that follow from a third ZSTD_dtlm_full and ZSTD_dct_auto are what a CDict is loaded with, and nothing else is, so TableFillPurpose says it all. The branch of dfast for a context with ZSTD_dtlm_full was never taken. Filling the tables of fast and dfast cannot fail. --- src/zstd/compress/cctx.rs | 34 ++++++------------------ src/zstd/compress/double_fast.rs | 45 +++++++------------------------- src/zstd/compress/fast.rs | 30 +++++++-------------- src/zstd/compress/internal.rs | 9 +------ src/zstd/compress/ldm.rs | 32 ++++++++--------------- src/zstd/compress/mt.rs | 31 ++++------------------ src/zstd/compress/stream.rs | 26 +++++------------- 7 files changed, 50 insertions(+), 157 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 93b9c78e564a..5908a1b3cc22 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -12,9 +12,9 @@ use super::fse::FseRepeat; use super::huf::{self, HufRepeat}; use super::internal::{ BlockSrc, BufId, Buffers, CompressedBlockState, CompressionParameters, DictMatchState, - DictTableLoadMethod, EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, Match, - MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, RowTags, SeqChunk, SeqStore, Strategy, - Table, TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, + EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, Match, MatchState, + ParamSwitch, Prefix, RawSeq, RawSeqStore, RowTags, SeqChunk, SeqStore, Strategy, Table, + TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, min_gain, no_compress_block, offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, }; @@ -72,13 +72,6 @@ enum ResetTarget { CCtx, } -/// `ZSTD_dictContentType_e`. Nothing asks for `ZSTD_dct_fullDict`. -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum DictContentType { - Auto, - RawContent, -} - /// Some bytes of one of the [`Buffers`]. #[derive(Clone, Copy)] pub(crate) struct Span { @@ -310,7 +303,6 @@ fn load_dictionary_content( params: &CCtxParams, bufs: &Buffers<'_>, mut src: Span, - dtlm: DictTableLoadMethod, tfp: TableFillPurpose, ) -> Result<()> { { @@ -356,8 +348,8 @@ fn load_dictionary_content( let prefix = window_prefix(&ms.window, bufs); match params.c_params.strat() { - Strategy::Fast => fast::fill_hash_table(ms, prefix, src.end, dtlm, tfp)?, - Strategy::DFast => double_fast::fill_double_hash_table(ms, prefix, src.end, dtlm, tfp)?, + Strategy::Fast => fast::fill_hash_table(ms, prefix, src.end, tfp), + Strategy::DFast => double_fast::fill_double_hash_table(ms, prefix, src.end, tfp), Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { if params.use_row_match_finder == ParamSwitch::Enable { ms.tag_table.fill(0); @@ -495,8 +487,6 @@ fn compress_insert_dictionary( params: &CCtxParams, bufs: &Buffers<'_>, dict: Option, - dict_content_type: DictContentType, - dtlm: DictTableLoadMethod, tfp: TableFillPurpose, ) -> Result { let Some(dict) = dict.filter(|dict| dict.len() >= 8) else { @@ -506,10 +496,8 @@ fn compress_insert_dictionary( bs.reset(); let bytes = dict.bytes(bufs)?; - if dict_content_type == DictContentType::RawContent - || read_le32(bytes, 0)? != ZSTD_MAGIC_DICTIONARY - { - load_dictionary_content(ms, params, bufs, dict, dtlm, tfp)?; + if tfp == TableFillPurpose::ForCCtx || read_le32(bytes, 0)? != ZSTD_MAGIC_DICTIONARY { + load_dictionary_content(ms, params, bufs, dict, tfp)?; return Ok(0); } @@ -524,7 +512,7 @@ fn compress_insert_dictionary( start: dict.start + e_size, ..dict }; - load_dictionary_content(ms, params, bufs, content, dtlm, tfp)?; + load_dictionary_content(ms, params, bufs, content, tfp)?; Ok(dict_id) } @@ -581,8 +569,6 @@ impl CDict { start: 0, end: cdict.dict_content.len(), }), - DictContentType::Auto, - DictTableLoadMethod::Full, TableFillPurpose::ForCDict, )?; Ok(cdict) @@ -1144,8 +1130,6 @@ impl FrameCompressor { &mut self, input: &Input<'_>, dict: Option, - dict_content_type: DictContentType, - dtlm: DictTableLoadMethod, params: &CCtxParams, pledged_src_size: u64, buffered: bool, @@ -1176,8 +1160,6 @@ impl FrameCompressor { &self.applied_params, &input.bufs, dict, - dict_content_type, - dtlm, TableFillPurpose::ForCCtx, )?, None => 0, diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index f9391c640ba1..113063097f4d 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -5,44 +5,31 @@ use super::fast::{ put_tagged, put_tagged_in_between, restore_rep, }; use super::internal::{ - BlockSrc, DictTableLoadMethod, HASH_READ_SIZE, Hasher, K_SEARCH_STRENGTH, MatchState, Near, - Prefix, REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, hashed_get, hashed_put, hashed_swap, - le64_at, offset_to_offbase, table_get, table_of_size, table_prefetch, table_put, + BlockSrc, HASH_READ_SIZE, Hasher, K_SEARCH_STRENGTH, MatchState, Near, Prefix, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, hashed_get, hashed_put, hashed_swap, le64_at, + offset_to_offbase, table_of_size, table_prefetch, table_put, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; use crate::util::{BUG, SliceExt}; /// `ZSTD_fillDoubleHashTableForCDict`: the tables are empty. -fn fill_double_hash_table_for_cdict( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { +fn fill_double_hash_table_for_cdict(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) { let long = Hasher::new(ms.tagged_hash_log(), 8); let small = Hasher::new(ms.tagged_chain_log(), ms.c_params.min_match); let positions = ThirdPositions::new(prefix, ms.next_to_update, end); let hash_large = ms.hash_table.as_mut_slice(); let hash_small = ms.chain_table.as_mut_slice(); - if dtlm == DictTableLoadMethod::Full { - put_tagged_in_between(hash_large, long, positions.clone()); - } + put_tagged_in_between(hash_large, long, positions.clone()); for at in positions { let bytes = le64_at::<0, _>(at.bytes); put_tagged(hash_small, small.hash(bytes), at.index); put_tagged(hash_large, long.hash(bytes), at.index); } - Ok(()) } /// `ZSTD_fillDoubleHashTableForCCtx` -fn fill_double_hash_table_for_cctx( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { +fn fill_double_hash_table_for_cctx(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) { let long = Hasher::new(ms.c_params.hash_log, 8); let small = Hasher::new(ms.c_params.chain_log, ms.c_params.min_match); let positions = ThirdPositions::new(prefix, ms.next_to_update, end); @@ -52,20 +39,7 @@ fn fill_double_hash_table_for_cctx( let bytes = le64_at::<0, _>(at.bytes); table_put(hash_small, small.hash(bytes), at.index); table_put(hash_large, long.hash(bytes), at.index); - // The positions in between only go in the table of long hashes, where there is nothing yet. - if dtlm == DictTableLoadMethod::Full { - for (bytes, index) in [ - (le64_at::<1, _>(at.bytes), at.index + 1), - (le64_at::<2, _>(at.bytes), at.index + 2), - ] { - let hash = long.hash(bytes); - if table_get(hash_large, hash) == 0 { - table_put(hash_large, hash, index); - } - } - } } - Ok(()) } /// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. @@ -73,12 +47,11 @@ pub(crate) fn fill_double_hash_table( ms: &mut MatchState, prefix: Prefix<'_>, end: usize, - dtlm: DictTableLoadMethod, tfp: TableFillPurpose, -) -> Result<()> { +) { match tfp { - TableFillPurpose::ForCDict => fill_double_hash_table_for_cdict(ms, prefix, end, dtlm), - TableFillPurpose::ForCCtx => fill_double_hash_table_for_cctx(ms, prefix, end, dtlm), + TableFillPurpose::ForCDict => fill_double_hash_table_for_cdict(ms, prefix, end), + TableFillPurpose::ForCCtx => fill_double_hash_table_for_cctx(ms, prefix, end), } } diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 45cf4aab4fd2..6550ab8d91ce 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -3,10 +3,10 @@ use core::sync::atomic::{Ordering, compiler_fence}; use super::internal::{ - BlockSrc, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, Hasher, - K_SEARCH_STRENGTH, MatchState, Near, Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, - TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, compare_packed_tags, hashed_get, hashed_put, - hashed_swap, le32_at, le64_at, offset_to_offbase, table_of_size, table_put, + BlockSrc, CompressionParameters, HASH_READ_SIZE, Hasher, K_SEARCH_STRENGTH, MatchState, Near, + Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, hashed_get, hashed_put, hashed_swap, le32_at, le64_at, offset_to_offbase, + table_of_size, table_put, }; use crate::common::ZSTD_REP_NUM; use crate::error::Result; @@ -164,33 +164,24 @@ pub(super) fn put_tagged_in_between( } /// `ZSTD_fillHashTableForCDict`: the table is empty. -fn fill_hash_table_for_cdict( - ms: &mut MatchState, - prefix: Prefix<'_>, - end: usize, - dtlm: DictTableLoadMethod, -) -> Result<()> { +fn fill_hash_table_for_cdict(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) { let hasher = Hasher::new(ms.tagged_hash_log(), ms.c_params.min_match); let positions = ThirdPositions::new(prefix, ms.next_to_update, end); let hash_table = ms.hash_table.as_mut_slice(); - if dtlm == DictTableLoadMethod::Full { - put_tagged_in_between(hash_table, hasher, positions.clone()); - } + put_tagged_in_between(hash_table, hasher, positions.clone()); for at in positions { put_tagged(hash_table, hasher.hash(le64_at::<0, _>(at.bytes)), at.index); } - Ok(()) } -/// `ZSTD_fillHashTableForCCtx`, which is only for `ZSTD_dtlm_fast`. -fn fill_hash_table_for_cctx(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) -> Result<()> { +/// `ZSTD_fillHashTableForCCtx` +fn fill_hash_table_for_cctx(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) { let hasher = Hasher::new(ms.c_params.hash_log, ms.c_params.min_match); let positions = ThirdPositions::new(prefix, ms.next_to_update, end); let hash_table = ms.hash_table.as_mut_slice(); for at in positions { table_put(hash_table, hasher.hash(le64_at::<0, _>(at.bytes)), at.index); } - Ok(()) } /// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. @@ -198,11 +189,10 @@ pub(crate) fn fill_hash_table( ms: &mut MatchState, prefix: Prefix<'_>, end: usize, - dtlm: DictTableLoadMethod, tfp: TableFillPurpose, -) -> Result<()> { +) { match tfp { - TableFillPurpose::ForCDict => fill_hash_table_for_cdict(ms, prefix, end, dtlm), + TableFillPurpose::ForCDict => fill_hash_table_for_cdict(ms, prefix, end), TableFillPurpose::ForCCtx => fill_hash_table_for_cctx(ms, prefix, end), } } diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index bce75ec9a0b3..1cf70df9914f 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -94,14 +94,7 @@ pub(crate) enum DictMode { DictMatchState, } -/// `ZSTD_dictTableLoadMethod_e` -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum DictTableLoadMethod { - Fast, - Full, -} - -/// `ZSTD_tableFillPurpose_e` +/// `ZSTD_tableFillPurpose_e`. A `CDict` is loaded with `ZSTD_dtlm_full` and `ZSTD_dct_auto`, and nothing else is. #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum TableFillPurpose { ForCCtx, diff --git a/src/zstd/compress/ldm.rs b/src/zstd/compress/ldm.rs index 6959f6dbcb50..70b058b95571 100644 --- a/src/zstd/compress/ldm.rs +++ b/src/zstd/compress/ldm.rs @@ -1,9 +1,9 @@ //! `zstd_ldm.c`: long distance matching. use super::internal::{ - BlockSrc, Buffers, CompressionParameters, DictTableLoadMethod, HASH_READ_SIZE, LdmEntry, - LdmParams, LdmState, MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, - TableFillPurpose, count, count_2segments, offset_to_offbase, + BlockSrc, Buffers, CompressionParameters, HASH_READ_SIZE, LdmEntry, LdmParams, LdmState, + MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, TableFillPurpose, + count, count_2segments, offset_to_offbase, }; use super::params::{ZSTD_HASHLOG_MAX, ZSTD_HASHLOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX}; use super::{compress_block, double_fast, fast}; @@ -289,29 +289,19 @@ fn count_backwards_match_2segments( } /// `ZSTD_ldm_fillFastTables`: the strategies that index only what they search have to be told about what they skip. -fn fill_fast_tables(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) -> Result<()> { +fn fill_fast_tables(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) { match ms.c_params.strat() { - Strategy::Fast => fast::fill_hash_table( - ms, - prefix, - end, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - ), - Strategy::DFast => double_fast::fill_double_hash_table( - ms, - prefix, - end, - DictTableLoadMethod::Fast, - TableFillPurpose::ForCCtx, - ), + Strategy::Fast => fast::fill_hash_table(ms, prefix, end, TableFillPurpose::ForCCtx), + Strategy::DFast => { + double_fast::fill_double_hash_table(ms, prefix, end, TableFillPurpose::ForCCtx) + } Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 | Strategy::BtOpt | Strategy::BtUltra - | Strategy::BtUltra2 => Ok(()), + | Strategy::BtUltra2 => {} } } @@ -693,7 +683,7 @@ pub(crate) fn block_compress( } limit_table_update(ms, src, ip); - fill_fast_tables(ms, src.prefix(), ip)?; + fill_fast_tables(ms, src.prefix(), ip); // The block compressor takes care of the literals, and what it leaves goes with the long distance match. let literals = BlockSrc { start: ip, @@ -722,7 +712,7 @@ pub(crate) fn block_compress( ip += sequence.match_length as usize; } limit_table_update(ms, src, ip); - fill_fast_tables(ms, src.prefix(), ip)?; + fill_fast_tables(ms, src.prefix(), ip); let last = BlockSrc { start: ip, ..*src }; compress_block( ms, diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 1fabeeb5543e..c9333c4a63b9 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -11,12 +11,9 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpmc::{Receiver, Sender, TryRecvError, channel}; -use super::cctx::{ - CDict, DictContentType, FrameCompressor, Input, Span, reset_table, write_last_empty_block, -}; +use super::cctx::{CDict, FrameCompressor, Input, Span, reset_table, write_last_empty_block}; use super::internal::{ - BufId, Buffers, DictTableLoadMethod, LdmParams, LdmState, ParamSwitch, RawSeq, RawSeqStore, - Strategy, Window, + BufId, Buffers, LdmParams, LdmState, ParamSwitch, RawSeq, RawSeqStore, Strategy, Window, }; use super::ldm; use super::params::{ @@ -215,15 +212,7 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - }; let cctx = &mut state.cctx; if cdict.is_some() { - cctx.begin( - &input, - None, - DictContentType::Auto, - DictTableLoadMethod::Fast, - &job_params, - job.full_frame_size, - false, - )?; + cctx.begin(&input, None, &job_params, job.full_frame_size, false)?; } else { let pledged_src_size = if job.first_job { job.full_frame_size @@ -238,8 +227,6 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - start: 0, end: job.prefix_size, }), - DictContentType::RawContent, - DictTableLoadMethod::Fast, &job_params, pledged_src_size, false, @@ -858,16 +845,8 @@ mod tests { let cancelled = Arc::new(AtomicBool::new(false)); let mut cctx = FrameCompressor::for_worker(Arc::clone(&cancelled)); let begin = |cctx: &mut FrameCompressor| { - cctx.begin( - &input, - Some(span(0, 32)), - DictContentType::RawContent, - DictTableLoadMethod::Fast, - ¶ms, - 32, - false, - ) - .map(|_| ()) + cctx.begin(&input, Some(span(0, 32)), ¶ms, 32, false) + .map(|_| ()) }; let mut dst = [0u8; 128]; diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs index e76febfb69b1..492f07fd21b2 100644 --- a/src/zstd/compress/stream.rs +++ b/src/zstd/compress/stream.rs @@ -2,8 +2,8 @@ use std::sync::Arc; -use super::cctx::{CDict, DictContentType, FrameCompressor, Input, Span}; -use super::internal::{BufId, Buffers, CompressedBlockState, DictTableLoadMethod}; +use super::cctx::{CDict, FrameCompressor, Input, Span}; +use super::internal::{BufId, Buffers, CompressedBlockState}; use super::mt::MtCtx; use super::params::{ CCtxParams, CParamMode, ZSTD_CLEVEL_DEFAULT, ZSTDMT_JOBSIZE_MIN, compress_bound, @@ -229,15 +229,9 @@ impl CCtx { }, cdict, }; - let sizes = self.frame.begin( - &input, - None, - DictContentType::Auto, - DictTableLoadMethod::Fast, - ¶ms, - pledged_src_size, - true, - )?; + let sizes = self + .frame + .begin(&input, None, ¶ms, pledged_src_size, true)?; // They are part of the workspace in the C code. resize_buffer( &mut self.in_buff, @@ -506,15 +500,7 @@ pub fn compress(dst: &mut [u8], src: &[u8], compression_level: i32) -> Result Date: Wed, 28 Oct 2026 04:32:41 +0000 Subject: [PATCH 142/199] zstd: split three tests that took over 3 s each in a debug build With the machine busy they ran into the timeout of 5 s. Same inputs and expected values, one test for each level, distance or set of parameters. --- test/js/bun/util/zstd.test.ts | 117 ++++++++++++++++------------------ 1 file changed, 56 insertions(+), 61 deletions(-) diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 6a17618bb87b..8ea383f9d421 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1113,48 +1113,41 @@ describe("the output is that of zstd 1.5.7", () => { const large = corpus.subarray(0, 300_000); const small = corpus.subarray(300_000, 303_000); - it("at every level", () => { - const results: Record = {}; - for (let level = 1; level <= 22; level++) { - // The highest levels are slow. - const data = level < 19 ? large : large.subarray(0, 140_000); - results[level] = `${digest(zstdCompressSync(data, { level }))}, ${digest(zstdCompressSync(small, { level }))}`; - } - // Not -1: node:zlib takes that for a parameter that is not set. - for (const level of [-2, -7, -100]) { - const params = { [c.ZSTD_c_compressionLevel]: level }; - results[level] = - `${digest(zlib.zstdCompressSync(large, { params }))}, ${digest(zlib.zstdCompressSync(small, { params }))}`; - } - expect(results).toMatchInlineSnapshot(` - { - "-100": "167967 45693a3bc89d2838, 2112 a1f04bdd2c75d853", - "-2": "55329 a7cd8da33a4cce8b, 954 cb3a52b86b5193c7", - "-7": "65641 25bb487794d2f45f, 1081 6841140ea417de55", - "1": "46828 2a61d18b13a9df88, 824 3a3b2b799ed70f7b", - "10": "42353 d560f70047c278d, 807 bc69857972f7737c", - "11": "42124 cc37ec9a45f8d164, 808 95559a23325a524d", - "12": "42124 cc37ec9a45f8d164, 806 5984ef401fd2141", - "13": "42364 fd1a09c03715fc5f, 807 6648f2fa58a397d4", - "14": "42287 87ee023408ff3e30, 807 6648f2fa58a397d4", - "15": "42233 109ef580b5685e85, 804 52073852c548e55e", - "16": "42522 3293e655be1b0330, 807 6648f2fa58a397d4", - "17": "41809 7cb274a8772cd4cc, 806 b609660f6b3d21d9", - "18": "41253 d616eadc7ae1a5de, 804 52073852c548e55e", - "19": "18189 44c06af9c0a7b97f, 804 52073852c548e55e", - "2": "45933 2795a6f87171faac, 811 bd7ae68cb7d95862", - "20": "18125 7c0ef33ba30541a2, 804 52073852c548e55e", - "21": "18125 7c0ef33ba30541a2, 804 52073852c548e55e", - "22": "18125 e1b7fc6756c67456, 804 52073852c548e55e", - "3": "45122 344d3c35e5ad0340, 810 e35cec5c3500c856", - "4": "45093 7634242fc968e2c3, 822 6b45b45a5d921bfc", - "5": "44424 daecc1abec496995, 810 4489026b2c29b8dc", - "6": "42948 a879387b2601910d, 809 b15c4cf338e1d77a", - "7": "42675 1f9ad80ad0656d09, 804 603c9d665714a2", - "8": "42562 26911127c1e54142, 804 603c9d665714a2", - "9": "42562 26911127c1e54142, 807 bc69857972f7737c", - } - `); + // Not -1: node:zlib takes that for a parameter that is not set. + it.each([ + [-100, "167967 45693a3bc89d2838, 2112 a1f04bdd2c75d853"], + [-7, "65641 25bb487794d2f45f, 1081 6841140ea417de55"], + [-2, "55329 a7cd8da33a4cce8b, 954 cb3a52b86b5193c7"], + [1, "46828 2a61d18b13a9df88, 824 3a3b2b799ed70f7b"], + [2, "45933 2795a6f87171faac, 811 bd7ae68cb7d95862"], + [3, "45122 344d3c35e5ad0340, 810 e35cec5c3500c856"], + [4, "45093 7634242fc968e2c3, 822 6b45b45a5d921bfc"], + [5, "44424 daecc1abec496995, 810 4489026b2c29b8dc"], + [6, "42948 a879387b2601910d, 809 b15c4cf338e1d77a"], + [7, "42675 1f9ad80ad0656d09, 804 603c9d665714a2"], + [8, "42562 26911127c1e54142, 804 603c9d665714a2"], + [9, "42562 26911127c1e54142, 807 bc69857972f7737c"], + [10, "42353 d560f70047c278d, 807 bc69857972f7737c"], + [11, "42124 cc37ec9a45f8d164, 808 95559a23325a524d"], + [12, "42124 cc37ec9a45f8d164, 806 5984ef401fd2141"], + [13, "42364 fd1a09c03715fc5f, 807 6648f2fa58a397d4"], + [14, "42287 87ee023408ff3e30, 807 6648f2fa58a397d4"], + [15, "42233 109ef580b5685e85, 804 52073852c548e55e"], + [16, "42522 3293e655be1b0330, 807 6648f2fa58a397d4"], + [17, "41809 7cb274a8772cd4cc, 806 b609660f6b3d21d9"], + [18, "41253 d616eadc7ae1a5de, 804 52073852c548e55e"], + [19, "18189 44c06af9c0a7b97f, 804 52073852c548e55e"], + [20, "18125 7c0ef33ba30541a2, 804 52073852c548e55e"], + [21, "18125 7c0ef33ba30541a2, 804 52073852c548e55e"], + [22, "18125 e1b7fc6756c67456, 804 52073852c548e55e"], + ])("at level %i", (level, expected) => { + const compress = (data: Uint8Array) => + level > 0 + ? zstdCompressSync(data, { level }) + : zlib.zstdCompressSync(data, { params: { [c.ZSTD_c_compressionLevel]: level } }); + // The highest levels are slow. + const data = level < 19 ? large : large.subarray(0, 140_000); + expect(`${digest(compress(data))}, ${digest(compress(small))}`).toBe(expected); }); it("with parameters", () => { @@ -1690,8 +1683,9 @@ describe("bugs that zstd has had", () => { } }); - it("offsets up to the window are decoded after a stream's buffer has wrapped around", async () => { - for (const distance of [1000, 1017, 1023, 1024]) { + it.each([1000, 1017, 1023, 1024])( + "offsets up to the window are decoded after a stream's buffer has wrapped around: %i", + async distance => { const input = Buffer.alloc(100_000, noise(distance, distance)); for (const level of [5, 13, 19]) { const params = { [c.ZSTD_c_windowLog]: 10, [c.ZSTD_c_compressionLevel]: level }; @@ -1700,8 +1694,8 @@ describe("bugs that zstd has had", () => { expect(await through(zlib.createZstdDecompress(), pieces(frame, 1))).toEqual(input); expect(await through(zlib.createZstdDecompress(), [frame])).toEqual(input); } - } - }); + }, + ); it("a larger window is decoded if allowed", async () => { const [, sync, streamed] = await decoded(hex("28b52ffd0090190000616263"), { @@ -2048,22 +2042,23 @@ describe("bugs that zstd has had", () => { for (const size of [1 << 20, 64 << 10, 1000]) expect(await cut(size)).toBe("1398687 a871b693"); }); - it("make the same twice", async () => { + it.each([ + [3, 0], + [3, 1], + [16, 0], + [16, 1], + ])("make the same twice: level %i, long distance matching %i", async (level, longDistance) => { const input = noise(3 << 19, 23, 7); - for (const longDistance of [0, 1]) { - for (const level of [3, 16]) { - const params = { - [c.ZSTD_c_nbWorkers]: 2, - [c.ZSTD_c_jobSize]: 512 << 10, - [c.ZSTD_c_compressionLevel]: level, - [c.ZSTD_c_enableLongDistanceMatching]: longDistance, - }; - const made = () => through(zlib.createZstdCompress({ params }), pieces(input, 100_000)) as Promise; - const [frame, again] = await Promise.all([made(), made()]); - expect(frame.equals(again)).toBe(true); - expect(zlib.zstdDecompressSync(frame).equals(input)).toBe(true); - } - } + const params = { + [c.ZSTD_c_nbWorkers]: 2, + [c.ZSTD_c_jobSize]: 512 << 10, + [c.ZSTD_c_compressionLevel]: level, + [c.ZSTD_c_enableLongDistanceMatching]: longDistance, + }; + const made = () => through(zlib.createZstdCompress({ params }), pieces(input, 100_000)) as Promise; + const [frame, again] = await Promise.all([made(), made()]); + expect(frame.equals(again)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(input)).toBe(true); }); it("with nothing left for the last job", async () => { From 1c78c763d78e8563c48eb1243f537bbbb41a35c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 05:24:25 +0000 Subject: [PATCH 143/199] zstd: fast and dfast ask whether there is a repeat offset only once its bytes match The compiler put that question first, at two instructions per turn of the inner loop. Release build with LTO, per call: 4 KB of text at level 1 155,022 to 152,409 instructions and 35,798 to 35,314 cycles, 100 KB 2,766,667 to 2,712,282 and 587,663 to 581,710; at level 3 4 KB 191,655 to 186,320 and 43,089 to 42,142, 100 KB 3,029,924 to 2,943,608 and 730,069 to 719,297. --- src/zstd/compress/double_fast.rs | 4 ++-- src/zstd/compress/fast.rs | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 113063097f4d..b0a4a032caee 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -2,7 +2,7 @@ use super::fast::{ DictBlock, ExtDictBlock, Found, MatchSink, NoDictBlock, Segments, TaggedTable, ThirdPositions, - put_tagged, put_tagged_in_between, restore_rep, + after, put_tagged, put_tagged_in_between, restore_rep, }; use super::internal::{ BlockSrc, HASH_READ_SIZE, Hasher, K_SEARCH_STRENGTH, MatchState, Near, Prefix, @@ -141,7 +141,7 @@ impl Finder { let idxs0 = hashed_swap(hash_small, small.hash(bytes0), curr); if sink.read32_before(ip + 1, offset_1) == Some((bytes0 >> 8) as u32) - && offset_1 > 0 + && after(offset_1 > 0) { ip += 1; let end = ip + 4 + sink.count_before(ip + 4, offset_1); diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 6550ab8d91ce..eda9a2b24ae6 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -552,6 +552,13 @@ pub(super) struct Found { pub(super) offset: u32, } +/// `condition`, which the compiler then does not look at before what is on its left. +#[inline(always)] +pub(super) fn after(condition: bool) -> bool { + compiler_fence(Ordering::SeqCst); + condition +} + /// What `fast` looks for matches with, when there is no dictionary and the window is one segment. struct Finder { /// The index of the first position. @@ -615,7 +622,7 @@ impl Finder { let idx0 = hashed_swap(hash_table, hasher.hash(bytes0), current0); if sink.read32_before(ip2, rep_offset1) == Some(le32_at::<0, _>(ahead)) - && rep_offset1 > 0 + && after(rep_offset1 > 0) { hashed_put(hash_table, hasher.hash(bytes0 >> 8), current0 + 1); let back = u32::from(sink.same_byte_before(ip2, rep_offset1)); From 1df8f8c3921a5af3764caa4e1d180ca0f48eed64 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 05:24:25 +0000 Subject: [PATCH 144/199] zstd: the length of a match is a comparison of vectors, the first one outside the loop Two 128-bit integers were compared as vectors and then read again as four words to find the first difference, after the registers of a loop were set up that 9 matches in 10 do not enter. Release build with LTO, per call: 4 KB of text at level 1 152,409 to 149,268 instructions and 35,314 to 35,033 cycles, 100 KB 2,712,282 to 2,619,071 and 581,710 to 575,921; at level 3 4 KB 195,142 to 191,655, 100 KB 3,124,775 to 3,029,924. 15 MB of text at level 1 443 to 456 MB/s. --- src/zstd/compress/internal.rs | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 1cf70df9914f..5ac4c58a3abf 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -807,35 +807,27 @@ impl<'a> Near<'a> { /// `ZSTD_count`, up to the end. `match_pos` is less than `pos`. #[inline(always)] pub(crate) fn count(self, pos: u32, match_pos: u32) -> u32 { - let mut len = 0; - // Where 16 bytes differ takes no branch, and few matches are longer. - while let (Some(bytes), Some(match_bytes)) = - (self.chunk16(pos + len), self.chunk16(match_pos + len)) - { - let diff = u128::from_le_bytes(*bytes) ^ u128::from_le_bytes(*match_bytes); - if diff != 0 { - return len + (diff.trailing_zeros() >> 3); - } - len += 16; - } - while let Some(byte) = self.read8(pos + len) - && self.read8(match_pos + len) == Some(byte) - { - len += 1; - } - len + self.count_in(pos, self, match_pos) } /// How long what is at `pos` is the same as what is at `other_pos` in `other`, up to the end of either. #[inline(always)] pub(crate) fn count_in(self, pos: u32, other: Near<'_>, other_pos: u32) -> u32 { let mut len = 0; + // Where 16 bytes differ takes no branch, and few matches are longer: what a loop has to prepare is left for those. + if let (Some(bytes), Some(other_bytes)) = (self.chunk16(pos), other.chunk16(other_pos)) { + let differ = differences(bytes, other_bytes); + if differ != 0 { + return differ.trailing_zeros(); + } + len = 16; + } while let (Some(bytes), Some(other_bytes)) = (self.chunk16(pos + len), other.chunk16(other_pos + len)) { - let diff = u128::from_le_bytes(*bytes) ^ u128::from_le_bytes(*other_bytes); - if diff != 0 { - return len + (diff.trailing_zeros() >> 3); + let differ = differences(bytes, other_bytes); + if differ != 0 { + return len + differ.trailing_zeros(); } len += 16; } From 7245ee8033e8677a2e69f083906fb01d141467fa Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 05:23:00 +0000 Subject: [PATCH 145/199] zstd: the lookups between two reloads of four Huffman streams are written out The release build (ThinLTO at O2) left them a loop of 4 or 5 turns, and with it a check of the position in the window at every lookup. 4 KB of text, level 1, per call of decompress_alloc: 75,700 to 71,970 instructions, 17,250 to 16,620 cycles. 15 MB of text and 20 MB of binaries: 1 to 3% faster at all levels. --- src/zstd/decompress/huf.rs | 55 +++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index ccda8954d847..edc5c98e158c 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -782,12 +782,39 @@ impl<'a> Streams<'a> { } } + /// `LOOKUPS` in each stream. Not a loop: one that is not unrolled checks `done` at every turn. + #[inline(always)] + fn round( + table: &[Entry; TABLE_SIZE], + bits: [&mut u64; 4], + windows: &mut [&mut [u8; WINDOW]; 4], + done: &mut [usize; 4], + ) { + let [bits0, bits1, bits2, bits3] = bits; + let [window0, window1, window2, window3] = windows; + let [done0, done1, done2, done3] = done; + let mut lookups = || { + lookup::(table, bits0, window0, done0); + lookup::(table, bits1, window1, done1); + lookup::(table, bits2, window2, done2); + lookup::(table, bits3, window3, done3); + }; + lookups(); + lookups(); + lookups(); + lookups(); + if LOOKUPS > 4 { + lookups(); + } + } + #[inline(always)] fn window(out: &mut [u8], op: u32) -> Option<&mut [u8; WINDOW]> { out.get_mut(op as usize..op as usize + WINDOW)? .as_mut_array() } + const { assert!(LOOKUPS == 4 || LOOKUPS == 5) }; let [s0, s1, s2, s3] = &mut self.0; let (mut bits0, mut bits1, mut bits2, mut bits3) = (s0.bits, s1.bits, s2.bits, s3.bits); let (mut ip0, mut ip1, mut ip2, mut ip3) = (s0.ip, s1.ip, s2.ip, s3.ip); @@ -801,29 +828,31 @@ impl<'a> Streams<'a> { ) else { break; }; - let (mut done0, mut done1, mut done2, mut done3) = (0, 0, 0, 0); - for _ in 0..LOOKUPS { - lookup::(table, &mut bits0, window0, &mut done0); - lookup::(table, &mut bits1, window1, &mut done1); - lookup::(table, &mut bits2, window2, &mut done2); - lookup::(table, &mut bits3, window3, &mut done3); - } + let mut windows = [window0, window1, window2, window3]; + let mut done = [0; 4]; + round::( + table, + [&mut bits0, &mut bits1, &mut bits2, &mut bits3], + &mut windows, + &mut done, + ); let mut more = reload(s0.src, &mut ip0, &mut bits0) && reload(s1.src, &mut ip1, &mut bits1) && reload(s2.src, &mut ip2, &mut bits2) && reload(s3.src, &mut ip3, &mut bits3); if more { - for _ in 0..LOOKUPS { - lookup::(table, &mut bits0, window0, &mut done0); - lookup::(table, &mut bits1, window1, &mut done1); - lookup::(table, &mut bits2, window2, &mut done2); - lookup::(table, &mut bits3, window3, &mut done3); - } + round::( + table, + [&mut bits0, &mut bits1, &mut bits2, &mut bits3], + &mut windows, + &mut done, + ); more = reload(s0.src, &mut ip0, &mut bits0) && reload(s1.src, &mut ip1, &mut bits1) && reload(s2.src, &mut ip2, &mut bits2) && reload(s3.src, &mut ip3, &mut bits3); } + let [done0, done1, done2, done3] = done; op0 += done0 as u32; op1 += done1 as u32; op2 += done2 as u32; From e5c4c3cf8fbf1d160d26c23f6639f892e3e99dfe Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 05:32:25 +0000 Subject: [PATCH 146/199] zstd: few literals of long codes are decoded one to a lookup Below 2048 literals the table is indexed with as few bits as the longest code has (8 at least), and at 5 bits or more to a literal nearly every lookup of 8 or 9 bits yields one symbol. A table where all do is filled sooner, and a loop that knows it needs no count of what each lookup wrote: 6 instructions to a symbol and not 10, 7 lookups between reloads and not 5, and it stops 7 symbols from the end of the output and not 32, which leaves less to the tails. The entries have the same form, so all else decodes with such a table as it is. It is for the block that it comes with: for a block that uses the table again, it is filled as before. A table that holds no third symbols clears no room for them. 7 lookups of 8 bits are 56 bits, which a stream has after a reload but not before the first one, where the end mark can take 8 of the 63. So the loop reloads before the lookups, and how many bits the lookups of either loop can take is checked when it is compiled. Built as a release build is, on an AMD EPYC. 4 KB of text, level 1, per call of decompress_alloc: 85,080 to 80,430 instructions, 18,740 to 18,140 cycles (C: 75,800 and 18,070). Some 3,000 different frames in turn, level 1, against C: 700 bytes 1.05 to 0.98, 2000 bytes 0.98 to 0.93, 4000 bytes 0.97 to 0.93. --- src/zstd/decompress/huf.rs | 135 ++++++++++++++++++++++++++++++---- test/js/bun/util/zstd.test.ts | 69 +++++++++++++++++ 2 files changed, 190 insertions(+), 14 deletions(-) diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index edc5c98e158c..3516178b201c 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -33,6 +33,9 @@ const FAST_LOG: u32 = 11; const FAST_MIN_LITERALS: usize = 2048; /// From how many literals on the larger table is considered. const LARGE_MIN_LITERALS: usize = 1 << 14; +/// From how many bits to a literal on few lookups of at most `SINGLE_MAX_LOG` bits would yield more than one. +const SINGLE_MIN_BITS: usize = 5; +const SINGLE_MAX_LOG: u32 = 9; impl Code { /// How many symbols a lookup of `log` bits yields, in 65536ths, if symbols are as frequent as their codes are short. @@ -87,6 +90,11 @@ impl Code { } FAST_LOG } + + /// Whether four streams of `nb_bytes` in all are decoded sooner to `nb_literals` one to a lookup of `log` bits. + fn is_best_single(log: u32, nb_literals: usize, nb_bytes: usize) -> bool { + log <= SINGLE_MAX_LOG && 8 * nb_bytes >= SINGLE_MIN_BITS * nb_literals + } } /// What a lookup yields. From the lowest byte on: up to 3 symbols, then how many bits they take in 6 bits and how many @@ -263,6 +271,8 @@ pub(crate) struct HufDTable { table_type: TableType, /// How many bits `table` is indexed with. log: u32, + /// Every entry of `table` is one symbol. + single: bool, table: [Entry; TABLE_SIZE], } @@ -277,17 +287,19 @@ impl HufDTable { }, table_type: TableType::X1, log: 0, + single: false, table: [0; TABLE_SIZE], } } /// Reads the description of a table at the start of `src`, and returns its size. The table is to decode - /// `nb_literals` with. + /// `nb_literals` with, from the rest of `src` if that is `four_streams`. fn read( &mut self, src: &[u8], table_type: TableType, nb_literals: usize, + four_streams: bool, spread: &mut Spread, ) -> Result { let Code { @@ -299,25 +311,33 @@ impl HufDTable { self.code.nb_symbols = stats.nb_symbols; self.code.table_log = stats.table_log; self.table_type = table_type; - self.build(self.code.best_log(nb_literals))?; + let log = self.code.best_log(nb_literals); + let nb_bytes = src.len().saturating_sub(stats.size); + let single = four_streams && Code::is_best_single(log, nb_literals, nb_bytes); + self.build(log, single)?; Ok(stats.size) } - /// Makes the table fit to decode `nb_literals` with. + /// Makes the table fit to decode `nb_literals` of another block with. pub(crate) fn prepare(&mut self, nb_literals: usize) -> Result<()> { if self.log < FAST_LOG && nb_literals >= FAST_MIN_LITERALS { - self.build(self.code.table_log.max(FAST_LOG))?; + self.build(self.code.table_log.max(FAST_LOG), false)?; + } else if self.single { + self.build(self.log, false)?; } Ok(()) } /// Fills `table` for lookups of `log` bits, which is at least as many as the longest code has. - fn build(&mut self, log: u32) -> Result<()> { + fn build(&mut self, log: u32, single: bool) -> Result<()> { self.log = log; + self.single = single; let sorted = Sorted::new(&self.code, log)?; // The tables of what a third symbol adds end with the one for this many bits. let room = log.saturating_sub(2 * sorted.shortest()); - if room < 7 { + if single || room < sorted.shortest() { + self.fill(&sorted, &mut []) + } else if room < 7 { self.fill(&sorted, &mut [0; 1 << 7]) } else { self.fill(&sorted, &mut [0; TABLE_SIZE / 2]) @@ -326,8 +346,9 @@ impl HufDTable { fn fill(&mut self, sorted: &Sorted, thirds: &mut [Entry]) -> Result<()> { let log = sorted.log; - let shortest = sorted.shortest(); - for room in shortest..=log.saturating_sub(2 * shortest) { + // From how many bits on what a symbol leaves of a lookup holds another. + let min_room = if self.single { log } else { sorted.shortest() }; + for room in min_room..=log.saturating_sub(2 * min_room) { sorted.fill(thirds.span_mut(1 << room, 1 << room)?, room, 0, 2, None)?; } for weight in 1..=sorted.max_weight { @@ -337,7 +358,7 @@ impl HufDTable { let entries = self .table .span_mut(sorted.position(weight, log)?, symbols.len() << room)?; - if room < shortest { + if room < min_room { fill_runs(entries, symbols, room, entry(0, 0, nb_bits), 0); continue; } @@ -381,7 +402,11 @@ impl HufDTable { return false; }; if streams.0.iter().all(|stream| stream.bits != 0) { - self.decode_most(&mut streams); + if self.single { + self.decode_most_single(&mut streams); + } else { + self.decode_most(&mut streams); + } } if !streams.0.iter().all(Stream::fits_tail) { return self.decode_stragglers(streams); @@ -400,6 +425,15 @@ impl HufDTable { } } + /// The same with a table where every entry is one symbol. + #[inline(never)] + fn decode_most_single(&self, streams: &mut Streams<'_>) { + match self.log { + 8 => streams.decode_most_single::<8, 7>(&self.table), + _ => streams.decode_most_single::(&self.table), + } + } + /// The same for streams of which some have much left. #[inline(never)] fn decode_stragglers(&self, streams: Streams<'_>) -> bool { @@ -814,7 +848,7 @@ impl<'a> Streams<'a> { .as_mut_array() } - const { assert!(LOOKUPS == 4 || LOOKUPS == 5) }; + const { assert!((LOOKUPS == 4 || LOOKUPS == 5) && LOG as usize * LOOKUPS <= 55) }; let [s0, s1, s2, s3] = &mut self.0; let (mut bits0, mut bits1, mut bits2, mut bits3) = (s0.bits, s1.bits, s2.bits, s3.bits); let (mut ip0, mut ip1, mut ip2, mut ip3) = (s0.ip, s1.ip, s2.ip, s3.ip); @@ -865,6 +899,79 @@ impl<'a> Streams<'a> { (s0.ip, s1.ip, s2.ip, s3.ip) = (ip0, ip1, ip2, ip3); (s0.op, s1.op, s2.op, s3.op) = (op0, op1, op2, op3); } + + /// The same with a table where every entry is one symbol. `LOOKUPS` of `LOG` bits are at most 56 bits. + #[inline(always)] + fn decode_most_single( + &mut self, + table: &[Entry; TABLE_SIZE], + ) { + #[inline(always)] + fn lookup( + table: &[Entry; TABLE_SIZE], + bits: &mut u64, + to: Option<&mut u8>, + ) { + let index = (*bits >> (64 - LOG)) as usize; + // The highest byte of the entry, read on its own where that is simple: the next lookup waits for it. + let nb_bits = if cfg!(target_endian = "little") { + bun_pod::bytes_of(table).get(4 * index + 3).copied() + } else { + table.get(index).map(|entry| (entry >> 24) as u8) + }; + if let (Some(to), Some(nb_bits), Some(&entry)) = (to, nb_bits, table.get(index)) { + *bits <<= nb_bits & 63; + *to = entry as u8; + } + } + + #[inline(always)] + fn rounds<'a, const LOOKUPS: usize>(stream: &'a mut Stream<'_>) -> &'a mut [[u8; LOOKUPS]] { + let left = stream.out.get_mut(stream.op as usize..).unwrap_or_default(); + left.as_chunks_mut().0 + } + + const { assert!(LOOKUPS <= 7 && LOG as usize * LOOKUPS <= 56) }; + let [s0, s1, s2, s3] = &mut self.0; + let (mut bits0, mut bits1, mut bits2, mut bits3) = (s0.bits, s1.bits, s2.bits, s3.bits); + let (mut ip0, mut ip1, mut ip2, mut ip3) = (s0.ip, s1.ip, s2.ip, s3.ip); + let (src0, src1, src2, src3) = (s0.src, s1.src, s2.src, s3.src); + let mut nb_rounds = 0; + for (((out0, out1), out2), out3) in rounds::(s0) + .iter_mut() + .zip(rounds::(s1)) + .zip(rounds::(s2)) + .zip(rounds::(s3)) + { + // A stream that is not reloaded yet can have as few as 55 bits. + if !(reload(src0, &mut ip0, &mut bits0) + && reload(src1, &mut ip1, &mut bits1) + && reload(src2, &mut ip2, &mut bits2) + && reload(src3, &mut ip3, &mut bits3)) + { + break; + } + let mut lookups = |index: usize| { + lookup::(table, &mut bits0, out0.get_mut(index)); + lookup::(table, &mut bits1, out1.get_mut(index)); + lookup::(table, &mut bits2, out2.get_mut(index)); + lookup::(table, &mut bits3, out3.get_mut(index)); + }; + lookups(0); + lookups(1); + lookups(2); + lookups(3); + lookups(4); + lookups(5); + lookups(6); + nb_rounds += 1; + } + (s0.bits, s1.bits, s2.bits, s3.bits) = (bits0, bits1, bits2, bits3); + (s0.ip, s1.ip, s2.ip, s3.ip) = (ip0, ip1, ip2, ip3); + for stream in &mut self.0 { + stream.op += nb_rounds * LOOKUPS as u32; + } + } } /// `HUF_readDTableX2_wksp`. Returns the size of the table's description in `src`. @@ -873,7 +980,7 @@ pub(crate) fn read_dtable_x2( src: &[u8], spread: &mut Spread, ) -> Result { - dtable.read(src, TableType::X2, usize::MAX, spread) + dtable.read(src, TableType::X2, usize::MAX, false, spread) } /// `HUF_decompress1X_usingDTable` @@ -895,7 +1002,7 @@ pub(crate) fn decompress_1x1( src: &[u8], spread: &mut Spread, ) -> Result { - let h_size = dtable.read(src, TableType::X1, dst.len(), spread)?; + let h_size = dtable.read(src, TableType::X1, dst.len(), false, spread)?; if h_size >= src.len() { return Err(Error::SrcSizeWrong); } @@ -932,7 +1039,7 @@ pub(crate) fn decompress_4x_huf_only( } else { TableType::X1 }; - let h_size = dtable.read(src, table_type, dst.len(), spread)?; + let h_size = dtable.read(src, table_type, dst.len(), true, spread)?; if h_size >= src.len() { return Err(Error::SrcSizeWrong); } diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 8ea383f9d421..c29c6e8bb685 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -109,6 +109,75 @@ describe("Zstandard compression", async () => { ).toBeLessThan(10); } + describe("few literals whose codes are long", () => { + // How often each of how many symbols is there, out of 1536: a code of 5 bits for 48 times, of 9 bits for 3 times. + const upTo8Bits = [ + [48, 8], + [6, 192], + ]; + const upTo9Bits = [ + [48, 8], + [6, 140], + [3, 104], + ]; + let x = 1; + /** In a random order, but that each quarter, which is a Huffman stream of its own, starts with the longest codes. */ + function literals(mix: number[][]) { + const input: number[] = []; + let symbol = 0; + for (const [count, nbSymbols] of mix) { + for (const end = symbol + nbSymbols; symbol < end; symbol++) input.push(...Array(count).fill(symbol)); + } + const isRare = (other: number) => other >= symbol - mix.at(-1)![1]; + for (let i = input.length - 1; i > 0; i--) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + const j = (x >>> 0) % (i + 1); + [input[i], input[j]] = [input[j], input[i]]; + } + for (let start = 0; start < input.length; start += input.length / 4) { + for (let i = start, from = start + 8; i < start + 8; i++) { + while (!isRare(input[from])) from++; + if (!isRare(input[i])) [input[i], input[from]] = [input[from], input[i]]; + } + } + return Uint8Array.from(input); + } + + // Where a stream ends within a byte, which is what varies, is how many bits there are before more have to be read. + it.each([ + [8, upTo8Bits], + [9, upTo9Bits], + ])("of up to %i bits", (_, mix) => { + for (let round = 0; round < 40; round++) { + const input = literals(mix); + const frame = zstdCompressSync(input); + expect(frame.length).toBeLessThan(1500); + expect(Buffer.compare(zstdDecompressSync(frame), input)).toBe(0); + } + }); + + it("with a table that later blocks use again", () => { + // Blocks of a target size end with a match. + const input = Buffer.concat( + Array.from({ length: 4 }, () => literals(upTo8Bits)).flatMap(part => [part, part.subarray(0, 64)]), + ); + const ZSTD_c_targetCBlockSize = 130; + const frame = zlib.zstdCompressSync(input, { params: { [ZSTD_c_targetCBlockSize]: 1340 } }); + // The blocks are after the magic number, the descriptor and a content size of 2 bytes. + expect(frame[4]).toBe(0x60); + const literalsTypes: number[] = []; + for (let at = 7, header = 0; !(header & 1); at += 3 + (header >> 3)) { + header = frame.readUIntLE(at, 3); + literalsTypes.push(frame[at + 3] & 3); + } + const [compressed, treeless] = [2, 3]; + expect(literalsTypes).toEqual([compressed, treeless, treeless, treeless]); + expect(Buffer.compare(zstdDecompressSync(frame), input)).toBe(0); + }); + }); + // Test with known zstd-compressed data describe("zstd CLI compatibility", () => { for (const { name, compressed, original } of [ From efbf33dd4930a4bf6f258e244f7d93a1d45eddee Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 08:50:24 +0000 Subject: [PATCH 147/199] zstd: the room for a frame is zeroed block by block compress_alloc and compress_append made zeros of all the room that a frame can take, which is as much as its input, before safe code could write to it, although a third of it or less is written. Now the room is capacity, and the length of the Vec grows ahead of each block by what that block can take. A block in one part gets room for itself and 1 KB. More makes no difference: what comes out no smaller than it went in is stored as it is. (With 8 bytes to spare instead of 1024, 19,000 frames still come out as from C; with 4 they do not.) Where a block is made in parts (levels 16 and up, or a target size), each of which is stored or not, it is not that simple: there can be more than 197 parts, and the last one can be a few bytes short of what the encoders want beyond what they write. Those get all the room at once, as before; they take too long for zeros to matter. Bun.zstdCompressSync, release build, against Bun 1.4.4, best and median of 48 calls. 20 MB of an executable at level 1: 0.99 and 1.00 to 0.96 and 0.97 (cycles in the kernel per call 20.2 to 15.2 million, C 14.9); at level 3: 0.93 and 0.94 to 0.93 and 0.92. 15 MB of text at level 1: 0.94 and 0.95 to 0.92 and 0.92. Inputs of one block are as before. --- src/zstd/compress/cctx.rs | 76 ++++++++++++++++++++++++++++++----- src/zstd/compress/stream.rs | 13 +++++- src/zstd/lib.rs | 54 ++++++++++++++++++++++--- test/js/bun/util/zstd.test.ts | 18 +++++++++ 4 files changed, 143 insertions(+), 18 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 5908a1b3cc22..0f89fa62d37d 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -109,6 +109,38 @@ pub(crate) struct Input<'a> { pub(crate) cdict: Option<&'a CDict>, } +/// Where a frame goes. +pub(crate) enum Output<'a> { + Slice(&'a mut [u8]), + /// What is from `start` to `end` in `vec`, which has the capacity for it. Only what is asked for becomes part of its + /// length, as zeros: most of the room for a frame is never written to. + Spare { + vec: &'a mut Vec, + start: usize, + end: usize, + }, +} + +/// The header of a block, and far more than any encoder wants beyond what it writes. +const ROOM_TO_SPARE: usize = 1024; + +impl Output<'_> { + /// What there is from `pos` on, or `needed` bytes of it at least. + fn from(&mut self, pos: usize, needed: usize) -> Result<&mut [u8]> { + match self { + Output::Slice(slice) => slice.sub_mut(pos..), + Output::Spare { vec, start, end } => { + let pos = *start + pos; + let len = pos.saturating_add(needed).min(*end); + if vec.len() < len { + vec.resize(len, 0); + } + vec.sub_mut(pos..) + } + } + } +} + fn window_prefix<'a>(window: &Window, bufs: &Buffers<'a>) -> Prefix<'a> { Prefix { buf: bufs.get(window.base_buf), @@ -1635,14 +1667,15 @@ impl FrameCompressor { /// `ZSTD_compress_frameChunk` fn compress_frame_chunk( &mut self, - dst: &mut [u8], + dst: &mut Output<'_>, + start: usize, input: &Input<'_>, src: Span, last_frame_chunk: bool, ) -> Result { let bytes = src.bytes(&input.bufs)?; let mut ip = src.start; - let mut op = 0usize; + let mut op = start; let max_dist = 1u32 << self.applied_params.c_params.window_log; let mut savings = self.consumed_src_size as i64 - self.produced_c_size as i64; @@ -1660,7 +1693,18 @@ impl FrameCompressor { start: ip, end: ip + block_size, }; - let out = dst.sub_mut(op..)?; + let in_parts = self.applied_params.target_c_block_size != 0 + || self.applied_params.post_block_splitter == ParamSwitch::Enable; + // A block in one part that comes out no smaller than it went in is stored as it is, so it makes no difference + // when an encoder runs out of this much room. Parts, which take long to make anyway, get all there is. + let out = dst.from( + op, + if in_parts { + usize::MAX + } else { + block_size + ROOM_TO_SPARE + }, + )?; if out.len() < ZSTD_BLOCKHEADERSIZE + MIN_CBLOCK_SIZE + 1 { return Err(Error::DstSizeTooSmall); @@ -1707,16 +1751,16 @@ impl FrameCompressor { self.is_first_block = false; } - if last_frame_chunk && op > 0 { + if last_frame_chunk && op > start { self.stage = CompressionStage::Ending; } - Ok(op) + Ok(op - start) } /// `ZSTD_compressContinue_internal` fn compress_continue_internal( &mut self, - dst: &mut [u8], + dst: &mut Output<'_>, input: &Input<'_>, src: Span, last_frame_chunk: bool, @@ -1728,7 +1772,7 @@ impl FrameCompressor { if self.stage == CompressionStage::Init { fh_size = write_frame_header( - dst, + dst.from(0, ZSTD_FRAMEHEADERSIZE_MAX)?, &self.applied_params, self.pledged_src_size_plus_one.wrapping_sub(1), self.dict_id, @@ -1754,8 +1798,7 @@ impl FrameCompressor { (self.ldm_state.window).update(src.start_addr(), src.buf, src.len(), false); } - let c_size = - self.compress_frame_chunk(dst.sub_mut(fh_size..)?, input, src, last_frame_chunk)?; + let c_size = self.compress_frame_chunk(dst, fh_size, input, src, last_frame_chunk)?; self.consumed_src_size += src.len() as u64; self.produced_c_size += (c_size + fh_size) as u64; if self.pledged_src_size_plus_one != 0 @@ -1773,7 +1816,7 @@ impl FrameCompressor { input: &Input<'_>, src: Span, ) -> Result { - self.compress_continue_internal(dst, input, src, false) + self.compress_continue_internal(&mut Output::Slice(dst), input, src, false) } /// `ZSTD_writeEpilogue` @@ -1818,8 +1861,19 @@ impl FrameCompressor { input: &Input<'_>, src: Span, ) -> Result { + self.compress_end_to(&mut Output::Slice(dst), input, src) + } + + /// [`Self::compress_end`] to either kind of output. + pub(crate) fn compress_end_to( + &mut self, + dst: &mut Output<'_>, + input: &Input<'_>, + src: Span, + ) -> Result { + const EPILOGUE_SIZE_MAX: usize = ZSTD_BLOCKHEADERSIZE + 4; let c_size = self.compress_continue_internal(dst, input, src, true)?; - let end_result = self.write_epilogue(dst.sub_mut(c_size..)?)?; + let end_result = self.write_epilogue(dst.from(c_size, EPILOGUE_SIZE_MAX)?)?; if self.pledged_src_size_plus_one != 0 && self.pledged_src_size_plus_one != self.consumed_src_size + 1 { diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs index 492f07fd21b2..3926ab4a877f 100644 --- a/src/zstd/compress/stream.rs +++ b/src/zstd/compress/stream.rs @@ -2,7 +2,7 @@ use std::sync::Arc; -use super::cctx::{CDict, FrameCompressor, Input, Span}; +use super::cctx::{CDict, FrameCompressor, Input, Output, Span}; use super::internal::{BufId, Buffers, CompressedBlockState}; use super::mt::MtCtx; use super::params::{ @@ -477,6 +477,15 @@ impl CCtx { /// `ZSTD_compress`: returns the size of the frame. pub fn compress(dst: &mut [u8], src: &[u8], compression_level: i32) -> Result { + compress_to(&mut Output::Slice(dst), src, compression_level) +} + +/// [`compress`] to either kind of output. +pub(crate) fn compress_to( + dst: &mut Output<'_>, + src: &[u8], + compression_level: i32, +) -> Result { let mut frame = FrameCompressor::new(); let params = get_params_internal( compression_level, @@ -501,7 +510,7 @@ pub fn compress(dst: &mut [u8], src: &[u8], compression_level: i32) -> Result { pub fn compress_append(out: &mut Vec, src: &[u8], level: i32) -> Result { let start = out.len(); let room = (out.capacity() - start).min(compress_bound(src.len())); - out.resize(start + room, 0); - let result = compress(out.get_mut(start..).unwrap_or_default(), src, level); + let result = compress_to( + &mut Output::Spare { + vec: out, + start, + end: start + room, + }, + src, + level, + ); out.truncate(start + result.unwrap_or(0)); result } /// [`compress`] to a new buffer. pub fn compress_alloc(src: &[u8], level: i32) -> Result> { - let mut out = util::try_zeroed(compress_bound(src.len()))?; - let size = compress(&mut out, src, level)?; - out.truncate(size); + let mut out = Vec::new(); + out.try_reserve_exact(compress_bound(src.len())) + .map_err(|_| Error::MemoryAllocation)?; + compress_append(&mut out, src, level)?; Ok(out) } @@ -441,6 +451,7 @@ pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { #[cfg(test)] mod tests { use super::*; + use util::SliceExt; #[test] fn little_more_is_zeroed_than_is_written() { @@ -456,4 +467,37 @@ mod tests { out.advance(room); } } + + #[test] + #[cfg_attr(miri, ignore = "takes over 10 minutes")] + fn room_that_grows_gives_what_room_that_is_there_gives() -> Result<()> { + // Blocks that compress well, hardly, and not at all. + let mut state = 1u32; + let src: Vec = (0..400_000u32) + .map(|i| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + match i >> 17 { + 0 => (state % 5) as u8 * (i % 7) as u8, + 1 => (state >> 24) as u8, + _ => (state >> 24) as u8 | 1, + } + }) + .collect(); + for level in [1, 3, 6, 13, 16] { + let mut whole = vec![0; compress_bound(src.len())]; + let size = compress(&mut whole, &src, level)?; + assert!(compress_alloc(&src, level)? == *whole.sub(..size)?); + for capacity in [whole.len() + 1000, size, size - 1, size / 2, 0] { + let mut out = Vec::with_capacity(1 + capacity); + out.push(7); + let room = (out.capacity() - 1).min(whole.len()); + let appended = compress_append(&mut out, &src, level); + assert_eq!(appended, compress(whole.sub_mut(..room)?, &src, level)); + assert!(out.split_first() == Some((&7, whole.sub(..appended.unwrap_or(0))?))); + } + } + Ok(()) + } } diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index c29c6e8bb685..5b589cc6d0c4 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1219,6 +1219,24 @@ describe("the output is that of zstd 1.5.7", () => { expect(`${digest(compress(data))}, ${digest(compress(small))}`).toBe(expected); }); + it("where blocks that are stored are between blocks that are compressed", () => { + const noise = new Uint8Array(200_000); + for (let i = 0, x = 9; i < noise.length; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + noise[i] = x; + } + const data = Buffer.concat([large.subarray(0, 150_000), noise, large.subarray(150_000)]); + expect([1, 3, 6, 13, 16].map(level => digest(zstdCompressSync(data, { level })))).toEqual([ + "248567 de53cc88abf776bf", + "245529 c13322b7116144a", + "245428 8fdcf68c52aa046d", + "244844 ad91ca42d335a19", + "244843 e1282705ef659832", + ]); + }); + it("with parameters", () => { const cases: Record> = { checksum: { [c.ZSTD_c_checksumFlag]: 1 }, From 7f195c7a28520793806f6fa9dafab91eb3d194c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 13:37:28 +0000 Subject: [PATCH 148/199] zstd: the counts of a table are read with less to wait for Each count had to wait about 13 cycles for the one before: what is left to share out was kept on the stack, the threshold was worked out again for every symbol, and a run of symbols without a count always loaded new bits. Whether a value is stored with a bit less, and whether it stands for less than it says, are now two comparisons that need one addition each, what is left is one of two values that are known beforehand, the threshold changes in a branch that is rarely taken, and a run is read from the bits that are there if they are enough. 4.0 to 2.2 ns a symbol for the same headers over and over (C 2.4), 4.3 to 3.7 for different ones (C 4.3). 3% of the time to decode 4 KB. --- src/zstd/common/entropy.rs | 122 ++++++++++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 30 deletions(-) diff --git a/src/zstd/common/entropy.rs b/src/zstd/common/entropy.rs index f69c82de1212..85079c2a654a 100644 --- a/src/zstd/common/entropy.rs +++ b/src/zstd/common/entropy.rs @@ -31,12 +31,13 @@ fn read_valid_ncount( max_symbol_value: u32, header: &[u8], ) -> Option { - /// The bits from `position` on, from the lowest on: 57 of them or more, then zeroes. + /// The bits from `position` on, from the lowest on: `MIN_LOADED` of them or more, then zeroes. #[inline(always)] fn load(header: &[u8], position: usize) -> Option { let bytes = header.get(position / 8..position / 8 + 8)?; Some(u64::from_le_bytes(bytes.try_into().ok()?) >> (position % 8)) } + const MIN_LOADED: u32 = 57; /// After how many bits more are loaded. The 17 that are left are enough for a count. const MAX_USED: u32 = 40; @@ -57,51 +58,56 @@ fn read_valid_ncount( let mut threshold = 1i32 << table_log; let mut nb_bits = table_log + 1; let mut symbol = 0usize; - let mut previous0 = false; - while remaining > 1 { - if previous0 { - // Each pair of set bits stands for three more symbols of count 0. - loop { - position += used as usize; - used = 0; - bits = load(header, position)?; - let repeats = ((!bits).trailing_zeros() / 2).min(24); - symbol += 3 * repeats as usize; - bits >>= 2 * repeats; - used += 2 * repeats; - if repeats < 24 { - break; - } - } - symbol += (bits & 3) as usize; - bits >>= 2; - used += 2; - } + loop { if used > MAX_USED { position += used as usize; used = 0; bits = load(header, position)?; } - let max = 2 * threshold - 1 - remaining; + // With `max` for `2 * threshold - 1 - remaining`: a value below `max` is stored with a bit less, and one from + // `threshold + max` on stands for `max` less. Written so that little has to wait for `remaining`. let low = bits as i32 & (threshold - 1); let all = bits as i32 & (2 * threshold - 1); - let is_short = low < max; - let long = all - if all >= threshold { max } else { 0 }; - // The stored value is the count plus one, and a count of -1 takes one slot. - let count = if is_short { low } else { long } - 1; - let count_bits = nb_bits - u32::from(is_short); + let count_bits = nb_bits - u32::from(low + remaining < 2 * threshold - 1); bits >>= count_bits; used += count_bits; - remaining -= count.abs(); + // The stored value is the count plus one, and a count of -1 takes one slot. + let count; + if all + remaining >= 3 * threshold - 1 { + count = all + remaining - 2 * threshold; + remaining = 2 * threshold - all; + } else { + count = low - 1; + remaining -= count.abs(); + } *counts.get_mut(symbol)? = count as i16; symbol += 1; - previous0 = count == 0; - if remaining < threshold && remaining > 1 { + if remaining < threshold { + if remaining <= 1 { + break; + } nb_bits = remaining.ilog2() + 1; threshold = 1 << (nb_bits - 1); } + + if count == 0 { + // Each pair of set bits stands for three more symbols of count 0. + let mut repeats = (!bits).trailing_zeros() / 2; + if 2 * repeats + 2 > MIN_LOADED - used { + position += used as usize; + used = 0; + bits = load(header, position)?; + repeats = (!bits).trailing_zeros() / 2; + if 3 * repeats as usize > counts.len() { + return None; + } + } + symbol += 3 * repeats as usize + (bits >> (2 * repeats) & 3) as usize; + bits >>= 2 * repeats + 2; + used += 2 * repeats + 2; + } } if remaining != 1 { return None; @@ -375,3 +381,59 @@ pub(crate) fn huf_read_stats_wksp( table_log, }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::compress::fse::write_ncount; + + #[test] + fn the_quick_way_reads_what_the_careful_way_reads() -> Result<()> { + let mut state = 1u64; + let mut random = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + let nb_headers = if cfg!(miri) { 300 } else { 100_000 }; + let mut nb_quick = 0; + for index in 0..nb_headers { + let max_symbol_value = [35, 52, 31, 255, 7].at(index % 5)?; + let mut header = [0u8; 48]; + for byte in &mut header { + *byte = random() as u8; + } + if index % 2 == 0 { + header[0] = header[0] & 0xF0 | (random() % 8) as u8; + } else { + // A few symbols, far from each other. + let table_log = 5 + random() as u32 % 5; + let mut counts = [0i16; 64]; + let mut left = 1i16 << table_log; + while left > 0 { + let count = 1 + (random() % left as u64) as i16; + *counts.at_mut(random() as usize % (max_symbol_value.min(63) + 1))? += count; + left -= count; + } + let last = counts.iter().rposition(|&count| count != 0).unwrap_or(0); + write_ncount(&mut header, &counts, last as u32, table_log)?; + } + let max_symbol_value = max_symbol_value as u32; + let (mut quick, mut careful) = ([1i16; 256], [1i16; 256]); + let Some(read) = read_valid_ncount(&mut quick, max_symbol_value, &header) else { + continue; + }; + nb_quick += 1; + let expected = read_any_ncount(&mut careful, max_symbol_value, &header)?; + assert_eq!( + (read.size, read.max_symbol_value, read.table_log), + (expected.size, expected.max_symbol_value, expected.table_log) + ); + let nb_symbols = read.max_symbol_value as usize + 1; + assert_eq!(quick.sub(..nb_symbols)?, careful.sub(..nb_symbols)?); + } + assert!(nb_quick > nb_headers / 4); + Ok(()) + } +} From eacaef1d2882aa8cbbb7d2e89131c593fd92b7be Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 13:37:29 +0000 Subject: [PATCH 149/199] zstd: long copies are split where glibc's are slow glibc 2.39 copies more than 2112 bytes at once with rep movsb, which on Zen 4 runs at 3.5 GB/s instead of 57 to 97 if the destination is 1 to 15 bytes further into its page than the source, and at half speed up to about 100 (https://sourceware.org/bugzilla/show_bug.cgi?id=30994). Bun's allocator puts what is 2 KB or more at the start of a page, so literals that are stored went from one such buffer to 12 bytes into another: 128 KB of noise took 87 us at level 1 where the C library, whose literals are somewhere in its workspace, takes 49, and 16 now. write_at, move_within and the new copy and append copy 2 KB at a time in that case, on x86_64 with glibc only. append returns an error instead of allocating. --- src/zstd/compress/mt.rs | 7 +-- src/zstd/compress/stream.rs | 18 +++--- src/zstd/util.rs | 114 ++++++++++++++++++++++++++++++++++-- 3 files changed, 121 insertions(+), 18 deletions(-) diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index c9333c4a63b9..88d55d5e9cb2 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -24,7 +24,7 @@ use super::stream::EndDirective; use crate::common::ZSTD_BLOCKSIZE_MAX; use crate::common::xxhash::Xxh64; use crate::error::{Error, Result}; -use crate::util::{BUG, SliceExt, try_copy}; +use crate::util::{BUG, SliceExt, append, try_copy}; use crate::{InBuffer, OutBuffer}; /// `SerialState`: what has to see the input in order. @@ -93,7 +93,7 @@ impl SerialState { let (over, beyond) = src.split_at(written.len().min(src.len())); written.write_at(0, over)?; (self.round_buff.try_reserve(beyond.len())).map_err(|_| Error::MemoryAllocation)?; - self.round_buff.extend_from_slice(beyond); + append(&mut self.round_buff, beyond)?; (self.ldm_state.window).update( BufId::InBuff.start() + round_pos, BufId::InBuff, @@ -766,8 +766,7 @@ impl MtCtx { let filled = self.in_buff.len() - self.in_buff_prefix_size; let to_load = (input.src.len() - input.pos).min(self.target_section_size - filled); self.reserve_input(to_load)?; - self.in_buff - .extend_from_slice(input.src.span(input.pos, to_load)?); + append(&mut self.in_buff, input.src.span(input.pos, to_load)?)?; input.pos += to_load; forward_input_progress = to_load > 0; } diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs index 3926ab4a877f..60f163342dbd 100644 --- a/src/zstd/compress/stream.rs +++ b/src/zstd/compress/stream.rs @@ -11,7 +11,7 @@ use super::params::{ resolve_enable_ldm, resolve_row_match_finder_mode, }; use crate::error::{Error, Result}; -use crate::util::{SliceExt, try_copy}; +use crate::util::{SliceExt, append, try_copy}; use crate::{InBuffer, OutBuffer}; /// `ZSTD_EndDirective` @@ -64,12 +64,10 @@ impl Default for CCtx { } /// `ZSTD_limitCopy` -fn limit_copy(dst: &mut [u8], src: &[u8]) -> usize { +fn limit_copy(dst: &mut [u8], src: &[u8]) -> Result { let length = dst.len().min(src.len()); - if let (Some(dst), Some(src)) = (dst.get_mut(..length), src.get(..length)) { - dst.copy_from_slice(src); - } - length + dst.write_at(0, src.sub(..length)?)?; + Ok(length) } /// Gives `buffer` room for `new_size` bytes instead of `size`, none of which is touched. @@ -86,13 +84,13 @@ fn resize_buffer(buffer: &mut Vec, size: &mut usize, new_size: usize) -> Res Ok(()) } -/// Writes `src` at `pos`, which is at most the length of `buffer`, over what is there and beyond. +/// Writes `src` at `pos`, which is at most the length of `buffer`, over what is there and beyond, for which it has the +/// capacity. fn write_or_append(buffer: &mut Vec, pos: usize, src: &[u8]) -> Result<()> { let over = buffer.sub_mut(pos..)?; let (first, rest) = src.split_at_checked(over.len()).unwrap_or((src, &[])); over.write_at(0, first)?; - buffer.extend_from_slice(rest); - Ok(()) + append(buffer, rest) } impl CCtx { @@ -388,7 +386,7 @@ impl CCtx { let flushed = limit_copy( output.dst.sub_mut(op..)?, self.out_buff.span(self.out_buff_flushed_size, to_flush)?, - ); + )?; op += flushed; self.out_buff_flushed_size += flushed; if to_flush != flushed { diff --git a/src/zstd/util.rs b/src/zstd/util.rs index c534f1663002..e771fdc92c82 100644 --- a/src/zstd/util.rs +++ b/src/zstd/util.rs @@ -95,7 +95,12 @@ impl SliceExt for [T] { if src_end > self.len() || dst_end > self.len() { return Err(BUG); } - self.copy_within(src..src_end, dst); + let size = size_of::(); + if dst >= src_end && is_slow_to_copy((dst - src) * size, len * size) { + move_in_parts(self, src, dst, len); + } else { + self.copy_within(src..src_end, dst); + } Ok(()) } @@ -104,9 +109,72 @@ impl SliceExt for [T] { where T: Copy, { - self.span_mut(start, from.len())?.copy_from_slice(from); - Ok(()) + copy(self.span_mut(start, from.len())?, from) + } +} + +/// `to.copy_from_slice(from)` +#[inline(always)] +pub(crate) fn copy(to: &mut [T], from: &[T]) -> Result<()> { + if to.len() != from.len() { + return Err(BUG); } + let distance = to.as_ptr().addr().wrapping_sub(from.as_ptr().addr()); + if is_slow_to_copy(distance, size_of_val(from)) { + copy_in_parts(to, from); + } else { + to.copy_from_slice(from); + } + Ok(()) +} + +/// glibc 2.39 copies more than 2112 bytes at once with `rep movsb`, which takes up to 25 times as long on Zen 4 +/// if the destination is a little further into its page than the source: +/// . +const MAX_COPY: usize = 2048; + +/// Whether `size` bytes are, at once, to `distance` bytes after where they are. +#[inline(always)] +fn is_slow_to_copy(distance: usize, size: usize) -> bool { + cfg!(all(target_arch = "x86_64", target_env = "gnu")) + && size > MAX_COPY + && distance.wrapping_sub(1) % 4096 < 127 +} + +#[cold] +#[inline(never)] +fn copy_in_parts(to: &mut [T], from: &[T]) { + let part = (MAX_COPY / size_of::()).max(1); + for (to, from) in to.chunks_mut(part).zip(from.chunks(part)) { + to.copy_from_slice(from); + } +} + +/// `slice.copy_within(src..src + len, dst)`, which is in it and starts no earlier than the source ends. +#[cold] +#[inline(never)] +fn move_in_parts(slice: &mut [T], src: usize, dst: usize, len: usize) { + let part = (MAX_COPY / size_of::()).max(1); + for done in (0..len).step_by(part) { + let end = src + (done + part).min(len); + slice.copy_within(src + done..end, dst + done); + } +} + +/// `to.extend_from_slice(from)`, without a new allocation. +#[inline] +pub(crate) fn append(to: &mut Vec, from: &[u8]) -> Result<()> { + if from.len() > to.capacity() - to.len() { + return Err(BUG); + } + let distance = (to.as_ptr_range().end.addr()).wrapping_sub(from.as_ptr().addr()); + if is_slow_to_copy(distance, from.len()) { + from.chunks(MAX_COPY) + .for_each(|part| to.extend_from_slice(part)); + } else { + to.extend_from_slice(from); + } + Ok(()) } #[inline(always)] @@ -171,6 +239,44 @@ pub(crate) fn try_copy(bytes: &[u8]) -> Result> { let mut vec = Vec::new(); vec.try_reserve_exact(bytes.len()) .map_err(|_| Error::MemoryAllocation)?; - vec.extend_from_slice(bytes); + append(&mut vec, bytes)?; Ok(vec) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn long_copies_are_whole_at_every_distance() -> Result<()> { + const PAGE: usize = 4096; + // Miri takes its time. The distances that are slow come first. + let step = if cfg!(miri) { 61 } else { 1 }; + let from: Vec = (0..2 * MAX_COPY + 77).map(|i| (i % 251) as u8).collect(); + let mut to = vec![0; 3 * PAGE + from.len() + 1]; + // As far into its page as `from`. + let base = PAGE + from.as_ptr().addr() % PAGE - to.as_ptr().addr() % PAGE; + for start in (base..base + PAGE).step_by(step) { + to.write_at(start, &from)?; + assert!(*to.span(start, from.len())? == *from); + assert_eq!((to.at(start - 1)?, to.at(start + from.len())?), (0, 0)); + to.fill(0); + + let mut vec = vec![0; start]; + vec.reserve(from.len()); + assert_eq!(append(&mut vec, to.sub(..from.len() + PAGE)?), Err(BUG)); + append(&mut vec, &from)?; + assert!(*vec.sub(start..)? == *from); + } + let mut within = from.repeat(3); + let len = MAX_COPY + 9; + for distance in (PAGE..PAGE + 200).chain(len..PAGE).step_by(step) { + let expected = within.span(5, len)?.to_vec(); + within.move_within(5, 5 + distance, len)?; + assert!(*within.span(5 + distance, len)? == *expected); + within.clear(); + (0..3).for_each(|_| within.extend_from_slice(&from)); + } + Ok(()) + } +} From 6b2def71240c680ab5764312e1765e92aa4a35ba Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 13:38:06 +0000 Subject: [PATCH 150/199] zstd: what one call decodes is not zeroed before it is written decompress_alloc zeroed all of its output and then decoded into it. For what decodes at the speed of memory that is half of the time: blocks that are stored, blocks of one byte, long matches. The output is now a Vec that grows, as the buffer of a streaming decoder does. Stored blocks, blocks of one byte, and literals and matches of 256 bytes or more that do not fit in what is there are appended. A block of more than 32 sequences, or one that is let to write 4 KB or less, has its room zeroed at once and is decoded as before; others zero ahead when the copies in chunks run out of room. How a buffer grows is an argument of every write, and only output of one call is ever cut back. 1 MB, us, before, now and C: zeros 25, 12, 16; a period of 1 KB 29, 18, 22; a period of 100 KB 37, 26, 26; noise 34, 23, 23. Nothing changes for text or for small frames. Long literals and matches in the common path go through the copy that is split where glibc's is slow: a period of 4100 bytes took 6 times as long as one of 6144. --- scripts/rust-miri.ts | 2 +- src/zstd/decompress/block.rs | 194 ++++++++++++++++++++++++------- src/zstd/decompress/mod.rs | 87 +++++++++----- src/zstd/lib.rs | 209 ++++++++++++++++++++++++++++++---- src/zstd/util.rs | 22 ++++ test/js/bun/util/zstd.test.ts | 108 ++++++++++++++++++ 6 files changed, 534 insertions(+), 88 deletions(-) diff --git a/scripts/rust-miri.ts b/scripts/rust-miri.ts index 1ce61c22fc78..b0939c4d4931 100644 --- a/scripts/rust-miri.ts +++ b/scripts/rust-miri.ts @@ -41,6 +41,7 @@ const MIRI_CRATES = [ "bun_sema", "bun_ast", "bun_paths", + "bun_zstd", "bun_hash", "bun_base64", "bun_bundler", @@ -56,7 +57,6 @@ const MIRI_CRATES = [ "bun_threading", "bun_url", "bun_wyhash", - "bun_zstd", ]; function run(cmd: string, args: string[], opts: Parameters[2] = {}) { diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 6a7844987de0..09cd241e1d1e 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -18,7 +18,7 @@ use crate::common::{ REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; use crate::error::{Error, Result}; -use crate::util::{BUG, SliceExt, read_le16, read_le24, read_le32}; +use crate::util::{BUG, SliceExt, append, append_within, copy, read_le16, read_le24, read_le32}; /// `blockProperties_t` pub(crate) struct BlockProperties { @@ -103,12 +103,25 @@ impl ExtDict<'_> { } } - /// `memmove(&buf[dst], dictEnd - back, len)` + /// `memmove(&out[dst], dictEnd - back, len)` #[inline] - fn copy_to(&self, buf: &mut [u8], dst: usize, back: usize, len: usize) -> Result<()> { + fn copy_to( + &self, + out: &mut OutputBuffer<'_>, + dst: usize, + back: usize, + len: usize, + growth: Growth, + ) -> Result<()> { match *self { - ExtDict::Dict(dict) => buf.write_at(dst, dict.span(dict.len() - back, len)?), - ExtDict::Within { end, .. } => buf.move_within(end - back, dst, len), + ExtDict::Dict(dict) => out.write(dst, dict.span(dict.len() - back, len)?, growth), + ExtDict::Within { end, .. } => { + let Growth::Zeros { limit } = growth else { + return Err(BUG); + }; + out.extend_to(dst + len, limit); + out.bytes().move_within(end - back, dst, len) + } } } } @@ -116,13 +129,14 @@ impl ExtDict<'_> { /// A buffer that output goes to. pub(crate) enum OutputBuffer<'a> { Whole(&'a mut [u8]), - /// What has been written so far of a buffer. It has the capacity for all that a block is let to write. + /// What has been written so far of a buffer, or somewhat more. It has the capacity for all that a block is let to + /// write. Used(&'a mut Vec), } impl OutputBuffer<'_> { #[inline(always)] - fn bytes(&mut self) -> &mut [u8] { + pub(crate) fn bytes(&mut self) -> &mut [u8] { match self { Self::Whole(bytes) => bytes, Self::Used(bytes) => bytes, @@ -130,7 +144,7 @@ impl OutputBuffer<'_> { } #[inline(always)] - fn reborrow(&mut self) -> OutputBuffer<'_> { + pub(crate) fn reborrow(&mut self) -> OutputBuffer<'_> { match self { Self::Whole(bytes) => OutputBuffer::Whole(bytes), Self::Used(bytes) => OutputBuffer::Used(bytes), @@ -146,6 +160,73 @@ impl OutputBuffer<'_> { bytes.resize((end + ahead).min(limit), 0); } } + + /// Where the `len` bytes from `start` on go. + fn room(&mut self, start: usize, len: usize, growth: Growth) -> Result> { + if let Growth::Zeros { limit } = growth { + self.extend_to(start + len, limit); + } + match self { + Self::Used(bytes) if start + len > bytes.len() => { + if start > bytes.len() || !matches!(growth, Growth::Append) { + return Err(BUG); + } + bytes.truncate(start); + Ok(Room::AtEnd(bytes)) + } + _ => Ok(Room::There(self.bytes().span_mut(start, len)?)), + } + } + + /// `memcpy(&self[start], from, from.len())` + pub(crate) fn write(&mut self, start: usize, from: &[u8], growth: Growth) -> Result<()> { + match self.room(start, from.len(), growth)? { + Room::There(room) => room.write_at(0, from), + Room::AtEnd(bytes) => append(bytes, from), + } + } + + /// `memset(&self[start], byte, len)` + pub(crate) fn fill( + &mut self, + start: usize, + byte: u8, + len: usize, + growth: Growth, + ) -> Result<()> { + match self.room(start, len, growth)? { + Room::There(room) => room.fill(byte), + Room::AtEnd(bytes) if len > bytes.capacity() - start => return Err(BUG), + Room::AtEnd(bytes) => bytes.resize(start + len, byte), + } + Ok(()) + } + + /// [`copy_match_exact`] + fn repeat(&mut self, src: usize, dst: usize, len: usize, growth: Growth) -> Result<()> { + if src >= dst { + return Err(BUG); + } + if let Room::AtEnd(bytes) = self.room(dst, len, growth)? { + return in_whole_periods(dst - src, len, |_, chunk| append_within(bytes, src, chunk)); + } + copy_match_exact(self.bytes(), src, dst, len) + } +} + +/// What to do about a buffer that is `Used` only up to before the end of what is to be written. +#[derive(Clone, Copy)] +pub(crate) enum Growth { + /// Zeros up to there, and somewhat further if `limit` allows. + Zeros { limit: usize }, + /// What is written becomes its end, without being zeroed first. Only for output in one call, where nothing that + /// comes after what is being written is of any use. + Append, +} + +enum Room<'a> { + There(&'a mut [u8]), + AtEnd(&'a mut Vec), } /// Where a block is written. @@ -437,7 +518,7 @@ impl BlockDecoder { } self.scratch() .lit_buffer(lit_size)? - .copy_from_slice(src.span(lh_size, lit_size)?); + .write_at(0, src.span(lh_size, lit_size)?)?; return Ok(Literals { c_size: lh_size + lit_size, size: lit_size, @@ -664,23 +745,39 @@ impl BlockDecoder { && literals.location == LitLocation::NotInDst && matches!(ext_dict, ExtDict::Dict(_)); let (out, copy_to) = match buf { - OutputBuffer::Whole(bytes) if is_small => { + OutputBuffer::Used(_) if streaming == Streaming::Yes => (buf, None), + buf if is_small => { let used = capacity + WILDCOPY_OVERLENGTH; *small_output_used = used.max(*small_output_used); ( OutputBuffer::Whole(small_output.sub_mut(..used)?), - Some(bytes.span_mut(start, capacity)?), + Some((buf, start)), ) } OutputBuffer::Whole(bytes) => (OutputBuffer::Whole(bytes.sub_mut(..oend)?), None), - used => (used, None), + // Sequences that are few can be long, and then what they make is best appended. A block whose literals are + // in `src` is let to write all there is room for. A frame before may have been let to write further. + OutputBuffer::Used(bytes) + if nb_seq > MAX_ONE_BY_ONE + && oend - start <= 2 * (block_size_max + WILDCOPY_OVERLENGTH) + || oend - start <= Scratch::MAX_SMALL_OUTPUT + || bytes.len() > oend => + { + if bytes.len() < oend { + bytes.resize(oend, 0); + } + (OutputBuffer::Whole(bytes.sub_mut(..oend)?), None) + } + OutputBuffer::Used(_) => (buf, None), }; + let in_one_call = streaming == Streaming::No && matches!(out, OutputBuffer::Used(_)); let (start, prefix_start, oend) = match copy_to { Some(_) => (0, 0, capacity), None => (start, prefix_start, oend), }; let mut exec = Executor { out, + in_one_call, oend, op: start, prefix_start, @@ -704,8 +801,8 @@ impl BlockDecoder { } exec.last_literals()?; let size = exec.op - start; - if let Some(to) = copy_to { - to.write_at(0, exec.out.bytes().sub(..size)?)?; + if let Some((mut to, start)) = copy_to { + to.write(start, exec.out.bytes().sub(..size)?, Growth::Append)?; } Ok(size) } @@ -1149,15 +1246,23 @@ fn copy_16(buf: &mut [u8], src: usize, dst: usize) -> bool { #[cold] #[inline(never)] fn copy_match_exact(buf: &mut [u8], src: usize, dst: usize, len: usize) -> Result<()> { - let offset = dst - src; - if offset >= len { - return buf.move_within(src, dst, len); - } - // What is copied so far is whole periods, so it can be copied again from the start, twice as much each time. - let mut done = 0usize; + in_whole_periods(dst - src, len, |done, chunk| { + buf.move_within(src, dst + done, chunk) + }) +} + +/// For a match of `len` bytes `offset` bytes back: `copy` gets how much of it is copied and how much to copy next, from +/// where the match starts. What is copied so far is whole periods, so it can be copied again, twice as much each time. +#[inline(always)] +fn in_whole_periods( + offset: usize, + len: usize, + mut copy: impl FnMut(usize, usize) -> Result<()>, +) -> Result<()> { + let mut done = 0; while done < len { let chunk = (offset + done).min(len - done); - buf.move_within(src, dst + done, chunk)?; + copy(done, chunk)?; done += chunk; } Ok(()) @@ -1194,6 +1299,8 @@ fn resolve_offset(sequences: &mut [RawSequence], index: usize) -> u32 { struct Executor<'buf, 'a> { /// What is beyond `oend` is room for copies in chunks. out: OutputBuffer<'buf>, + /// All the output is written in one call, so nothing that comes after `op` is of any use. + in_one_call: bool, /// Where the output of the block has to end. oend: usize, op: usize, @@ -1210,6 +1317,18 @@ struct Executor<'buf, 'a> { } impl<'a> Executor<'_, 'a> { + /// How `out` is to grow for `size` bytes that are written with exact copies: if with zeros, up to `limit` at most. + #[inline(always)] + fn growth(&self, size: usize, limit: usize) -> Growth { + /// Less is likely followed by more of the kind, which is copied in chunks where there is room. + const MIN_SIZE_TO_APPEND: usize = 256; + if self.in_one_call && size >= MIN_SIZE_TO_APPEND { + Growth::Append + } else { + Growth::Zeros { limit } + } + } + /// `litBufferEnd - litPtr` #[inline(always)] fn lits_left(&self) -> usize { @@ -1253,8 +1372,6 @@ impl<'a> Executor<'_, 'a> { if lit_length + match_length > self.oend - op { return Err(Error::DstSizeTooSmall); } - self.out - .extend_to(op + lit_length + match_length, self.oend); if lit_length > self.lits_left() { return Err(Error::CorruptionDetected); } @@ -1265,9 +1382,8 @@ impl<'a> Executor<'_, 'a> { } } - self.out - .bytes() - .write_at(op, self.lits.sub(..lit_length)?)?; + let growth = self.growth(lit_length + match_length, self.oend); + self.out.write(op, self.lits.sub(..lit_length)?, growth)?; self.lits = self.lits.sub(lit_length..)?; op += lit_length; self.op = op + match_length; @@ -1282,15 +1398,16 @@ impl<'a> Executor<'_, 'a> { if back >= match_length { return self .ext_dict - .copy_to(self.out.bytes(), op, back, match_length); + .copy_to(&mut self.out, op, back, match_length, growth); } // The match spans the two segments. - self.ext_dict.copy_to(self.out.bytes(), op, back, back)?; + self.ext_dict + .copy_to(&mut self.out, op, back, back, growth)?; op += back; match_length -= back; match_pos = self.prefix_start; } - copy_match_exact(self.out.bytes(), match_pos, op, match_length) + self.out.repeat(match_pos, op, match_length, growth) } /// `ZSTD_execSequence` and `ZSTD_execSequenceSplitLitBuffer`, if there is room to copy in chunks and the match is @@ -1322,8 +1439,11 @@ impl<'a> Executor<'_, 'a> { *room.first_chunk_mut::<16>()? = *self.lits.first_chunk()?; if lit_length > 16 { - room.get_mut(16..lit_length)? - .copy_from_slice(self.lits.get(16..lit_length)?); + copy( + room.get_mut(16..lit_length)?, + self.lits.get(16..lit_length)?, + ) + .ok()?; } let o_lit_end = op + lit_length; @@ -1348,8 +1468,7 @@ impl<'a> Executor<'_, 'a> { if match_length > 16 { let rounded = (match_length + 15) & !15; if offset >= rounded { - room.get_mut(16..rounded)? - .copy_from_slice(from.get(16..rounded)?); + copy(room.get_mut(16..rounded)?, from.get(16..rounded)?).ok()?; } else { let buf = self.out.bytes(); copy_match_exact(buf, o_lit_end - offset, o_lit_end, match_length).ok()?; @@ -1421,10 +1540,9 @@ impl<'a> Executor<'_, 'a> { if leftover_lit > self.oend - self.op { return Err(Error::DstSizeTooSmall); } - self.out.extend_to(self.op + leftover_lit, self.oend); + let growth = self.growth(leftover_lit, self.oend); self.out - .bytes() - .write_at(self.op, self.lits.sub(..leftover_lit)?)?; + .write(self.op, self.lits.sub(..leftover_lit)?, growth)?; sequence.lit_length -= leftover_lit as u32; self.op += leftover_lit; } @@ -1780,10 +1898,10 @@ impl<'a> Executor<'_, 'a> { if size > self.oend - self.op { return Err(Error::DstSizeTooSmall); } - self.out.extend_to(self.op + size, self.op + size); + let growth = self.growth(size, self.op + size); for part in parts { if !part.is_empty() { - self.out.bytes().write_at(self.op, part)?; + self.out.write(self.op, part, growth)?; self.op += part.len(); } } diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 8c7239c271a0..d12dda6ccfc9 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -8,7 +8,8 @@ mod seq_table; use std::borrow::Cow; use block::{ - BlockDecoder, BlockOutput, EntropyDTables, ExtDict, OutputBuffer, Streaming, get_c_block_size, + BlockDecoder, BlockOutput, EntropyDTables, ExtDict, Growth, OutputBuffer, Streaming, + get_c_block_size, }; use seq_table::{LL_CODES, ML_CODES, OF_CODES, SeqTableSources, TableSource}; @@ -24,7 +25,7 @@ use crate::common::{ ZSTD_WORKSPACETOOLARGE_MAXDURATION, }; use crate::error::{Error, Result}; -use crate::util::{SliceExt, read_le16, read_le32, read_le64, try_copy}; +use crate::util::{SliceExt, append, read_le16, read_le32, read_le64, try_copy}; use crate::{InBuffer, OutBuffer}; /// `ZSTD_FRAMEHEADERSIZE_PREFIX`: what it takes to know the size of a frame header. @@ -623,8 +624,28 @@ impl DCtx { /// `ZSTD_decompressDCtx`: decompresses all the frames of `src`, which has to end with one, to the start of `dst`. /// Returns how many bytes that is. pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + let (size, dst) = (dst.len(), &mut OutputBuffer::Whole(dst)); self.state - .decompress_multi_frame(dst, 0, src, self.ddict.as_deref()) + .decompress_multi_frame(dst, size, 0, src, self.ddict.as_deref()) + } + + /// The same to a new buffer of `capacity` bytes, little more of which is touched than is written. With `ddict`, if + /// there is one, and not with a dictionary that was loaded. + pub(crate) fn decompress_to_vec( + &mut self, + capacity: usize, + src: &[u8], + ddict: Option<&DDict<'_>>, + ) -> Result> { + let mut dst = Vec::new(); + dst.try_reserve_exact(capacity) + .map_err(|_| Error::MemoryAllocation)?; + let used = &mut OutputBuffer::Used(&mut dst); + let size = self + .state + .decompress_multi_frame(used, capacity, 0, src, ddict)?; + dst.truncate(size); + Ok(dst) } /// `ZSTD_decompress_usingDDict` @@ -634,7 +655,9 @@ impl DCtx { src: &[u8], ddict: &DDict<'_>, ) -> Result { - self.state.decompress_multi_frame(dst, 0, src, Some(ddict)) + let (size, dst) = (dst.len(), &mut OutputBuffer::Whole(dst)); + self.state + .decompress_multi_frame(dst, size, 0, src, Some(ddict)) } /// `ZSTD_DCtx_loadDictionary`: `dict` is copied, and used for every frame from now on. None if it is empty. @@ -689,17 +712,12 @@ impl DCtx { } } -/// `ZSTD_decompress` -pub(crate) fn decompress(dst: &mut [u8], src: &[u8]) -> Result { - DCtx::new().decompress(dst, src) -} - /// `ZSTD_copyRawBlock` fn copy_raw_block(dst: &mut [u8], src: &[u8]) -> Result { - let Some(dst) = dst.get_mut(..src.len()) else { + if src.len() > dst.len() { return Err(Error::DstSizeTooSmall); - }; - dst.copy_from_slice(src); + } + dst.write_at(0, src)?; Ok(src.len()) } @@ -822,7 +840,8 @@ impl DState { /// Returns how much was written, and how much was read. fn decompress_frame( &mut self, - dst: &mut [u8], + dst: &mut OutputBuffer<'_>, + dst_size: usize, ostart: usize, src: &[u8], ddict: Option<&DDict<'_>>, @@ -852,11 +871,10 @@ impl DState { let decoded_size = match properties.block_type { BlockType::Compressed => { - let capacity = dst.len() - op; let decoded_size = self.decompress_block( - OutputBuffer::Whole(dst), + dst.reborrow(), op, - capacity, + dst_size - op, block, ddict, Streaming::No, @@ -867,16 +885,25 @@ impl DState { } decoded_size } - BlockType::Raw => copy_raw_block(dst.sub_mut(op..)?, block)?, - BlockType::Rle => set_rle_block( - dst.sub_mut(op..)?, - block.at(0)?, - properties.orig_size as usize, - )?, + BlockType::Raw => { + if block.len() > dst_size - op { + return Err(Error::DstSizeTooSmall); + } + dst.write(op, block, Growth::Append)?; + block.len() + } + BlockType::Rle => { + let size = properties.orig_size as usize; + if size > dst_size - op { + return Err(Error::DstSizeTooSmall); + } + dst.fill(op, block.at(0)?, size, Growth::Append)?; + size + } BlockType::Reserved => return Err(Error::CorruptionDetected), }; if self.validate_checksum { - self.xxh_state.update(dst.span(op, decoded_size)?); + self.xxh_state.update(dst.bytes().span(op, decoded_size)?); } op += decoded_size; ip += c_block_size; @@ -905,7 +932,8 @@ impl DState { /// `ZSTD_decompressMultiFrame`: all the frames of `src`, to `dst` from `dst_start` on. fn decompress_multi_frame( &mut self, - dst: &mut [u8], + dst: &mut OutputBuffer<'_>, + dst_size: usize, dst_start: usize, mut src: &[u8], ddict: Option<&DDict<'_>>, @@ -920,9 +948,9 @@ impl DState { } self.begin_using_ddict(ddict); - self.check_continuity(op, dst.len() - op); + self.check_continuity(op, dst_size - op); - match self.decompress_frame(dst, op, src, ddict) { + match self.decompress_frame(dst, dst_size, op, src, ddict) { // More likely `src` is longer than its frames than damaged where a magic number is expected. Err(Error::PrefixUnknown) if more_than_1_frame => { return Err(Error::SrcSizeWrong); @@ -1202,7 +1230,8 @@ impl DState { { if let Ok(c_size) = find_frame_compressed_size(src.sub(istart..)?) { let decompressed_size = self.decompress_multi_frame( - output.dst, + &mut OutputBuffer::Whole(output.dst), + oend, op, src.span(istart, c_size)?, ddict, @@ -1267,9 +1296,7 @@ impl DState { return Err(Error::CorruptionDetected); } self.in_buff.bytes.truncate(self.in_pos); - self.in_buff - .bytes - .extend_from_slice(src.span(ip, loaded_size)?); + append(&mut self.in_buff.bytes, src.span(ip, loaded_size)?)?; } ip += loaded_size; self.in_pos += loaded_size; diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 01b403873528..f4716d6bba13 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -25,10 +25,10 @@ pub use compress::internal::Strategy; pub use compress::params::{ZSTD_CLEVEL_DEFAULT, c_param, compress_bound}; use compress::stream::compress_to; pub use compress::stream::{CCtx, EndDirective, compress}; +use decompress::find_decompressed_size; pub use decompress::{ DCtx, DDict, FrameExtent, ZSTD_D_WINDOW_LOG_MAX, frame_extent, get_frame_content_size, }; -use decompress::{decompress, find_decompressed_size}; pub use error::{Error, Result, ZstdError}; // Callers share contexts between threads behind their own `unsafe impl`s, which should have nothing to vouch for here. @@ -142,15 +142,14 @@ pub fn compress_alloc(src: &[u8], level: i32) -> Result> { /// With that much room after its output, a decoder copies in chunks up to the end. const OUTPUT_SLACK: usize = common::WILDCOPY_OVERLENGTH; -/// Zeros for a decoder to write `size` bytes to. -fn output_buffer(size: usize) -> Result, ZstdError> { +/// How much room to give a decoder that is to write `size` bytes. +fn output_capacity(size: usize) -> usize { // Not where it would be most of the buffer: to give that back can take a copy. - let slack = if size >= 2 * OUTPUT_SLACK { - OUTPUT_SLACK + if size >= 2 * OUTPUT_SLACK { + size + OUTPUT_SLACK } else { - 0 - }; - util::try_zeroed(size + slack).map_err(|_| ZstdError::OutOfMemory) + size + } } /// Decompress data, automatically allocating the output buffer. @@ -185,11 +184,9 @@ pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { } // Fast path: size is known and within reasonable limits - let mut output = output_buffer(size as usize)?; - let written = decompress(&mut output, src) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) + DCtx::new() + .decompress_to_vec(output_capacity(size as usize), src, None) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed)) } /// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is @@ -201,12 +198,10 @@ pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result if size > MAX_SIZE { return Err(ZstdError::InvalidZstdData); } - let mut output = output_buffer(size as usize)?; - let written = DDict::prefix(prefix) - .and_then(|prefix| DCtx::new().decompress_using_ddict(&mut output, src, &prefix)) - .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; - output.truncate(written); - Ok(output) + let capacity = output_capacity(size as usize); + DDict::prefix(prefix) + .and_then(|prefix| DCtx::new().decompress_to_vec(capacity, src, Some(&prefix))) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed)) } #[derive(Clone, Copy, PartialEq, Eq)] @@ -500,4 +495,180 @@ mod tests { } Ok(()) } + + /// A frame with a block, or more, for each of `pieces`. + fn frame_of(cctx: &mut CCtx, pieces: &[&[u8]]) -> Result> { + let mut frame = vec![0; compress_bound(pieces.concat().len()) + 16 * pieces.len()]; + let mut output = OutBuffer { + dst: &mut frame, + pos: 0, + }; + for (index, src) in pieces.iter().enumerate() { + let end = if index + 1 == pieces.len() { + EndDirective::End + } else { + EndDirective::Flush + }; + assert_eq!( + cctx.compress_stream2(&mut output, &mut InBuffer { src, pos: 0 }, end), + Ok(0) + ); + } + let size = output.pos; + frame.truncate(size); + Ok(frame) + } + + #[test] + fn long_matches_in_a_dictionary_are_appended() -> Result<()> { + let mut state = 7u32; + let mut noise = move |len: usize| -> Vec { + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + (state >> 8) as u8 + }) + .collect() + }; + // Miri takes long to load a dictionary. A block that is let to write 4 KB or less has that zeroed at once. + let unit = if cfg!(miri) { 1000 } else { 4000 }; + let dict = noise(3 * unit); + // A match in the dictionary, one from there into the output, and one in the output. + let first = [&noise(20), dict.sub(unit / 2..unit / 2 + unit)?].concat(); + let pieces = [ + &first, + &[&noise(20), dict.sub(100..100 + unit)?].concat(), + &[ + &noise(20), + dict.sub(3 * unit - unit / 2..)?, + first.sub(..unit / 2)?, + ] + .concat(), + &[&noise(20), first.sub(100..unit)?, &noise(30)].concat(), + ]; + let src = pieces.map(|piece| &**piece).concat(); + let mut cctx = CCtx::new(); + cctx.load_dictionary(&dict)?; + let frame = frame_of(&mut cctx, &pieces.map(|piece| &**piece))?; + assert!(frame.len() < 200); + + let ddict = DDict::by_reference(&dict)?; + let step = if cfg!(miri) { 997 } else { 1 }; + for capacity in + (0..src.len()) + .step_by(step) + .chain([src.len(), src.len() + 32, src.len() + 8192]) + { + let mut there = vec![0; capacity]; + let written = DCtx::new().decompress_using_ddict(&mut there, &frame, &ddict); + let appended = DCtx::new().decompress_to_vec(capacity, &frame, Some(&ddict)); + assert_eq!(appended.as_ref().map(Vec::len), written.as_ref().copied()); + assert_eq!( + appended.is_ok_and(|appended| appended == src), + capacity >= src.len() + ); + } + Ok(()) + } + + #[test] + fn output_that_is_appended_is_what_output_that_is_there_is() -> Result<()> { + let mut state = 1u32; + let mut random = move || { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state >> 8 + }; + let block = + |last: u32, kind: u32, size: u32| ((size << 3 | kind << 1 | last).to_le_bytes(), 3); + // Blocks that are stored and blocks of one byte, with a window of 1 KB. + let mut stored = vec![0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00]; + for (last, kind, size, content) in + [(0, 0, 300, 300), (0, 1, 500, 1), (0, 0, 0, 0), (1, 0, 9, 9)] + { + let (header, len) = block(last, kind, size); + stored.extend_from_slice(header.sub(..len)?); + stored.extend((0..content).map(|_| random() as u8)); + } + let frame = |src: &[u8]| compress_alloc(src, 3); + // Miri takes long for literals and sequences, and no time for long matches. A block that is let to write 4 KB or + // less has that zeroed at once. + let scale = if cfg!(miri) { 5 } else { 1 }; + let period: Vec = (0..700 / scale).map(|_| random() as u8).collect(); + let mut periods: Vec = period.iter().cycle().take(9000).copied().collect(); + periods.extend((0..40).map(|_| random() as u8)); + let few_sequences = frame(&periods)?; + // The same in blocks of 1 KB, which go on with the offsets of the blocks before. + let mut cctx = CCtx::new(); + cctx.set_parameter(c_param::WINDOW_LOG, 10)?; + let small_blocks = frame_of(&mut cctx, &[&periods])?; + let run = frame(&[b'r'; 9000])?; + let only_literals = frame( + &(0..4300) + .map(|_| (random() % 16 * (random() % 3)) as u8) + .collect::>(), + )?; + let words = [&b"one "[..], b"three ", b"four ", b"seven ", b"eleven "]; + let many_sequences = frame( + &(0..900 / scale) + .flat_map(|_| *words.get(random() as usize % 5).unwrap_or(&&b""[..])) + .copied() + .collect::>(), + )?; + + let same = |frames: &[u8], capacity: usize| { + let mut there = vec![0; capacity]; + let written = DCtx::new().decompress(&mut there, frames); + let appended = DCtx::new().decompress_to_vec(capacity, frames, None); + assert_eq!( + appended.as_ref().map(Vec::len), + written.as_ref().copied(), + "{capacity}" + ); + assert!( + appended.is_ok_and(|appended| Some(&*appended) == there.get(..appended.len())) + || written.is_err() + ); + written + }; + let orders = [ + [ + &many_sequences, + &stored, + &few_sequences, + &run, + &small_blocks, + &only_literals, + ], + [ + &only_literals, + &small_blocks, + &run, + &stored, + &many_sequences, + &few_sequences, + ], + ]; + for order in orders.iter().take(if cfg!(miri) { 1 } else { 2 }) { + let frames = order.map(|frame| &**frame).concat(); + let size = same(&frames, 1 << 16)?; + let (step, around) = if cfg!(miri) { (size, 1) } else { (7, 40) }; + for capacity in (0..size).step_by(step).chain(size - around..size + around) { + assert_eq!(same(&frames, capacity).is_ok(), capacity >= size); + } + let step = if cfg!(miri) { 8 * frames.len() / 3 } else { 1 }; + for bit in (0..8 * frames.len()).skip(step / 2).step_by(step) { + let mut damaged = frames.clone(); + if let Some(byte) = damaged.get_mut(bit / 8) { + *byte ^= 1 << (bit % 8); + } + let _ = same(&damaged, size + OUTPUT_SLACK); + let _ = same(&damaged, size + 8192); + } + } + Ok(()) + } } diff --git a/src/zstd/util.rs b/src/zstd/util.rs index e771fdc92c82..19037519dc8d 100644 --- a/src/zstd/util.rs +++ b/src/zstd/util.rs @@ -177,6 +177,23 @@ pub(crate) fn append(to: &mut Vec, from: &[u8]) -> Result<()> { Ok(()) } +/// `to.extend_from_within(start..start + len)`, without a new allocation. +#[inline] +pub(crate) fn append_within(to: &mut Vec, start: usize, len: usize) -> Result<()> { + let end = start + len; + if end > to.len() || len > to.capacity() - to.len() { + return Err(BUG); + } + if is_slow_to_copy(to.len() - start, len) { + (start..end) + .step_by(MAX_COPY) + .for_each(|part| to.extend_from_within(part..(part + MAX_COPY).min(end))); + } else { + to.extend_from_within(start..end); + } + Ok(()) +} + #[inline(always)] pub(crate) fn read_le16(data: &[u8], pos: usize) -> Result { Ok(u16::from_le_bytes(*data.array_at(pos)?)) @@ -271,6 +288,11 @@ mod tests { let mut within = from.repeat(3); let len = MAX_COPY + 9; for distance in (PAGE..PAGE + 200).chain(len..PAGE).step_by(step) { + within.truncate(3 * from.len()); + within.reserve(len); + append_within(&mut within, 3 * from.len() - distance, len)?; + assert!(within.span(3 * from.len() - distance, len)? == within.sub(3 * from.len()..)?); + let expected = within.span(5, len)?.to_vec(); within.move_within(5, 5 + distance, len)?; assert!(*within.span(5 + distance, len)? == *expected); diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 5b589cc6d0c4..82d85bdbec5c 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -343,6 +343,114 @@ describe("Zstandard compression", async () => { } }); +describe("decompressing frames whose size is known", () => { + let x = 0x9e3779b9; + const noise = (length: number) => + Buffer.from( + Uint8Array.from({ length }, () => { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + return x; + }), + ); + const repeated = (period: Buffer, length: number) => Buffer.alloc(length, period); + // Blocks that are stored, blocks of one byte, blocks of a few long matches, and blocks of many short ones. + const pieces = [ + noise(140_000), + Buffer.alloc(270_000, 7), + repeated(noise(1000), 140_000), + repeated(Buffer.from("one two three two one three "), 20_000), + repeated(noise(60_000), 200_000), + Buffer.from(noise(30_000).map(byte => 97 + (byte & 3) * ((byte >> 2) % 3))), + noise(9), + ]; + const whole = Buffer.concat(pieces); + + // "raw", "rle", or how many sequences a compressed block has. + function blocks(frame: Uint8Array) { + const descriptor = frame[4]; + const isSingleSegment = (descriptor >> 5) & 1; + let at = 5 + (1 - isSingleSegment) + [0, 1, 2, 4][descriptor & 3] + [isSingleSegment, 2, 4, 8][descriptor >> 6]; + const kinds: (string | number)[] = []; + for (let isLast = 0; !isLast;) { + const header = frame[at] | (frame[at + 1] << 8) | (frame[at + 2] << 16); + at += 3; + isLast = header & 1; + const type = (header >> 1) & 3; + if (type === 2) { + const format = (frame[at] >> 2) & 3; + const bits = + frame[at] + + frame[at + 1] * 2 ** 8 + + frame[at + 2] * 2 ** 16 + + frame[at + 3] * 2 ** 24 + + frame[at + 4] * 2 ** 32; + let literals: number; + if ((frame[at] & 3) < 2) { + const size = format === 1 ? (bits >> 4) & 0xfff : format === 3 ? (bits >> 4) & 0xfffff : frame[at] >> 3; + literals = [1, 2, 1, 3][format] + ((frame[at] & 3) === 0 ? size : 1); + } else { + const width = [10, 10, 14, 18][format]; + literals = [3, 3, 4, 5][format] + (Math.floor(bits / 2 ** (4 + width)) % 2 ** width); + } + const [first, second, third] = frame.subarray(at + literals); + kinds.push(first < 128 ? first : first < 255 ? ((first - 128) << 8) + second : second + (third << 8) + 0x7f00); + } else { + kinds.push(["raw", "rle"][type]); + } + at += type === 1 ? 1 : header >> 3; + } + return kinds; + } + + it("the frames have blocks of these kinds", () => { + expect(pieces.map(piece => blocks(zstdCompressSync(piece, { level: 1 })))).toEqual([ + ["raw", "raw"], + [1, "rle", "rle"], + [1, 1], + [4], + [1, 1], + [3647], + ["raw"], + ]); + }); + + it.each([1, 3])("stretches of noise, runs and long matches in one frame of level %i", level => { + expect(zstdDecompressSync(zstdCompressSync(whole, { level })).equals(whole)).toBe(true); + }); + + it("the same in a frame each, whose blocks are let to write more or less", () => { + const frames = pieces.map((piece, i) => + zlib.zstdCompressSync(piece, { params: { [zlib.constants.ZSTD_c_windowLog]: [17, 10, 20, 12][i % 4] } }), + ); + expect(zstdDecompressSync(Buffer.concat(frames)).equals(whole)).toBe(true); + expect(zstdDecompressSync(Buffer.concat(frames.toReversed())).equals(Buffer.concat(pieces.toReversed()))).toBe( + true, + ); + }); + + it.each([4097, 4100, 4222, 8200])("long matches from %i bytes back, which are copied in parts", period => { + const data = repeated(noise(period), 40_000); + const frame = zstdCompressSync(data, { level: 3 }); + expect(zstdDecompressSync(frame).equals(data)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + }); + + it("a frame that makes more than it says is refused", () => { + for (const piece of pieces.filter(piece => piece.length >= 135_000)) { + const frame = zstdCompressSync(piece.subarray(0, 135_000), { level: 1 }); + // The content size, 4 bytes of it from 64 KB on, after the magic number and the descriptor. + expect(frame.readUInt32LE(5)).toBe(135_000); + for (const size of [134_999, 131_072, 70_000]) { + const lying = Buffer.from(frame); + lying.writeUInt32LE(size, 5); + expect(() => zstdDecompressSync(lying)).toThrow("Decompression failed: DecompressionFailed"); + } + } + }); +}); + // zstdDecompressSync sizes its output from the frame header when the content size is // present and at most 16 MiB. Anything else is decoded in a stream into a buffer that // starts at a guess (the input size, or the 16 MiB limit) and grows as needed. From 5a031e0a971d88da1f07fb333da835d7c4f8822e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 13:38:06 +0000 Subject: [PATCH 151/199] ci: skip the bun_sema spelling test under Miri The change of #47181 to this file: the test takes Miri hours, so the job never finishes. It drops out once main has it. --- src/sema/check/regexp_scanner.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sema/check/regexp_scanner.rs b/src/sema/check/regexp_scanner.rs index 8ca63a1ce1bd..ac14fdcaa95f 100644 --- a/src/sema/check/regexp_scanner.rs +++ b/src/sema/check/regexp_scanner.rs @@ -1826,7 +1826,9 @@ mod spelling_tests { names } + /// Skipped under Miri: the 380,880 comparisons run no unsafe code, and Miri needs hours for them. #[test] + #[cfg_attr(miri, ignore)] fn characters_refuse_only_names_that_are_too_far_apart() { let names = names(); let mut accepted = 0; From e4d6720f510e4b6759e42ca808799acf1105c23b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 17:31:47 +0000 Subject: [PATCH 152/199] zstd: decode long Huffman codes one symbol to a lookup A table whose entries hold up to three symbols pays off when codes are short. When few lookups would yield more than one symbol, a table with one symbol to an entry is faster: the loop reads the count of bits and the symbol with one load of two bytes and writes one byte. That form was only used for small blocks with codes of up to 9 bits. It now goes up to 11 bits, and from 2048 literals on it is chosen by how many symbols a lookup yields on average, which is worked out once per code. The other loop reads the count of bits with a load of its own, which is off the path from one lookup to the next. 1 MB in the harness, MB/s, before, after, and the C library: bytes that are the least of four 1160, 1411, 1478; base64 2100, 2144, 1421; text 1597, 1607, 1368; integers 1312, 1355, 1510. --- src/zstd/decompress/block.rs | 6 +- src/zstd/decompress/huf.rs | 293 +++++++++++++++++++++++++++++----- test/js/bun/util/zstd.test.ts | 41 ++++- 3 files changed, 295 insertions(+), 45 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 09cd241e1d1e..29cb6c397c98 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -455,7 +455,11 @@ impl BlockDecoder { let table = match (huf_source, dict_entropy) { (TableSource::Dict, Some(dict)) => &dict.huf_table, _ => { - entropy.huf_table.prepare(lit_size)?; + entropy.huf_table.prepare( + lit_size, + compressed.len(), + !single_stream, + )?; &entropy.huf_table } }; diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index 3516178b201c..f17f549463c6 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -33,13 +33,17 @@ const FAST_LOG: u32 = 11; const FAST_MIN_LITERALS: usize = 2048; /// From how many literals on the larger table is considered. const LARGE_MIN_LITERALS: usize = 1 << 14; -/// From how many bits to a literal on few lookups of at most `SINGLE_MAX_LOG` bits would yield more than one. -const SINGLE_MIN_BITS: usize = 5; -const SINGLE_MAX_LOG: u32 = 9; +/// The longest code of a table that yields one symbol to a lookup. +const SINGLE_MAX_LOG: u32 = 11; +/// Few literals are not worth a closer look at their code: from how many bits to a literal on few lookups of at most +/// `SINGLE_SMALL_LOG` bits would yield more than one. +const SINGLE_SMALL_MIN_BITS: usize = 5; +const SINGLE_SMALL_LOG: u32 = 9; impl Code { /// How many symbols a lookup of `log` bits yields, in 65536ths, if symbols are as frequent as their codes are short. - fn symbols_per_lookup(&self, log: u32) -> u64 { + /// Without `thirds` a third symbol does not count, which takes less time. + fn symbols_per_lookup(&self, log: u32, thirds: bool) -> u64 { let table_log = self.table_log as usize; // For each number of bits, out of `1 << table_log`, how many symbols have a code no longer than that. let mut one = [0u64; NB_RANKS + 1]; @@ -67,6 +71,9 @@ impl Code { }) .sum() }; + if !thirds { + return (1 << 16) + ((after(&one, log as usize) << 16) >> (2 * table_log)); + } let mut two = [0u64; NB_RANKS + 1]; for (room, two) in two.iter_mut().enumerate() { *two = after(&one, room); @@ -84,21 +91,34 @@ impl Code { // The larger table takes twice the time to fill, half of the cache, and a refill for 4 lookups and not 5. if self.table_log > FAST_LOG || nb_literals >= LARGE_MIN_LITERALS - && self.symbols_per_lookup(FAST_LOG + 1) * 4 > self.symbols_per_lookup(FAST_LOG) * 5 + && self.symbols_per_lookup(FAST_LOG + 1, true) * 4 + > self.symbols_per_lookup(FAST_LOG, true) * 5 { return FAST_LOG + 1; } FAST_LOG } - /// Whether four streams of `nb_bytes` in all are decoded sooner to `nb_literals` one to a lookup of `log` bits. - fn is_best_single(log: u32, nb_literals: usize, nb_bytes: usize) -> bool { - log <= SINGLE_MAX_LOG && 8 * nb_bytes >= SINGLE_MIN_BITS * nb_literals + /// How many bits to look up at once to decode `nb_literals` from four streams of `nb_bytes` in all, and whether + /// that is sooner done one to a lookup. + fn best_form(&self, nb_literals: usize, nb_bytes: usize) -> (u32, bool) { + let log = self.best_log(nb_literals); + let single = if nb_literals < FAST_MIN_LITERALS { + log <= SINGLE_SMALL_LOG && 8 * nb_bytes >= SINGLE_SMALL_MIN_BITS * nb_literals + } else { + // A lookup that can yield several symbols takes a quarter more time. + self.table_log <= SINGLE_MAX_LOG && self.symbols_per_lookup(log, false) * 4 < 5 << 16 + }; + if single { + (self.table_log.max(MIN_LOG), true) + } else { + (log, false) + } } } /// What a lookup yields. From the lowest byte on: up to 3 symbols, then how many bits they take in 6 bits and how many -/// they are above. +/// they are above. What is after the last symbol is not always 0: see [`single_entry`]. type Entry = u32; const COUNT_SHIFT: u32 = 6; @@ -108,23 +128,30 @@ const fn entry(symbol: u8, index: u32, nb_bits: u32) -> Entry { ((symbol as u32) << (8 * index)) | ((nb_bits + (1 << COUNT_SHIFT)) << 24) } -/// For each of `symbols`, `1 << run_log` entries that are `base` and the symbol at `index`. -fn fill_runs(out: &mut [Entry], symbols: &[u8], run_log: u32, base: Entry, index: u32) { +/// An entry of one symbol that also has its highest byte and then the symbol in the middle, where there is no symbol: +/// one read yields both, the former where a shift takes its count from. `Streams::decode_most_single` reads only that. +const fn single_entry(symbol: u8, nb_bits: u32) -> Entry { + let entry = entry(symbol, 0, nb_bits); + entry | (entry >> 16) | ((symbol as u32) << 16) +} + +/// For each of `symbols`, `1 << run_log` entries that are `base` and the symbol times `unit`. +fn fill_runs(out: &mut [Entry], symbols: &[u8], run_log: u32, base: Entry, unit: Entry) { #[inline(always)] - fn runs(out: &mut [Entry], symbols: &[u8], base: Entry, index: u32) { + fn runs(out: &mut [Entry], symbols: &[u8], base: Entry, unit: Entry) { let (out, _) = out.as_chunks_mut::(); for (run, &symbol) in out.iter_mut().zip(symbols) { - *run = [base + (u32::from(symbol) << (8 * index)); RUN]; + *run = [base + u32::from(symbol) * unit; RUN]; } } match run_log { - 0 => runs::<1>(out, symbols, base, index), - 1 => runs::<2>(out, symbols, base, index), - 2 => runs::<4>(out, symbols, base, index), - 3 => runs::<8>(out, symbols, base, index), + 0 => runs::<1>(out, symbols, base, unit), + 1 => runs::<2>(out, symbols, base, unit), + 2 => runs::<4>(out, symbols, base, unit), + 3 => runs::<8>(out, symbols, base, unit), _ => { for (run, &symbol) in out.chunks_exact_mut(1 << run_log).zip(symbols) { - run.fill(base + (u32::from(symbol) << (8 * index))); + run.fill(base + u32::from(symbol) * unit); } } } @@ -256,7 +283,7 @@ impl Sorted { } } } - _ => fill_runs(out, symbols, room_after, base, index), + _ => fill_runs(out, symbols, room_after, base, 1 << (8 * index)), } } Ok(()) @@ -271,8 +298,10 @@ pub(crate) struct HufDTable { table_type: TableType, /// How many bits `table` is indexed with. log: u32, - /// Every entry of `table` is one symbol. + /// Every entry of `table` is a [`single_entry`]. single: bool, + /// [`Code::best_form`] for `FAST_MIN_LITERALS` and for `LARGE_MIN_LITERALS`, once it is asked for. + best_forms: [Option<(u32, bool)>; 2], table: [Entry; TABLE_SIZE], } @@ -288,6 +317,7 @@ impl HufDTable { table_type: TableType::X1, log: 0, single: false, + best_forms: [None; 2], table: [0; TABLE_SIZE], } } @@ -311,19 +341,53 @@ impl HufDTable { self.code.nb_symbols = stats.nb_symbols; self.code.table_log = stats.table_log; self.table_type = table_type; - let log = self.code.best_log(nb_literals); + self.best_forms = [None; 2]; let nb_bytes = src.len().saturating_sub(stats.size); - let single = four_streams && Code::is_best_single(log, nb_literals, nb_bytes); + let (log, single) = self.best_form(nb_literals, nb_bytes, four_streams); self.build(log, single)?; Ok(stats.size) } - /// Makes the table fit to decode `nb_literals` of another block with. - pub(crate) fn prepare(&mut self, nb_literals: usize) -> Result<()> { - if self.log < FAST_LOG && nb_literals >= FAST_MIN_LITERALS { - self.build(self.code.table_log.max(FAST_LOG), false)?; - } else if self.single { - self.build(self.log, false)?; + /// [`Code::best_form`], which is for four streams. + fn best_form( + &mut self, + nb_literals: usize, + nb_bytes: usize, + four_streams: bool, + ) -> (u32, bool) { + if !four_streams { + return (self.code.best_log(nb_literals), false); + } + if nb_literals < FAST_MIN_LITERALS { + return self.code.best_form(nb_literals, nb_bytes); + } + let [fast, large] = &mut self.best_forms; + let best_form = if nb_literals < LARGE_MIN_LITERALS { + fast + } else { + large + }; + *best_form.get_or_insert_with(|| self.code.best_form(nb_literals, nb_bytes)) + } + + /// Makes the table fit to decode `nb_literals` of another block with, from `nb_bytes`. + pub(crate) fn prepare( + &mut self, + nb_literals: usize, + nb_bytes: usize, + four_streams: bool, + ) -> Result<()> { + let (log, single) = self.best_form(nb_literals, nb_bytes, four_streams); + let is_worth_it = match (self.single, single) { + (true, true) => false, + // Few literals do not make up for the time that it takes to fill a large table once more. + (true, false) => nb_literals >= FAST_MIN_LITERALS || self.log <= SINGLE_SMALL_LOG, + // The largest table is only made if its lookups yield enough. + (false, true) => nb_literals >= FAST_MIN_LITERALS && self.log <= FAST_LOG, + (false, false) => self.log < log, + }; + if is_worth_it { + self.build(log, single)?; } Ok(()) } @@ -335,7 +399,9 @@ impl HufDTable { let sorted = Sorted::new(&self.code, log)?; // The tables of what a third symbol adds end with the one for this many bits. let room = log.saturating_sub(2 * sorted.shortest()); - if single || room < sorted.shortest() { + if single { + self.fill_single(&sorted) + } else if room < sorted.shortest() { self.fill(&sorted, &mut []) } else if room < 7 { self.fill(&sorted, &mut [0; 1 << 7]) @@ -344,10 +410,26 @@ impl HufDTable { } } + fn fill_single(&mut self, sorted: &Sorted) -> Result<()> { + let log = sorted.log; + for weight in 1..=sorted.max_weight { + let symbols = sorted.class(weight)?; + let nb_bits = sorted.nb_bits(weight); + let room = log - nb_bits; + let entries = self + .table + .span_mut(sorted.position(weight, log)?, symbols.len() << room)?; + let base = single_entry(0, nb_bits); + let unit = single_entry(1, nb_bits) - base; + fill_runs(entries, symbols, room, base, unit); + } + Ok(()) + } + fn fill(&mut self, sorted: &Sorted, thirds: &mut [Entry]) -> Result<()> { let log = sorted.log; // From how many bits on what a symbol leaves of a lookup holds another. - let min_room = if self.single { log } else { sorted.shortest() }; + let min_room = sorted.shortest(); for room in min_room..=log.saturating_sub(2 * min_room) { sorted.fill(thirds.span_mut(1 << room, 1 << room)?, room, 0, 2, None)?; } @@ -359,7 +441,7 @@ impl HufDTable { .table .span_mut(sorted.position(weight, log)?, symbols.len() << room)?; if room < min_room { - fill_runs(entries, symbols, room, entry(0, 0, nb_bits), 0); + fill_runs(entries, symbols, room, entry(0, 0, nb_bits), 1); continue; } let Some((&first_symbol, other_symbols)) = symbols.split_first() else { @@ -430,7 +512,9 @@ impl HufDTable { fn decode_most_single(&self, streams: &mut Streams<'_>) { match self.log { 8 => streams.decode_most_single::<8, 7>(&self.table), - _ => streams.decode_most_single::(&self.table), + 9 => streams.decode_most_single::<9, 6>(&self.table), + 10 => streams.decode_most_single::<10, 5>(&self.table), + _ => streams.decode_most_single::(&self.table), } } @@ -807,8 +891,15 @@ impl<'a> Streams<'a> { window: &mut [u8; WINDOW], done: &mut usize, ) { - if let Some(entry) = table.get((*bits >> (64 - LOG)) as usize) { - *bits <<= (entry >> 24) & 63; + let index = (*bits >> (64 - LOG)) as usize; + // The highest byte of the entry, read on its own where that is simple: the next lookup waits for it. + let nb_bits = if cfg!(target_endian = "little") { + bun_pod::bytes_of(table).get(4 * index + 3).copied() + } else { + table.get(index).map(|entry| (entry >> 24) as u8) + }; + if let (Some(entry), Some(nb_bits)) = (table.get(index), nb_bits) { + *bits <<= nb_bits & 63; if let Some(to) = window.get_mut(*done..*done + 4) { to.copy_from_slice(&entry.to_le_bytes()); } @@ -913,15 +1004,16 @@ impl<'a> Streams<'a> { to: Option<&mut u8>, ) { let index = (*bits >> (64 - LOG)) as usize; - // The highest byte of the entry, read on its own where that is simple: the next lookup waits for it. - let nb_bits = if cfg!(target_endian = "little") { - bun_pod::bytes_of(table).get(4 * index + 3).copied() + let middle = if cfg!(target_endian = "little") { + (bun_pod::bytes_of(table).get(4 * index + 1..)) + .and_then(|bytes| bytes.first_chunk()) + .map(|bytes| u16::from_le_bytes(*bytes)) } else { - table.get(index).map(|entry| (entry >> 24) as u8) + table.get(index).map(|entry| (entry >> 8) as u16) }; - if let (Some(to), Some(nb_bits), Some(&entry)) = (to, nb_bits, table.get(index)) { - *bits <<= nb_bits & 63; - *to = entry as u8; + if let (Some(to), Some(middle)) = (to, middle) { + *bits <<= middle & 63; + *to = (middle >> 8) as u8; } } @@ -1045,3 +1137,122 @@ pub(crate) fn decompress_4x_huf_only( } decompress_4x_using_dtable(dst, src.sub(h_size..)?, dtable) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::compress::huf::{ + HufCTable, build_ctable, compress_1x_using_ctable, compress_4x_using_ctable, write_ctable, + }; + + #[test] + fn a_table_is_made_to_suit_each_block() -> Result<()> { + let mut state = 5u32; + let mut random = move || { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state >> 8 + }; + // Few enough for codes of 8 bits at most that are not all as long. The lowest have the shortest codes. + const MAX_SYMBOL: u32 = 199; + let counts: [u32; MAX_SYMBOL as usize + 1] = core::array::from_fn(|symbol| { + let rank = MAX_SYMBOL + 1 - symbol as u32; + rank * rank * rank / 256 + 1 + }); + // How many literals, by how much they are lowered, and whether they are in four streams. + let blocks = [ + (300, 0, true), + (300, 3, true), + (FAST_MIN_LITERALS, 0, true), + (300, 3, true), + (FAST_MIN_LITERALS, 0, false), + (300, 0, false), + (LARGE_MIN_LITERALS, 3, true), + (200, 3, false), + (LARGE_MIN_LITERALS, 0, true), + ]; + // Miri takes a minute for 10,000 literals. + let (blocks, longest_codes, firsts, max_damaged) = if cfg!(miri) { + (blocks.sub(..5)?, 11..=11, [0].as_slice(), 300) + } else { + ( + blocks.sub(..)?, + 8..=12, + [0, 6, 8].as_slice(), + FAST_MIN_LITERALS, + ) + }; + // From which kind of table to which one a block went on with the code of the block before. + let mut changes = [[0; 2]; 2]; + let mut dtable = HufDTable::new(); + for (max_nb_bits, &first) in + longest_codes.flat_map(|log| firsts.iter().map(move |first| (log, first))) + { + let mut ctable = HufCTable::new(); + let table_log = build_ctable(&mut ctable, &counts, MAX_SYMBOL, max_nb_bits)?; + assert_eq!(table_log, max_nb_bits as usize); + let mut description = [0; NB_SYMBOLS]; + let size = write_ctable(&mut description, &ctable, MAX_SYMBOL, max_nb_bits)?; + let description = description.sub(..size)?; + + for (index, &(nb_literals, shift, four_streams)) in + blocks.iter().skip(first).chain(blocks).enumerate() + { + let literals: Vec = (0..nb_literals) + .map(|_| { + (((random() & 255).min(random() & 255) * MAX_SYMBOL / 256) >> shift) as u8 + }) + .collect(); + let mut streams = vec![0; 2 * nb_literals]; + let size = if four_streams { + compress_4x_using_ctable(&mut streams, &literals, &ctable)? + } else { + compress_1x_using_ctable(&mut streams, &literals, &ctable)? + }; + streams.truncate(size); + if index == 0 { + let src = [description, &streams].concat(); + let spread = &mut Spread::new(); + let read = dtable.read(&src, TableType::X1, nb_literals, four_streams, spread); + assert_eq!(read, Ok(description.len())); + } else { + let was_single = dtable.single; + dtable.prepare(nb_literals, size, four_streams)?; + *changes + .at_mut(usize::from(was_single))? + .at_mut(usize::from(dtable.single))? += 1; + } + let mut decoded = vec![0; nb_literals]; + assert!(if four_streams { + dtable.decode_4x(&mut decoded, &streams) + } else { + dtable.decode_1x(&mut decoded, &streams) + }); + assert_eq!(decoded, literals); + + // Streams that do not end where they should, some much shorter than the others. + let nb_damaged = match four_streams && nb_literals <= max_damaged { + true if cfg!(miri) => 2, + true => 6, + false => 0, + }; + for _ in 0..nb_damaged { + let mut damaged = streams.clone(); + let at = random() as usize % if random() % 2 == 0 { 6 } else { size }; + *damaged.at_mut(at)? ^= 1 << (random() % 8); + let mut exact = vec![0; nb_literals]; + let result = + huf_exact::decompress_4x(&mut exact, &damaged, &dtable.code, TableType::X1); + assert_eq!( + decompress_4x_using_dtable(&mut decoded, &damaged, &dtable), + result + ); + assert!(result.is_err() || decoded == exact); + } + } + } + assert!(changes.as_flattened().iter().all(|&count| count > 0)); + Ok(()) + } +} diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 82d85bdbec5c..60ff86782aae 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -367,8 +367,8 @@ describe("decompressing frames whose size is known", () => { ]; const whole = Buffer.concat(pieces); - // "raw", "rle", or how many sequences a compressed block has. - function blocks(frame: Uint8Array) { + // "raw", "rle", or how many sequences a compressed block has. Or else how its literals are written. + function blocks(frame: Uint8Array, ofLiterals = false) { const descriptor = frame[4]; const isSingleSegment = (descriptor >> 5) & 1; let at = 5 + (1 - isSingleSegment) + [0, 1, 2, 4][descriptor & 3] + [isSingleSegment, 2, 4, 8][descriptor >> 6]; @@ -395,7 +395,15 @@ describe("decompressing frames whose size is known", () => { literals = [3, 3, 4, 5][format] + (Math.floor(bits / 2 ** (4 + width)) % 2 ** width); } const [first, second, third] = frame.subarray(at + literals); - kinds.push(first < 128 ? first : first < 255 ? ((first - 128) << 8) + second : second + (third << 8) + 0x7f00); + kinds.push( + ofLiterals + ? ["raw", "rle", "code", "same code"][frame[at] & 3] + : first < 128 + ? first + : first < 255 + ? ((first - 128) << 8) + second + : second + (third << 8) + 0x7f00, + ); } else { kinds.push(["raw", "rle"][type]); } @@ -437,6 +445,33 @@ describe("decompressing frames whose size is known", () => { expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); }); + it.each([ + [0xffff, 0, ["raw", "raw", "raw", "code", "code", "code,same code,same code,same code"]], + [0xffffffff, 0, ["raw", "code", "code", "code", "code", "code,code,code,same code"]], + [0xffffffff, 2, ["code", "code", "code", "code", "code", "code,code,same code,same code"]], + ])( + "literals that take many bits: the least of the bytes of a number up to %i, shifted by %i", + (mask, shift, written) => { + let x = 0x9e3779b9; + const kinds = [300, 1500, 3000, 6000, 20_000, 400_000].map(length => { + const data = Buffer.from( + Uint8Array.from({ length }, () => { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + const y = x | ~mask; + return Math.min(y & 255, (y >> 8) & 255, (y >> 16) & 255, y >>> 24) >> shift; + }), + ); + const frame = zstdCompressSync(data, { level: 1 }); + expect(zstdDecompressSync(frame).equals(data)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + return blocks(frame, true).join(); + }); + expect(kinds).toEqual(written); + }, + ); + it("a frame that makes more than it says is refused", () => { for (const piece of pieces.filter(piece => piece.length >= 135_000)) { const frame = zstdCompressSync(piece.subarray(0, 135_000), { level: 1 }); From a77f13505b3a9f92bfc45decc719c9f275db9f0a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 17:32:16 +0000 Subject: [PATCH 153/199] zstd: a buffer whose content is of no use is not copied when it grows `Vec::try_reserve` on a buffer that has a length may copy it to the new allocation, and holds both until it has. With workers and long distance matching, a frame with a larger job size than the one before copied the whole buffer that the input goes through, up to 4 GB. The literals of the decoder and the sequences of a job did the same on a smaller scale. The helper that the sequence store had for this moves to util.rs, next to `reset_table`, and all of them use one or the other. --- src/zstd/compress/cctx.rs | 21 +-------------------- src/zstd/compress/internal.rs | 24 ++++++------------------ src/zstd/compress/mt.rs | 21 ++++++++------------- src/zstd/decompress/block.rs | 15 ++++++++++----- src/zstd/util.rs | 31 +++++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 56 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 0f89fa62d37d..e5f2b647da20 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -37,7 +37,7 @@ use crate::common::{ }; use crate::error::{Error, Result}; use crate::util::{ - SliceExt, highbit32, read_le32, try_zeroed, write_le16, write_le24, write_le32, write_le64, + SliceExt, highbit32, read_le32, reset_table, write_le16, write_le24, write_le32, write_le64, }; /// `ZSTD_compressionStage_e` @@ -166,25 +166,6 @@ fn invalidate_match_state(ms: &mut MatchState) { ms.has_dict_match_state = false; } -/// Gives `table` the length `len`. What it holds is 0, or if `clean` is false possibly what it held before. -pub(crate) fn reset_table( - table: &mut Vec, - len: usize, - clean: bool, -) -> Result<()> { - if len > table.capacity() { - // First, so that both are not allocated at once. - *table = Vec::new(); - *table = try_zeroed(len)?; - return Ok(()); - } - if clean { - table.clear(); - } - table.resize(len, T::default()); - Ok(()) -} - /// `ZSTD_cwksp_mark_tables_dirty` and then `ZSTD_cwksp_mark_tables_clean` fn forget_unused_indices(ms: &mut MatchState) { ms.hash_table.forget_rest(); diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 5ac4c58a3abf..f81139abc86f 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -12,7 +12,9 @@ use crate::common::{ ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, }; use crate::error::{Error, Result}; -use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64, try_zeroed, write_le24}; +use crate::util::{ + BUG, SliceExt, highbit32, make_room, read_le32, read_le64, try_zeroed, write_le24, +}; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; pub(crate) const HASH_READ_SIZE: usize = 8; @@ -360,22 +362,12 @@ impl SeqStore { /// `ZSTD_resetSeqStore`, with room for the sequences and the literals of a block of `block_size` bytes. pub(crate) fn start_block(&mut self, block_size: usize) -> Result<()> { - fn room(vec: &mut Vec, len: usize, max: usize) -> Result<()> { - if len <= vec.capacity() { - return Ok(()); - } - let len = len.max(2 * vec.capacity()).min(max); - // What it holds is not worth a copy. - *vec = Vec::new(); - vec.try_reserve_exact(len) - .map_err(|_| Error::MemoryAllocation) - } self.reset(); let block_size = block_size.min(self.max_nb_lit); let nb_seq = (block_size / MINMATCH as usize).min(self.max_nb_seq); - room(&mut self.sequences, nb_seq, self.max_nb_seq)?; + make_room(&mut self.sequences, nb_seq, self.max_nb_seq)?; let len = block_size + WILDCOPY_OVERLENGTH; - room( + make_room( &mut self.lit_buffer, len, self.max_nb_lit + WILDCOPY_OVERLENGTH, @@ -389,11 +381,7 @@ impl SeqStore { /// Makes `codes` at least as long as `sequences`. pub(crate) fn make_room_for_codes(&mut self) -> Result<()> { let len = self.sequences.len(); - if self.codes.capacity() < len { - self.codes = Vec::new(); - (self.codes.try_reserve_exact(self.sequences.capacity())) - .map_err(|_| Error::MemoryAllocation)?; - } + make_room(&mut self.codes, len, self.max_nb_seq)?; if self.codes.len() < len { self.codes.resize(len, SeqCodes::default()); } diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 88d55d5e9cb2..9891ed621ae1 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpmc::{Receiver, Sender, TryRecvError, channel}; -use super::cctx::{CDict, FrameCompressor, Input, Span, reset_table, write_last_empty_block}; +use super::cctx::{CDict, FrameCompressor, Input, Span, write_last_empty_block}; use super::internal::{ BufId, Buffers, LdmParams, LdmState, ParamSwitch, RawSeq, RawSeqStore, Strategy, Window, }; @@ -24,7 +24,7 @@ use super::stream::EndDirective; use crate::common::ZSTD_BLOCKSIZE_MAX; use crate::common::xxhash::Xxh64; use crate::error::{Error, Result}; -use crate::util::{BUG, SliceExt, append, try_copy}; +use crate::util::{BUG, SliceExt, append, make_room, reset_table, try_copy}; use crate::{InBuffer, OutBuffer}; /// `SerialState`: what has to see the input in order. @@ -70,8 +70,11 @@ impl SerialState { reset_table(&mut self.ldm_state.bucket_offsets, 1 << bucket_log, true)?; self.ldm_state.loaded_dict_end = 0; self.round_buff.clear(); - (self.round_buff.try_reserve_exact(round_buff_capacity)) - .map_err(|_| Error::MemoryAllocation)?; + make_room( + &mut self.round_buff, + round_buff_capacity, + round_buff_capacity, + )?; } else { self.ldm_state = LdmState::new(); self.round_buff = Vec::new(); @@ -159,14 +162,6 @@ struct WorkerState { const CHUNK_SIZE: usize = 4 * ZSTD_BLOCKSIZE_MAX; -/// Makes `seq` `len` long, in the allocation that it has if that is large enough. -fn resize_in_place(seq: &mut Vec, len: usize) -> Result<()> { - seq.clear(); - seq.try_reserve(len).map_err(|_| Error::MemoryAllocation)?; - seq.resize(len, RawSeq::default()); - Ok(()) -} - /// `ZSTDMT_compressionJob` fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) -> Result<()> { let mut job_params = job.params; @@ -185,7 +180,7 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - let generated = if cancelled.load(Ordering::SeqCst) { Err(Error::Generic) } else { - resize_in_place(&mut raw_seq_store.seq, job.nb_seq_bound) + reset_table(&mut raw_seq_store.seq, job.nb_seq_bound, true) .and_then(|()| serial.gen_sequences(&mut raw_seq_store, src, job.round_pos)) }; // The receiver is gone if the frame is given up. diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 29cb6c397c98..17a456e9bfbb 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -15,10 +15,13 @@ use crate::common::fse_decompress::Spread; use crate::common::{ BlockType, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MAX_OFF, MIN_CBLOCK_SIZE, MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, MINMATCH, ML_FSE_LOG, OFF_FSE_LOG, - REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, + REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, + ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, }; use crate::error::{Error, Result}; -use crate::util::{BUG, SliceExt, append, append_within, copy, read_le16, read_le24, read_le32}; +use crate::util::{ + BUG, SliceExt, append, append_within, copy, make_room, read_le16, read_le24, read_le32, +}; /// `blockProperties_t` pub(crate) struct BlockProperties { @@ -330,10 +333,12 @@ impl Scratch { fn lit_buffer(&mut self, lit_size: usize) -> Result<&mut [u8]> { let needed = lit_size + WILDCOPY_OVERLENGTH; + make_room( + &mut self.lit_buffer, + needed, + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH, + )?; if self.lit_buffer.len() < needed { - self.lit_buffer - .try_reserve(needed - self.lit_buffer.len()) - .map_err(|_| Error::MemoryAllocation)?; self.lit_buffer.resize(needed, 0); } // Literals are copied by more than there are: this much can be read along with the last ones. diff --git a/src/zstd/util.rs b/src/zstd/util.rs index 19037519dc8d..85c8a9a4441f 100644 --- a/src/zstd/util.rs +++ b/src/zstd/util.rs @@ -251,6 +251,37 @@ pub(crate) fn try_zeroed(len: usize) -> Result> { bun_pod::try_zeroed_vec(len).ok_or(Error::MemoryAllocation) } +/// Gives `table` the length `len`. What it holds is 0, or if `clean` is false possibly what it held before. +pub(crate) fn reset_table( + table: &mut Vec, + len: usize, + clean: bool, +) -> Result<()> { + if len > table.capacity() { + // First, so that both are not allocated at once. + *table = Vec::new(); + *table = try_zeroed(len)?; + return Ok(()); + } + if clean { + table.clear(); + } + table.resize(len, T::default()); + Ok(()) +} + +/// Makes the capacity of `vec` at least `len`. A `vec` that has to grow is emptied first, so that nothing is copied and +/// both are not allocated at once. It gets twice its capacity, or `max` if that is less. +pub(crate) fn make_room(vec: &mut Vec, len: usize, max: usize) -> Result<()> { + if len <= vec.capacity() { + return Ok(()); + } + let capacity = len.max((2 * vec.capacity()).min(max)); + *vec = Vec::new(); + vec.try_reserve_exact(capacity) + .map_err(|_| Error::MemoryAllocation) +} + /// A copy of `bytes`, or [`Error::MemoryAllocation`]. pub(crate) fn try_copy(bytes: &[u8]) -> Result> { let mut vec = Vec::new(); From a87cc412f551155477699888df5d6a3aae1e01ab Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 17:32:41 +0000 Subject: [PATCH 154/199] zstd: less for the optimal parser to do where there is no match At levels 16 and up, data with few matches spends its time in what is done at every position whatever is found there: - The three repeat offsets are compared at once, and the loop over them is only entered if one of them is a candidate. - The candidate of the table of 3-byte hashes is compared by its first bytes before the call that counts its length. - A position whose hash has no candidate in the window gets an empty tree without going through the walk. The tables of the parser have a position for each byte of a block instead of always 4099, so that a context for a short input has less to clear. Level 19 in Bun, time relative to before: noise, base64 and skewed bytes 0.82 to 0.86 for 1 KB and 16 KB and 0.91 to 0.95 for 128 KB and 1 MB; 1 KB of zeros 0.79; JSON 0.87 for 1 KB, 0.96 to 0.98 above; logs 0.95 to 1.00. --- src/zstd/compress/cctx.rs | 47 +++++++++++++++++--------- src/zstd/compress/internal.rs | 18 ++++------ src/zstd/compress/opt.rs | 63 ++++++++++++++++++++++------------- src/zstd/lib.rs | 21 ++++++++++++ test/js/bun/util/zstd.test.ts | 18 ++++++++++ 5 files changed, 116 insertions(+), 51 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index e5f2b647da20..c311f5c72561 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -12,9 +12,9 @@ use super::fse::FseRepeat; use super::huf::{self, HufRepeat}; use super::internal::{ BlockSrc, BufId, Buffers, CompressedBlockState, CompressionParameters, DictMatchState, - EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, Match, MatchState, - ParamSwitch, Prefix, RawSeq, RawSeqStore, RowTags, SeqChunk, SeqStore, Strategy, Table, - TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, + EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, Match, MatchState, OptState, + ParamSwitch, Prefix, PriceTable, RawSeq, RawSeqStore, RowTags, SeqChunk, SeqStore, Strategy, + Table, TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, min_gain, no_compress_block, offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, }; @@ -224,31 +224,43 @@ fn reset_match_state( ms.row_hash_log = c_params.hash_log - row_log; } - if for_who == ResetTarget::CCtx && c_params.strat() >= Strategy::BtOpt { - reset_table(&mut ms.opt.lit_freq, 1 << LITBITS, false)?; - reset_table(&mut ms.opt.lit_length_freq, MAX_LL as usize + 1, false)?; - reset_table(&mut ms.opt.match_length_freq, MAX_ML as usize + 1, false)?; - reset_table(&mut ms.opt.off_code_freq, MAX_OFF as usize + 1, false)?; - reset_table(&mut ms.opt.match_table, ZSTD_OPT_SIZE, false)?; - ms.opt.price_table.resize(ZSTD_OPT_SIZE)?; - } - ms.c_params = *c_params; Ok(()) } +/// The part of `ZSTD_reset_matchState` that is for the optimal parser. +fn reset_opt_state(opt: &mut OptState, block_size: usize) -> Result<()> { + reset_table(&mut opt.lit_freq, 1 << LITBITS, false)?; + reset_table(&mut opt.lit_length_freq, MAX_LL as usize + 1, false)?; + reset_table(&mut opt.match_length_freq, MAX_ML as usize + 1, false)?; + reset_table(&mut opt.off_code_freq, MAX_OFF as usize + 1, false)?; + let nb_positions = nb_opt_positions(block_size); + reset_table(&mut opt.match_table, nb_positions, false)?; + opt.price_table.resize(nb_positions) +} + +/// How many positions the tables of the optimal parser have: `ZSTD_OPT_SIZE`, or fewer if no block is that long. One +/// before each byte, one after the last, and one more that is only given a price. +fn nb_opt_positions(block_size: usize) -> usize { + ZSTD_OPT_SIZE.min(block_size + 2) +} + /// What the tables of the optimal parser take for a position: a `Match`, and 7 numbers in the `PriceTable`. const OPT_POSITION_SIZE: usize = size_of::() + 7 * size_of::(); /// How many bytes the tables of a context take. -fn tables_size(c_params: &CompressionParameters, use_row_match_finder: ParamSwitch) -> usize { +fn tables_size( + c_params: &CompressionParameters, + use_row_match_finder: ParamSwitch, + block_size: usize, +) -> usize { let sizes = MatchStateSizes::new(c_params, use_row_match_finder, true); let mut size = (sizes.h_size + sizes.chain_size + sizes.h3_size) * size_of::(); if row_match_finder_used(c_params.strat(), use_row_match_finder) { size += sizes.h_size; } if c_params.strat() >= Strategy::BtOpt { - size += ZSTD_OPT_SIZE * OPT_POSITION_SIZE; + size += nb_opt_positions(block_size) * OPT_POSITION_SIZE; } size } @@ -958,7 +970,7 @@ impl FrameCompressor { let new_tables = if self.workspace.is_some() { new_workspace } else { - let needed = tables_size(¶ms.c_params, params.use_row_match_finder); + let needed = tables_size(¶ms.c_params, params.use_row_match_finder, block_size); let oversized = tables_capacity(&self.match_state) >= needed.saturating_mul(ZSTD_WORKSPACETOOLARGE_FACTOR); self.oversized_duration = if oversized { @@ -976,7 +988,7 @@ impl FrameCompressor { ms.hash_table3 = Table::new(); ms.tag_table = Table::new(); ms.opt.match_table = Vec::new(); - ms.opt.price_table.resize(0)?; + ms.opt.price_table = PriceTable::default(); } self.match_state.c_params = params.c_params; @@ -1006,6 +1018,9 @@ impl FrameCompressor { ResetTarget::CCtx, (pledged_src_size != ZSTD_CONTENTSIZE_UNKNOWN).then_some(pledged_src_size), )?; + if params.c_params.strat() >= Strategy::BtOpt { + reset_opt_state(&mut self.match_state.opt, block_size)?; + } if params.ldm_params.enable_ldm == ParamSwitch::Enable { let ldm_h_size = 1usize << params.ldm_params.hash_log; diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index f81139abc86f..abede0e08d45 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -13,7 +13,7 @@ use crate::common::{ }; use crate::error::{Error, Result}; use crate::util::{ - BUG, SliceExt, highbit32, make_room, read_le32, read_le64, try_zeroed, write_le24, + BUG, SliceExt, highbit32, make_room, read_le32, read_le64, reset_table, try_zeroed, write_le24, }; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; @@ -1977,17 +1977,11 @@ pub(crate) struct PriceTable { impl PriceTable { /// Makes room for `len` positions. pub(crate) fn resize(&mut self, len: usize) -> Result<()> { - if self.price.len() != len { - *self = Self::default(); - *self = Self { - price: try_zeroed(len)?, - off: try_zeroed(len)?, - lens: try_zeroed(len)?, - rep: try_zeroed(len)?, - ml_price: try_zeroed(len)?, - }; - } - Ok(()) + reset_table(&mut self.price, len, false)?; + reset_table(&mut self.off, len, false)?; + reset_table(&mut self.lens, len, false)?; + reset_table(&mut self.rep, len, false)?; + reset_table(&mut self.ml_price, len, false) } } diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index a5a384972c55..19460d9b32a3 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -8,7 +8,7 @@ use super::huf::HufRepeat; use super::internal::{ BlockSrc, Common, Descent, DictMatchState, DictMode, DictTree, EntropyCTables, Hasher, Match, MatchState, Near, Nodes, OptState, ParamSwitch, Prefix, PriceTable, RawSeq, RawSeqStore, - SeqStore, Strategy, TreeInput, ZSTD_OPT_NUM, ZSTD_OPT_SIZE, bt_look_ahead, bt_nodes, bt_walk, + SeqStore, Strategy, TreeInput, ZSTD_OPT_NUM, bt_look_ahead, bt_nodes, bt_set_children, bt_walk, hash3_of, index_overlap_check, ll_code, ml_code, new_rep, offset_to_offbase, table_get, table_put, }; @@ -567,6 +567,10 @@ impl<'a> Trees<'a> { let match_index = table_get(self.hash_table, h); table_put(self.hash_table, h, curr); bt_look_ahead::<4, 2>(self.hash_table, self.bt, &self.input, self.hasher, ip); + if match_index < low { + bt_set_children(self.bt, curr, [0, 0]); + return Ok(self.nb_compares); + } let descent = Descent::new(self.bt, curr); bt_walk::( @@ -720,25 +724,29 @@ impl<'a> Trees<'a> { }; // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. let most_in_prefix = ip.min(curr.wrapping_sub(window_low)); - for (off, &rep_offset) in (1..).zip(&rep_offsets) { + let candidates = rep_offsets.map(|rep_offset| { // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. if rep_offset.wrapping_sub(1) < most_in_prefix { - let rep_match = ip.wrapping_sub(rep_offset); - let rep_bytes = near.read32(rep_match).ok_or(BUG)?; - if minmatch_differs(rep_bytes, bytes as u32, min_match) != 0 { + let rep_bytes = near.read32(ip.wrapping_sub(rep_offset))?; + Some(minmatch_differs(rep_bytes, bytes as u32, min_match) == 0) + } else { + Some(rep_offset.wrapping_sub(1) >= ip) + } + }); + if candidates != [Some(false); 3] { + for ((off, rep_offset), is_candidate) in (1..).zip(rep_offsets).zip(candidates) { + if !is_candidate.ok_or(BUG)? { continue; } - } else if rep_offset.wrapping_sub(1) < ip { - continue; - } - let rep_len = self.rep_len(ip, rep_offset, window_low)?; - if rep_len > best_length { - best_length = rep_len; - matches.put(mnum as usize, Match { off, len: rep_len })?; - mnum += 1; - if rep_len > self.sufficient_len || rep_len == near.len().wrapping_sub(ip) { - // The best there can be. - return Ok(mnum); + let rep_len = self.rep_len(ip, rep_offset, window_low)?; + if rep_len > best_length { + best_length = rep_len; + matches.put(mnum as usize, Match { off, len: rep_len })?; + mnum += 1; + if rep_len > self.sufficient_len || rep_len == near.len().wrapping_sub(ip) { + // The best there can be. + return Ok(mnum); + } } } } @@ -750,7 +758,13 @@ impl<'a> Trees<'a> { // Farther than that is likely to cost too much. if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) { let mlen = if match_index3 >= dict_limit { - count(near, ip, match_index3.wrapping_sub(dict_limit)) + let match_pos = match_index3.wrapping_sub(dict_limit); + let match_bytes = near.read32(match_pos).ok_or(BUG)?; + if minmatch_differs(match_bytes, bytes as u32, 3) != 0 { + 0 + } else { + count(near, ip, match_pos) + } } else { self.input.count_ext_dict(ip, match_index3)? }; @@ -1191,11 +1205,14 @@ fn parse_block( rescale_freqs(opt_state, src.symbol_costs, buf.sub(istart..iend)?)?; // Of the same size, so that one check is enough for all. - let opt_price: &mut [_; ZSTD_OPT_SIZE] = price_table.price.first_chunk_mut().ok_or(BUG)?; - let opt_off: &mut [_; ZSTD_OPT_SIZE] = price_table.off.first_chunk_mut().ok_or(BUG)?; - let opt_lens: &mut [_; ZSTD_OPT_SIZE] = price_table.lens.first_chunk_mut().ok_or(BUG)?; - let opt_rep: &mut [_; ZSTD_OPT_SIZE] = price_table.rep.first_chunk_mut().ok_or(BUG)?; - let ml_prices = price_table.ml_price.sub_mut(..=sufficient_len as usize)?; + let nb_positions = price_table.price.len(); + let opt_price = price_table.price.sub_mut(..nb_positions)?; + let opt_off = price_table.off.sub_mut(..nb_positions)?; + let opt_lens = price_table.lens.sub_mut(..nb_positions)?; + let opt_rep = price_table.rep.sub_mut(..nb_positions)?; + // No match is longer than a block. + let ml_prices = + (price_table.ml_price).sub_mut(..(sufficient_len as usize + 1).min(nb_positions))?; let mut prices = Prices::new(opt_state, ml_prices, opt_level)?; ip += usize::from(ip == prefix_start); @@ -1272,7 +1289,7 @@ fn parse_block( cur += 1; let at = cur as usize; let inr = ip + at; - if at + 1 >= ZSTD_OPT_SIZE { + if at + 1 >= nb_positions { return Err(BUG); } diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index f4716d6bba13..0a657ea6d686 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -519,6 +519,27 @@ mod tests { Ok(frame) } + #[test] + fn tables_that_change_size_give_the_same_frames() -> Result<()> { + let mut state = 11u32; + let src: Vec = (0..if cfg!(miri) { 300 } else { 6000 }) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + (state >> 8) as u8 % 5 + }) + .collect(); + // The tables of the optimal parser are as long as a block. + let mut cctx = CCtx::new(); + cctx.set_parameter(c_param::COMPRESSION_LEVEL, 19)?; + let first = frame_of(&mut cctx, &[&src])?; + frame_of(&mut cctx, &[src.sub(..src.len() / 30)?])?; + assert_eq!(frame_of(&mut cctx, &[&src])?, first); + assert_eq!(decompress_alloc(&first), Ok(src)); + Ok(()) + } + #[test] fn long_matches_in_a_dictionary_are_appended() -> Result<()> { let mut state = 7u32; diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 60ff86782aae..7589bb5fe23b 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1443,6 +1443,24 @@ describe("the output is that of zstd 1.5.7", () => { `); }); + it("where a block is one match, which is too short for the optimal parser to take at once", () => { + const digests = (data: Buffer, windowLog: number) => + [16, 19, 22].map(level => { + const params = { + [c.ZSTD_c_compressionLevel]: level, + [c.ZSTD_c_windowLog]: windowLog, + [c.ZSTD_c_targetLength]: 4096, + }; + const compressed = zlib.zstdCompressSync(data, { params }); + expect(zlib.zstdDecompressSync(compressed).equals(data)).toBe(true); + return digest(compressed); + }); + const periodic = Buffer.alloc(5_000, corpus.subarray(0, 700)); + expect(digests(periodic, 10)).toEqual(Array(3).fill("163 571cb5ea10428fa7")); + expect(digests(periodic, 11)).toEqual(Array(3).fill("142 671efcd2063db130")); + expect(digests(Buffer.alloc(1_000), 10)).toEqual(Array(3).fill("19 bf47b44041ed0aa6")); + }); + // Short input searches the tables of the dictionary, long input starts from a copy of them. function withDictionary(dictionary: Uint8Array) { const results: Record = {}; From 11efee8c0a66648f92b944637bc267d08ffa9f7a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 19:04:56 +0000 Subject: [PATCH 155/199] zstd: btlazy2 hands shorter runs of positions to the tight insertion loop The loop that adds positions to the trees inside the search has its constants on the stack. Runs of 32 positions and more went to a function of its own, which has them in registers. For segments of up to 64 KB, where the loop inside gives no hints at what is about to be read, that now starts at 8. 16 KB at level 9, us, before and after (C): noise 38.4, 35.6 (31.8); text, JSON and logs 1% less or the same. Larger inputs are not affected. --- src/zstd/compress/lazy.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 047ab789a188..1215ec4ed23c 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -101,7 +101,12 @@ impl<'a> Dubt<'a> { fn update(&mut self, curr: u32) -> Result<()> { let near = self.input.near; let next_to_update = *self.next_to_update; - if curr.saturating_sub(next_to_update) >= LONG_UPDATE { + let long_update = if self.input.far { + LONG_UPDATE_FAR + } else { + LONG_UPDATE + }; + if curr.saturating_sub(next_to_update) >= long_update { let first = next_to_update..curr; let dict_limit = self.input.dict_limit; return update_many( @@ -304,11 +309,12 @@ impl Dubt<'_> { } } -/// From that many positions on, [`Dubt::update`] leaves them to [`update_many`]. -const LONG_UPDATE: u32 = 32; +/// From that many positions on, [`Dubt::update`] leaves them to [`update_many`], whose loop is tighter. +const LONG_UPDATE: u32 = 8; +/// The same where `update` gives hints at what is about to be read, which `update_many` does not. +const LONG_UPDATE_FAR: u32 = 32; -/// `ZSTD_updateDUBT` of the indices `indices`, which are in `near`, whose first index is `dict_limit`. There are enough -/// of them not to wait for memory. +/// `ZSTD_updateDUBT` of the indices `indices`, which are in `near`, whose first index is `dict_limit`. #[cold] #[inline(never)] fn update_many( From 7ddcdb59ce69742e7a43a827c1d09db16c13e0d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 19:05:03 +0000 Subject: [PATCH 156/199] zstd: the block splitter reports an allocation that fails As in C, an estimate that fails means that a block is not split. In C an estimate allocates nothing. Here the first one makes room for the codes of the sequences, so when that failed the frame came out valid, with success reported, but not split where C splits it. --- src/zstd/compress/cctx.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index c311f5c72561..d2c134ea7f42 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -1423,14 +1423,19 @@ impl FrameCompressor { } let mut estimate = |from: usize, to: usize| -> Result> { let chunk = derive_seq_store_chunk(&self.seq_store, from, to)?; - Ok(build_entropy_statistics_and_estimate_sub_block_size( + let size = build_entropy_statistics_and_estimate_sub_block_size( &mut self.seq_store, &chunk, &block_state(&self.prev_c_block).entropy, &mut block_state_mut(&mut self.next_c_block).entropy, &self.applied_params, entropy_metadata, - )) + ); + match size { + // Not among what makes for no split: the C code allocates nothing here. + Err(Error::MemoryAllocation) => Err(Error::MemoryAllocation), + size => Ok(size), + } }; let estimated_original_size = estimate(start_idx, end_idx)?; let estimated_first_half_size = estimate(start_idx, mid_idx)?; From 0cfb03b5c5f0cf171189357d91990a6ee3723747 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 19:05:18 +0000 Subject: [PATCH 157/199] zstd: a block asks for room once its sequences are known Output to the spare capacity of a `Vec` has to be zeroed before an encoder writes to it. A block got room for its own size before the match finder ran, so 128 KB of zeros took 129 KB of zeroing for 20 bytes of output. It now asks when the literals and the sequences are known, which bound what they take. A block that is stored is appended, which takes no zeros. A block without a sequence leaves its literals where they are instead of copying them to the store. If they are too few, not to be compressed, or start and end like noise, which is what `ZSTD_compressLiterals` looks at first, the block is known to come out larger than it went in, and is stored without writing it as raw literals first. This is for blocks in one part. Those that may be split (levels 16 and up) or have a target size spend their time elsewhere. 128 KB through `compress_alloc`, us, before, after and the C library, at level 1: noise 15.6, 9.5, 13.9; zeros 10.2, 8.9, 7.8; a period of 1 KB 11.1, 9.9, 9.1; base64 107.3, 104.2, 100.1. At level 3: noise 22.6, 16.9, 21.7; zeros 14.6, 13.0, 9.7; a period of 1 KB 17.0, 14.9, 13.8; base64 115.3, 111.6, 107.1. --- src/zstd/compress/block_entropy.rs | 16 +++- src/zstd/compress/cctx.rs | 144 +++++++++++++++++++---------- src/zstd/compress/huf.rs | 46 +++++---- src/zstd/compress/internal.rs | 18 ++-- src/zstd/compress/sequences.rs | 6 ++ src/zstd/compress/superblock.rs | 10 +- src/zstd/lib.rs | 39 ++++++++ test/js/bun/util/zstd.test.ts | 28 ++++++ 8 files changed, 222 insertions(+), 85 deletions(-) diff --git a/src/zstd/compress/block_entropy.rs b/src/zstd/compress/block_entropy.rs index 096b65aa3152..a71a7c6cd989 100644 --- a/src/zstd/compress/block_entropy.rs +++ b/src/zstd/compress/block_entropy.rs @@ -197,19 +197,24 @@ fn entropy_compress_seq_store_internal( prev_entropy: &EntropyCTables, next_entropy: &mut EntropyCTables, cctx_params: &CCtxParams, + sampled_literals: Option<&[u8]>, ) -> Result { const SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO: usize = 20; let strategy = cctx_params.c_params.strat(); let nb_seq = chunk.nb_seq(); let lit_size = chunk.lit_size(); let mut op = 0usize; + debug_assert!(sampled_literals.is_none() || nb_seq == 0); { - let suspect_uncompressible = - nb_seq == 0 || lit_size / nb_seq >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO; + let suspect_uncompressible = sampled_literals.is_none() + && (nb_seq == 0 || lit_size / nb_seq >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); op += compress_literals( dst, - store.lit_buffer.sub(chunk.lit_start..chunk.lit_end)?, + match sampled_literals { + Some(literals) => literals, + None => store.lit_buffer.sub(chunk.lit_start..chunk.lit_end)?, + }, &prev_entropy.huf, &mut next_entropy.huf, strategy, @@ -266,7 +271,8 @@ fn entropy_compress_seq_store_internal( } /// `ZSTD_entropyCompressSeqStore`: the content of a compressed block for `chunk`, which stands for `block_size` bytes. -/// Returns 0 if the block is better left uncompressed. +/// Returns 0 if the block is better left uncompressed. A block without sequences can have its literals in +/// `sampled_literals` instead of the store, if [`huf::samples_are_incompressible`] has been asked and is false for them. pub(crate) fn entropy_compress_seq_store( store: &mut SeqStore, chunk: &SeqChunk, @@ -275,6 +281,7 @@ pub(crate) fn entropy_compress_seq_store( cctx_params: &CCtxParams, dst: &mut [u8], block_size: usize, + sampled_literals: Option<&[u8]>, ) -> Result { let c_size = match entropy_compress_seq_store_internal( dst, @@ -283,6 +290,7 @@ pub(crate) fn entropy_compress_seq_store( prev_entropy, next_entropy, cctx_params, + sampled_literals, ) { // There is room for the block as it is. Err(Error::DstSizeTooSmall) if block_size <= dst.len() => return Ok(0), diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index d2c134ea7f42..171dbddb698f 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -7,6 +7,7 @@ use std::sync::atomic::AtomicBool; use super::block_entropy::{ build_entropy_statistics_and_estimate_sub_block_size, count_seq_store_literals_bytes, count_seq_store_match_bytes, derive_seq_store_chunk, entropy_compress_seq_store, + literals_compression_is_disabled, }; use super::fse::FseRepeat; use super::huf::{self, HufRepeat}; @@ -15,8 +16,8 @@ use super::internal::{ EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, Match, MatchState, OptState, ParamSwitch, Prefix, PriceTable, RawSeq, RawSeqStore, RowTags, SeqChunk, SeqStore, Strategy, Table, TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, - ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, min_gain, no_compress_block, - offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, + ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, block_header, min_gain, + no_compress_block, offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, }; use super::literals::min_literals_to_compress; use super::params::{ @@ -26,7 +27,7 @@ use super::params::{ row_match_finder_used, }; use super::superblock::compress_super_block; -use super::{compress_block, double_fast, fast, lazy, ldm, opt, presplit}; +use super::{compress_block, double_fast, fast, lazy, ldm, opt, presplit, sequences}; use crate::common::entropy::read_ncount; use crate::common::xxhash::Xxh64; use crate::common::{ @@ -37,7 +38,8 @@ use crate::common::{ }; use crate::error::{Error, Result}; use crate::util::{ - SliceExt, highbit32, read_le32, reset_table, write_le16, write_le24, write_le32, write_le64, + BUG, SliceExt, append, highbit32, read_le32, reset_table, write_le16, write_le24, write_le32, + write_le64, }; /// `ZSTD_compressionStage_e` @@ -113,7 +115,7 @@ pub(crate) struct Input<'a> { pub(crate) enum Output<'a> { Slice(&'a mut [u8]), /// What is from `start` to `end` in `vec`, which has the capacity for it. Only what is asked for becomes part of its - /// length, as zeros: most of the room for a frame is never written to. + /// length, as zeros, and what is appended: most of the room for a frame is never written to. Spare { vec: &'a mut Vec, start: usize, @@ -121,10 +123,43 @@ pub(crate) enum Output<'a> { }, } -/// The header of a block, and far more than any encoder wants beyond what it writes. +/// The header of a block, those of its literals and its sequences, the description of a Huffman table, and far more than +/// any encoder wants beyond what it writes. const ROOM_TO_SPARE: usize = 1024; impl Output<'_> { + /// How many bytes there are from `pos` on. + fn room_from(&self, pos: usize) -> usize { + match self { + Output::Slice(slice) => slice.len().saturating_sub(pos), + Output::Spare { start, end, .. } => end.saturating_sub(*start + pos), + } + } + + /// [`no_compress_block`] at `pos`, up to which the output is written. + fn store_block(&mut self, pos: usize, src: &[u8], last_block: bool) -> Result { + match self { + Output::Slice(slice) => no_compress_block(slice.sub_mut(pos..)?, src, last_block), + Output::Spare { vec, start, end } => { + let pos = *start + pos; + let size = ZSTD_BLOCKHEADERSIZE + src.len(); + if pos + size > *end { + return Err(Error::DstSizeTooSmall); + } + if pos > vec.len() { + return Err(BUG); + } + // Appended, which takes no zeros first. + vec.truncate(pos); + let header = block_header(BlockType::Raw, src.len(), last_block); + let [header @ .., _] = header.to_le_bytes(); + append(vec, &header)?; + append(vec, src)?; + Ok(size) + } + } + } + /// What there is from `pos` on, or `needed` bytes of it at least. fn from(&mut self, pos: usize, needed: usize) -> Result<&mut [u8]> { match self { @@ -652,9 +687,9 @@ fn write_block_header( last_block: bool, ) -> Result<()> { let c_block_header = if c_size == 1 { - u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((block_size as u32) << 3) + block_header(BlockType::Rle, block_size, last_block) } else { - u32::from(last_block) + ((BlockType::Compressed as u32) << 1) + ((c_size as u32) << 3) + block_header(BlockType::Compressed, c_size, last_block) }; write_le24(op, 0, c_block_header) } @@ -788,7 +823,7 @@ pub(crate) fn write_last_empty_block(dst: &mut [u8]) -> Result { if dst.len() < ZSTD_BLOCKHEADERSIZE { return Err(Error::DstSizeTooSmall); } - write_le24(dst, 0, 1 + ((BlockType::Raw as u32) << 1))?; + write_le24(dst, 0, block_header(BlockType::Raw, 0, true))?; Ok(ZSTD_BLOCKHEADERSIZE) } @@ -1230,7 +1265,8 @@ impl FrameCompressor { /// `ZSTD_buildSeqStore` fn build_seq_store(&mut self, input: &Input<'_>, src: Span) -> Result { Ok(match self.find_sequences(input, src)? { - Some(rep) => { + Some((rep, last_ll_size)) => { + (self.seq_store).store_last_literals(src.bytes(&input.bufs)?, last_ll_size)?; block_state_mut(&mut self.next_c_block).rep = rep; BuildSeqStore::Compress } @@ -1238,13 +1274,13 @@ impl FrameCompressor { }) } - /// `ZSTD_buildSeqStore`, but for the repeat offsets after the block, which it returns rather than store. `None` is - /// `ZSTDbss_noCompress`. + /// `ZSTD_buildSeqStore`, but for the repeat offsets after the block and the literals after the last sequence, which + /// are not stored: it returns the former and the number of the latter. `None` is `ZSTDbss_noCompress`. fn find_sequences( &mut self, input: &Input<'_>, src: Span, - ) -> Result> { + ) -> Result> { let params = &self.applied_params; let ms = &mut self.match_state; let src_size = src.len(); @@ -1333,20 +1369,20 @@ impl FrameCompressor { dict_mode, )? }; - let last_literals = block.buf.sub(src.end - last_ll_size..src.end)?; - self.seq_store.store_last_literals(last_literals)?; - Ok(Some(rep_after)) + Ok(Some((rep_after, last_ll_size))) } - /// Whether the block is nothing but literals, too few for `ZSTD_compressLiterals` to try, which is larger than the - /// block is, if it fits in `dst_size` bytes at all. The tables for the next block need not be made to find that out. - fn is_too_short_to_compress(&self, dst_size: usize) -> bool { + /// Whether `ZSTD_compressLiterals` would leave `literals` as they are without a look at all of them. A block that is + /// nothing else then comes out larger than it went in: to find that out, the literals need not be copied and the + /// tables for the next block need not be made. + fn literals_stay_as_they_are(&self, literals: &[u8]) -> Result { let prev_huf = &block_state(&self.prev_c_block).entropy.huf; let strategy = self.applied_params.c_params.strat(); - let store = &self.seq_store; - store.sequences.is_empty() - && store.lit_len < min_literals_to_compress(strategy, prev_huf.repeat_mode) - && store.lit_len + 3 + 4 <= dst_size + Ok( + literals.len() < min_literals_to_compress(strategy, prev_huf.repeat_mode) + || literals_compression_is_disabled(&self.applied_params) + || huf::samples_are_incompressible(literals)?, + ) } /// `ZSTD_compressSeqStore_singleBlock` @@ -1377,6 +1413,7 @@ impl FrameCompressor { &self.applied_params, dst.sub_mut(ZSTD_BLOCKHEADERSIZE..)?, src.len(), + None, )?; if !self.is_first_block && c_seqs_size < RLE_MAX_LENGTH && is_rle(src) { @@ -1553,7 +1590,8 @@ impl FrameCompressor { /// better left uncompressed. fn compress_block_internal( &mut self, - dst: &mut [u8], + dst: &mut Output<'_>, + op: usize, input: &Input<'_>, src: Span, ) -> Result { @@ -1562,28 +1600,45 @@ impl FrameCompressor { let c_size = match self.find_sequences(input, src)? { None => Ok(0), - Some(rep) => { - let mut c_size = if self.is_too_short_to_compress(dst.len()) { + Some((rep, last_ll_size)) => { + let room = dst.room_from(op).saturating_sub(ZSTD_BLOCKHEADERSIZE); + // If they fit as they are, which spares thinking about what fails first. They stay out of the store. + let only_literals = + self.seq_store.sequences.is_empty() && bytes.len() + 3 + 4 <= room; + let mut c_size = if only_literals && self.literals_stay_as_they_are(bytes)? { Ok(0) } else { + let store = &mut self.seq_store; + let most = if only_literals { + huf::room_for_quick_loops(bytes.len()) + } else { + store.store_last_literals(bytes, last_ll_size)?; + // A block that comes out no smaller than it went in is stored as it is, so it makes no + // difference when an encoder runs out of that much room. + (huf::room_for_quick_loops(store.lit_len) + + sequences::max_size(store.sequences.len())) + .min(bytes.len()) + }; let next = block_state_mut(&mut self.next_c_block); next.rep = rep; - let whole = self.seq_store.whole(); + let whole = store.whole(); entropy_compress_seq_store( - &mut self.seq_store, + store, &whole, &block_state(&self.prev_c_block).entropy, &mut next.entropy, &self.applied_params, - dst, + (dst.from(op, most + ROOM_TO_SPARE)?).sub_mut(ZSTD_BLOCKHEADERSIZE..)?, bytes.len(), + only_literals.then_some(bytes), ) }; if !self.is_first_block && c_size.is_ok_and(|c_size| c_size < RLE_MAX_LENGTH) && is_rle(bytes) { - dst.put(0, bytes.at(0)?)?; + dst.from(op, ZSTD_BLOCKHEADERSIZE + 1)? + .put(ZSTD_BLOCKHEADERSIZE, bytes.at(0)?)?; c_size = Ok(1); } c_size @@ -1694,20 +1749,7 @@ impl FrameCompressor { start: ip, end: ip + block_size, }; - let in_parts = self.applied_params.target_c_block_size != 0 - || self.applied_params.post_block_splitter == ParamSwitch::Enable; - // A block in one part that comes out no smaller than it went in is stored as it is, so it makes no difference - // when an encoder runs out of this much room. Parts, which take long to make anyway, get all there is. - let out = dst.from( - op, - if in_parts { - usize::MAX - } else { - block_size + ROOM_TO_SPARE - }, - )?; - - if out.len() < ZSTD_BLOCKHEADERSIZE + MIN_CBLOCK_SIZE + 1 { + if dst.room_from(op) < ZSTD_BLOCKHEADERSIZE + MIN_CBLOCK_SIZE + 1 { return Err(Error::DstSizeTooSmall); } @@ -1728,19 +1770,19 @@ impl FrameCompressor { } ms.next_to_update = ms.next_to_update.max(ms.window.low_limit); + // Parts, which take long to make anyway, get all the room there is. let c_size = if self.applied_params.target_c_block_size != 0 { + let out = dst.from(op, usize::MAX)?; self.compress_block_target_c_block_size(out, input, block, last_block)? } else if self.applied_params.post_block_splitter == ParamSwitch::Enable { + let out = dst.from(op, usize::MAX)?; self.compress_block_split_block(out, input, block, last_block)? } else { - let c_size = self.compress_block_internal( - out.sub_mut(ZSTD_BLOCKHEADERSIZE..)?, - input, - block, - )?; + let c_size = self.compress_block_internal(dst, op, input, block)?; if c_size == 0 { - no_compress_block(out, block.bytes(&input.bufs)?, last_block)? + dst.store_block(op, block.bytes(&input.bufs)?, last_block)? } else { + let out = dst.from(op, ZSTD_BLOCKHEADERSIZE)?; write_block_header(out, c_size, block_size, last_block)?; c_size + ZSTD_BLOCKHEADERSIZE } @@ -1838,7 +1880,7 @@ impl FrameCompressor { if dst.len() - op < 3 { return Err(Error::DstSizeTooSmall); } - write_le24(dst, op, 1 + ((BlockType::Raw as u32) << 1))?; + write_le24(dst, op, block_header(BlockType::Raw, 0, true))?; op += ZSTD_BLOCKHEADERSIZE; } diff --git a/src/zstd/compress/huf.rs b/src/zstd/compress/huf.rs index 7426d0152d30..5f77bbe8c076 100644 --- a/src/zstd/compress/huf.rs +++ b/src/zstd/compress/huf.rs @@ -4,7 +4,7 @@ use super::fse::{self, FseCTable}; use super::hist; use super::internal::Strategy; use crate::common::entropy::huf_read_stats; -use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; +use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX, LIT_HUF_LOG}; use crate::error::{Error, Result}; use crate::util::{BUG, SliceExt, highbit32, write_le16}; @@ -873,6 +873,12 @@ const fn tight_compress_bound(src_size: usize, table_log: usize) -> usize { ((src_size * table_log) >> 3) + 8 } +/// With that much room, the streams of `nb_literals` are written the quick way, whatever their code and be it one stream +/// or four. As they are, the literals take less. +pub(crate) const fn room_for_quick_loops(nb_literals: usize) -> usize { + 4 * tight_compress_bound(nb_literals.div_ceil(4), LIT_HUF_LOG as usize) +} + /// `HUF_compress1X_usingCTable`: the table has a code for every symbol in `src`. Returns the size of the output, or 0 /// if `dst` is too small. pub(crate) fn compress_1x_using_ctable( @@ -1020,6 +1026,24 @@ pub(crate) fn optimal_table_log( opt_log } +/// Whether `src` is long enough to tell by its start and its end, and they say that it does not compress. +pub(crate) fn samples_are_incompressible(src: &[u8]) -> Result { + let src_size = src.len(); + if src_size < SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO { + return Ok(false); + } + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + let mut largest_total = 0usize; + for sample in [ + src.sub(..SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE)?, + src.sub(src_size - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE..)?, + ] { + let mut max_symbol_value = HUF_SYMBOLVALUE_MAX; + largest_total += hist::count_simple(&mut count, &mut max_symbol_value, sample)? as usize; + } + Ok(largest_total <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7) + 4) +} + /// `HUF_compress_internal` fn compress_internal( dst: &mut [u8], @@ -1058,26 +1082,12 @@ fn compress_internal( return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); } - let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; - // If the input may well be incompressible, look at its start and its end before all of it. - if flags.suspect_uncompressible - && src_size >= SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO - { - let mut largest_total = 0usize; - for sample in [ - src.sub(..SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE)?, - src.sub(src_size - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE..)?, - ] { - let mut max_symbol_value_sample = max_symbol_value; - largest_total += - hist::count_simple(&mut count, &mut max_symbol_value_sample, sample)? as usize; - } - if largest_total <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7) + 4 { - return Ok(0); - } + if flags.suspect_uncompressible && samples_are_incompressible(src)? { + return Ok(0); } + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; let largest = hist::count_wksp(&mut count, &mut max_symbol_value, src)?; if largest == src_size { // A single symbol: run-length encoding. diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index abede0e08d45..b26de0d76e5d 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -445,9 +445,10 @@ impl SeqStore { Ok(()) } - /// Appends the literals that follow the last sequence of a block. + /// Appends the literals that follow the last sequence of `block`, `count` of them. #[inline] - pub(crate) fn store_last_literals(&mut self, literals: &[u8]) -> Result<()> { + pub(crate) fn store_last_literals(&mut self, block: &[u8], count: usize) -> Result<()> { + let literals = block.sub(block.len().checked_sub(count).ok_or(BUG)?..)?; self.lit_buffer.write_at(self.lit_len, literals)?; self.lit_len += literals.len(); Ok(()) @@ -632,16 +633,19 @@ pub(crate) fn ml_code(ml_base: u32) -> u32 { /// `ZSTD_noCompressBlock` pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { - let c_block_header24 = - u32::from(last_block) + ((BlockType::Raw as u32) << 1) + ((src.len() as u32) << 3); if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { return Err(Error::DstSizeTooSmall); } - write_le24(dst, 0, c_block_header24)?; + write_le24(dst, 0, block_header(BlockType::Raw, src.len(), last_block))?; dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; Ok(ZSTD_BLOCKHEADERSIZE + src.len()) } +/// The 3 bytes before a block. `size` is that of its content, or for a run what it stands for. +pub(crate) const fn block_header(block_type: BlockType, size: usize, last_block: bool) -> u32 { + last_block as u32 + ((block_type as u32) << 1) + ((size as u32) << 3) +} + /// `ZSTD_rleCompressBlock` pub(crate) fn rle_compress_block( dst: &mut [u8], @@ -649,12 +653,10 @@ pub(crate) fn rle_compress_block( src_size: usize, last_block: bool, ) -> Result { - let c_block_header = - u32::from(last_block) + ((BlockType::Rle as u32) << 1) + ((src_size as u32) << 3); if dst.len() < 4 { return Err(Error::DstSizeTooSmall); } - write_le24(dst, 0, c_block_header)?; + write_le24(dst, 0, block_header(BlockType::Rle, src_size, last_block))?; dst.put(3, byte)?; Ok(4) } diff --git a/src/zstd/compress/sequences.rs b/src/zstd/compress/sequences.rs index adb489da53e2..4b1f9131194c 100644 --- a/src/zstd/compress/sequences.rs +++ b/src/zstd/compress/sequences.rs @@ -802,6 +802,12 @@ fn encode_sequences_into( Some(stream.bit_pos + 1) } +/// `nb_seq` sequences and the descriptions of their three tables take no more. A sequence has up to 16, 16 and 31 bits, +/// and 9, 9 and 8 for the states. +pub(crate) const fn max_size(nb_seq: usize) -> usize { + 3 * FSE_NCOUNTBOUND + nb_seq * 12 +} + /// `ZSTD_encodeSequences`: `codes` are those of `sequences`, of which there is at least one. Without `many_bits`, none /// has too many extra bits to be written at once. pub(crate) fn encode_sequences( diff --git a/src/zstd/compress/superblock.rs b/src/zstd/compress/superblock.rs index 452677b8fe2b..7aed3dbac5cc 100644 --- a/src/zstd/compress/superblock.rs +++ b/src/zstd/compress/superblock.rs @@ -4,7 +4,7 @@ use super::block_entropy::{additional_bits_of, build_block_entropy_stats, write_ use super::fse::FseCTable; use super::internal::{ CompressedBlockState, EntropyCTables, EntropyCTablesMetadata, FseCTables, FseCTablesMetadata, - HufCTables, HufCTablesMetadata, SeqDef, SeqStore, no_compress_block, update_rep, + HufCTables, HufCTablesMetadata, SeqDef, SeqStore, block_header, no_compress_block, update_rep, }; use super::literals::{compress_rle_literals_block, no_compress_literals}; use super::params::CCtxParams; @@ -233,9 +233,11 @@ fn compress_sub_block( op += c_seq_size; } let c_size = op - ZSTD_BLOCKHEADERSIZE; - let c_block_header24 = - u32::from(last_block) + ((BlockType::Compressed as u32) << 1) + ((c_size as u32) << 3); - write_le24(dst, 0, c_block_header24)?; + write_le24( + dst, + 0, + block_header(BlockType::Compressed, c_size, last_block), + )?; result.c_size = op; Ok(result) } diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 0a657ea6d686..2e1a948d6277 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -519,6 +519,45 @@ mod tests { Ok(frame) } + #[test] + fn literals_that_start_and_end_like_noise_are_stored_whatever_the_room() { + // Long enough for the C library to judge literals without a match by their first and last 4 KB. + const LEN: usize = 41_000; + const STORED: usize = 4 + 1 + 2 + 3 + LEN; + let mut state = 13u32; + let src: Vec = (0..LEN) + .map(|i| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + let byte = (state >> 8) as u8; + if (4096..LEN - 4096).contains(&i) { + byte % 64 + } else { + byte + } + }) + .collect(); + let cases = [ + (STORED + 7, Ok(STORED)), + (STORED + 6, Ok(STORED)), + (STORED, Ok(STORED)), + (STORED - 1, Err(Error::DstSizeTooSmall)), + ]; + // Miri takes 15 seconds for a call. + for (capacity, expected) in cases.into_iter().take(if cfg!(miri) { 1 } else { 4 }) { + let mut dst = vec![0; capacity]; + assert_eq!(compress(&mut dst, &src, 1), expected); + let mut out = Vec::with_capacity(capacity); + assert_eq!(out.capacity(), capacity); + assert_eq!(compress_append(&mut out, &src, 1), expected); + if expected.is_ok() { + assert!(dst.get(..STORED) == Some(&*out)); + assert!(decompress_alloc(&out).as_ref() == Ok(&src)); + } + } + } + #[test] fn tables_that_change_size_give_the_same_frames() -> Result<()> { let mut state = 11u32; diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 7589bb5fe23b..352ef1325422 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1380,6 +1380,34 @@ describe("the output is that of zstd 1.5.7", () => { ]); }); + it("where blocks are nothing but literals", () => { + // The lesser of two bytes has no matches worth taking, and compresses. At negative levels literals are not compressed. + const data = new Uint8Array(300_000); + for (let i = 0, x = 5; i < data.length; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + data[i] = Math.min(x & 255, (x >> 8) & 255); + } + const inOneCall = [1, 3, 9, 16].map(level => digest(zstdCompressSync(data, { level }))); + const streamed = [-5, 1, 3, 9, 16].map(level => + digest(zlib.zstdCompressSync(data, { params: { [c.ZSTD_c_compressionLevel]: level } })), + ); + expect(inOneCall).toEqual([ + "290920 e4a05b68a9febcae", + "290984 c04074b7a3113a0c", + "290920 e4a05b68a9febcae", + "290920 e4a05b68a9febcae", + ]); + expect(streamed).toEqual([ + "300018 3aa98e9a8e545874", + "290920 e4a05b68a9febcae", + "290936 bdaff5fb067b174b", + "290920 e4a05b68a9febcae", + "290920 e4a05b68a9febcae", + ]); + }); + it("with parameters", () => { const cases: Record> = { checksum: { [c.ZSTD_c_checksumFlag]: 1 }, From 40b288f6f34d983aafcf1e6c6c6adee6c5625a8f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 19:07:51 +0000 Subject: [PATCH 158/199] zstd: a unit test does not depend on where its buffers are `Vec::reserve` may double the capacity. How long the `Vec` is at that point depends on how far into their pages two buffers start, so for some addresses there was room for what the test expects to be refused. Under Miri, where addresses differ from run to run, it failed about one time in eight. --- src/zstd/util.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/zstd/util.rs b/src/zstd/util.rs index 85c8a9a4441f..5a47ad5a6cb2 100644 --- a/src/zstd/util.rs +++ b/src/zstd/util.rs @@ -311,8 +311,9 @@ mod tests { to.fill(0); let mut vec = vec![0; start]; - vec.reserve(from.len()); - assert_eq!(append(&mut vec, to.sub(..from.len() + PAGE)?), Err(BUG)); + vec.reserve_exact(from.len()); + let too_long = vec.capacity() - vec.len() + 1; + assert_eq!(append(&mut vec, to.sub(..too_long)?), Err(BUG)); append(&mut vec, &from)?; assert!(*vec.sub(start..)? == *from); } From 4a881e2caf672a92d8f4fbad9929462f3be14356 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 20:57:30 +0000 Subject: [PATCH 159/199] zstd: long matches are counted 64 bytes at a time --- src/zstd/compress/internal.rs | 58 +++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index b26de0d76e5d..d0b5b9bd8ca7 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -820,6 +820,9 @@ impl<'a> Near<'a> { return len + differ.trailing_zeros(); } len += 16; + if len == 64 { + len = self.same_blocks(pos, other, other_pos, len); + } } while let Some(byte) = self.read8(pos + len) && other.read8(other_pos + len) == Some(byte) @@ -829,6 +832,30 @@ impl<'a> Near<'a> { len } + /// `len`, which is how many bytes are known to be the same at `pos` and at `other_pos` in `other`, plus the blocks of + /// 64 bytes after them that are the same too. For the matches that are long. + #[inline(never)] + fn same_blocks(self, pos: u32, other: Near<'_>, other_pos: u32, mut len: u32) -> u32 { + while let Some(block) = self.window::<64>(pos + len) + && let Some(other_block) = other.window::<64>(other_pos + len) + { + // A vector. + let mut differ = [0u8; 16]; + let (chunks, other_chunks) = + (block.as_chunks::<16>().0, other_block.as_chunks::<16>().0); + for (bytes, other_bytes) in chunks.iter().zip(other_chunks) { + for ((differ, byte), other_byte) in differ.iter_mut().zip(bytes).zip(other_bytes) { + *differ |= byte ^ other_byte; + } + } + if differ != [0; 16] { + break; + } + len += 64; + } + len + } + /// [`Self::count`] of positions that are known to have `from` bytes in common, and which one is before the other in /// the order of what follows them. Only the sum of `match_pos` and `from` has to be a position. #[inline(always)] @@ -2398,3 +2425,34 @@ impl LdmState { } bun_pod::impl_zeroable!(Match, RawSeq, LdmEntry); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn matches_are_counted_up_to_where_they_end() -> Result<()> { + const DISTANCE: u32 = 7; + let lengths: &[u32] = if cfg!(miri) { + &[15, 16, 63, 64, 65, 80, 127, 128, 129, 192, 193] + } else { + &Vec::from_iter(0..=400) + }; + for &len in lengths { + for room in [0, 1, 16, 64, 200] { + let mut bytes: Vec = (0..DISTANCE + len + room) + .map(|n| (n % DISTANCE) as u8) + .collect(); + if room != 0 { + *bytes.at_mut((DISTANCE + len) as usize)? ^= 1; + } + assert_eq!( + Near::new(&bytes)?.count(DISTANCE, 0), + len, + "{room} more bytes" + ); + } + } + Ok(()) + } +} From 29f26b72a36fef764c2dae4fa12696cf9cc572a3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 20:57:30 +0000 Subject: [PATCH 160/199] zstd: the optimal parsers go over what has no match in one loop The search for the first match of a series stays in one function from a position to the next, and a long run of literals has its weights computed once. --- src/zstd/compress/opt.rs | 191 ++++++++++++++++++++++++++++------ test/js/bun/util/zstd.test.ts | 56 ++++++++++ 2 files changed, 213 insertions(+), 34 deletions(-) diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index 19460d9b32a3..abefa5abf7cf 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -297,9 +297,7 @@ impl<'a> Prices<'a> { ml, }; if prices.compressed_literals { - for (weight_of, &freq) in prices.lit_weight.iter_mut().zip(&*prices.lit_freq) { - *weight_of = weight(freq, opt_level); - } + prices.weigh_literals(); } for code in 0..=MAX_LL { prices.price_lit_length(code)?; @@ -424,6 +422,14 @@ impl<'a> Prices<'a> { self.match_base.wrapping_add(price.unwrap_or(0)) as i32 } + /// Sets `lit_weight` for all of `lit_freq`. + #[inline(never)] + fn weigh_literals(&mut self) { + for (weight_of, &freq) in self.lit_weight.iter_mut().zip(&*self.lit_freq) { + *weight_of = weight(freq, self.opt_level); + } + } + /// `ZSTD_updateStats` #[inline(always)] fn update_stats(&mut self, literals: &[u8], off_base: u32, match_length: u32) -> Result<()> { @@ -437,11 +443,20 @@ impl<'a> Prices<'a> { let lit_length = literals.len() as u32; if self.compressed_literals { - for &literal in literals { - let literal = usize::from(literal); - let freq = self.lit_freq.at(literal)?.wrapping_add(ZSTD_LITFREQ_ADD); - self.lit_freq.put(literal, freq)?; - self.lit_weight.put(literal, weight(freq, self.opt_level))?; + if literals.len() > self.lit_weight.len() { + // There are fewer weights to compute than literals. + for &literal in literals { + let freq = self.lit_freq.at_mut(usize::from(literal))?; + *freq = freq.wrapping_add(ZSTD_LITFREQ_ADD); + } + self.weigh_literals(); + } else { + for &literal in literals { + let literal = usize::from(literal); + let freq = self.lit_freq.at(literal)?.wrapping_add(ZSTD_LITFREQ_ADD); + self.lit_freq.put(literal, freq)?; + self.lit_weight.put(literal, weight(freq, self.opt_level))?; + } } self.sums.lit = self .sums @@ -704,7 +719,7 @@ impl<'a> Trees<'a> { let min_match = self.min_match; let bytes = near.read64(ip).ok_or(BUG)?; - let mut dict_root = match self.dms { + let dict_root = match self.dms { Some(dms) => { let hasher = self.hasher.with_log(dms.ms.c_params.hash_log); DictTree::root(dms, hasher.hash(bytes))? @@ -788,46 +803,121 @@ impl<'a> Trees<'a> { } } - // The farthest position any match refers to, which detects repetitive patterns. - let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut found = Found::new(mnum, best_length, curr); + let nb_compares = self.tree_matches(matches, ip, bytes, match_low, &mut found)?; + if dict_root != 0 && nb_compares != 0 { + // A copy, for the one that the loop above works on to need no place in memory. + let mut with_dict = found; + self.search_dms( + matches, + ip, + dict_root, + window_low, + nb_compares, + &mut with_dict, + )?; + found = with_dict; + } + + // Repetitive patterns are skipped. + self.next_to_update = found.match_end_idx.wrapping_sub(8); + Ok(found.mnum) + } + + /// The part of `ZSTD_insertBtAndGetAllMatches` that adds the position `ip` of `near`, which starts with `bytes`, to + /// its tree, and stores the matches that are met on the way. Returns how many more positions can be compared in the + /// tree of a dictionary. + #[inline(always)] + fn tree_matches( + &mut self, + matches: &mut [Match], + ip: u32, + bytes: u64, + match_low: u32, + found: &mut Found, + ) -> Result { + let curr = ip.wrapping_add(self.input.dict_limit); + let mut is_over = false; let nb_compares = self.walk(ip, bytes, match_low, |match_index, common| { - if common.len > best_length { - if common.len > match_end_idx.wrapping_sub(match_index) { - match_end_idx = match_index.wrapping_add(common.len); + if common.len > found.best_length { + if common.len > found.match_end_idx.wrapping_sub(match_index) { + found.match_end_idx = match_index.wrapping_add(common.len); } - best_length = common.len; + found.best_length = common.len; matches.put( - mnum as usize, + found.mnum as usize, Match { off: offset_to_offbase(curr.wrapping_sub(match_index)), len: common.len, }, )?; - mnum += 1; + found.mnum += 1; if common.len > ZSTD_OPT_NUM as u32 || common.match_is_smaller.is_none() { // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. - // The dictionary is not to be searched either. - dict_root = 0; + is_over = true; return Ok(false); } } Ok(true) })?; + Ok(if is_over { 0 } else { nb_compares }) + } - if dict_root != 0 && nb_compares != 0 { - let mut found = Found { - mnum, - best_length, - match_end_idx, - }; - self.search_dms(matches, ip, dict_root, window_low, nb_compares, &mut found)?; - mnum = found.mnum; - match_end_idx = found.match_end_idx; + /// [`Self::get_all_matches`] for `ip`, which comes after literals, and then for the positions after it, before + /// `limit`, for as long as there is no match. It stops before a position that starts like what is at a repeat + /// offset, or where the table of 3-byte hashes leads: that one is left to [`Self::get_all_matches`]. Returns where + /// it stops, and how many matches it has found there. + #[inline(never)] + fn skip_matchless( + &mut self, + matches: &mut [Match], + mut ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + let near = self.input.near; + let dict_limit = self.input.dict_limit; + let min_match = self.min_match; + // No repeat offset leads before the current segment, from `ip` on. + if ip.wrapping_add(dict_limit) != self.next_to_update + || self.dms.is_some() + || rep.iter().any(|rep_offset| *rep_offset > ip) + { + return Ok((ip, 0)); } - // Repetitive patterns are skipped. - self.next_to_update = match_end_idx.wrapping_sub(8); - Ok(mnum) + while ip < limit + && let Some(bytes) = near.read64(ip) + { + let curr = ip.wrapping_add(dict_limit); + let match_low = self.lowest_match_index(curr).max(1); + let starts_alike = |pos: u32, min_match: u32| { + let there = near.read32(pos); + there.is_none_or(|there| minmatch_differs(there, bytes as u32, min_match) == 0) + }; + if (rep.iter()).any(|rep_offset| starts_alike(ip - rep_offset, min_match)) { + break; + } + if min_match == 3 { + let match_index3 = self.insert_and_find_first_index_hash3(curr, bytes as u32)?; + if match_index3 >= match_low + && curr.wrapping_sub(match_index3) < (1 << 18) + && (match_index3 < dict_limit || starts_alike(match_index3 - dict_limit, 3)) + { + break; + } + } + + let mut found = Found::new(0, min_match - 1, curr); + self.tree_matches(matches, ip, bytes, match_low, &mut found)?; + if found.mnum != 0 { + self.next_to_update = found.match_end_idx.wrapping_sub(8); + return Ok((ip, found.mnum)); + } + ip += 1; + } + self.next_to_update = ip.wrapping_add(dict_limit); + Ok((ip, 0)) } /// The end of `ZSTD_insertBtAndGetAllMatches`: goes on in the tree of the dictionary whose root is `root`, for @@ -877,12 +967,26 @@ fn count(near: Near<'_>, pos: u32, match_pos: u32) -> u32 { } /// What a search has found so far. +#[derive(Clone, Copy)] struct Found { mnum: u32, best_length: u32, + /// The farthest position any match refers to, which detects repetitive patterns. match_end_idx: u32, } +impl Found { + /// What the search for `curr` has before it gets to the trees. + #[inline(always)] + const fn new(mnum: u32, best_length: u32, curr: u32) -> Self { + Self { + mnum, + best_length, + match_end_idx: curr.wrapping_add(8 + 1), + } + } +} + /// `ZSTD_readMINMATCH`: 0 if the first `length` bytes, 3 or 4, of the 4 in `a` and in `b` are the same. #[inline(always)] const fn minmatch_differs(a: u32, b: u32, length: u32) -> u32 { @@ -1053,6 +1157,12 @@ impl<'a> OptLdm<'a> { Ok(()) } + /// Whether there is no candidate left. + #[inline(always)] + const fn is_over(&self) -> bool { + self.pos >= self.seq.len() + } + /// `ZSTD_optLdm_processMatchCandidate` #[inline(always)] fn process_match_candidate( @@ -1063,7 +1173,7 @@ impl<'a> OptLdm<'a> { remaining_bytes: u32, min_match: u32, ) -> Result<()> { - if self.pos >= self.seq.len() { + if self.is_over() { return Ok(()); } @@ -1227,9 +1337,22 @@ fn parse_block( let mut cur = 'forward: { // Find a first match. { + let mut nb_matches = 0; + if ip != anchor && opt_ldm.is_over() { + let first = (ip - prefix_start) as u32; + let limit = (ilimit - prefix_start) as u32; + let (stop, found) = trees.skip_matchless(matches, first, limit, rep)?; + ip += (stop - first) as usize; + if ip >= ilimit { + break 'series; + } + nb_matches = found; + } let litlen = (ip - anchor) as u32; - let mut nb_matches = - trees.get_all_matches(matches, (ip - prefix_start) as u32, rep, litlen == 0)?; + if nb_matches == 0 { + let pos = (ip - prefix_start) as u32; + nb_matches = trees.get_all_matches(matches, pos, rep, litlen == 0)?; + } opt_ldm.process_match_candidate( matches, &mut nb_matches, diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 352ef1325422..ebb62b332e5b 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1408,6 +1408,62 @@ describe("the output is that of zstd 1.5.7", () => { ]); }); + it("where matches are far between, for the optimal parsers", () => { + const data = Buffer.alloc(100_000); + let x = 9; + const next = () => { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + return x >>> 0; + }; + const copy = (at: number, distance: number, length: number) => { + const end = Math.min(at + length, data.length); + while (at < end) data[at] = data[at++ - distance]; + return end; + }; + for (let at = 0, distance = 1; at < data.length;) { + if (at < 100 || next() % 300 !== 0) { + data[at++] = next() >> 8; + continue; + } + // From as far as the one before, from anywhere, or from one byte less far, which only counts right after a match. + const kind = next() % 4; + if (kind === 1 && distance > 1) distance--; + if (kind > 1) distance = 1 + (next() % Math.min(at, 1 << (3 + (next() % 15)))); + at = copy(at, distance, next() % 8 === 0 ? 300 : 3 + (next() % 12)); + if (next() % 2 === 0 && distance > 1) at = copy(at, distance - 1, 3 + (next() % 5)); + } + const smallTables = { [c.ZSTD_c_minMatch]: 4, [c.ZSTD_c_hashLog]: 8, [c.ZSTD_c_chainLog]: 8 }; + const cases: [Buffer | Uint8Array, Record][] = [ + [data, { [c.ZSTD_c_strategy]: c.ZSTD_btopt, [c.ZSTD_c_minMatch]: 3 }], + [data, { [c.ZSTD_c_compressionLevel]: 17 }], + [data, { [c.ZSTD_c_compressionLevel]: 19 }], + [data, { [c.ZSTD_c_strategy]: c.ZSTD_btultra, [c.ZSTD_c_minMatch]: 4 }], + [data, { [c.ZSTD_c_compressionLevel]: 19, [c.ZSTD_c_windowLog]: 12 }], + [data, { [c.ZSTD_c_compressionLevel]: 19, ...smallTables }], + [data, { [c.ZSTD_c_compressionLevel]: 19, ...smallTables, [c.ZSTD_c_enableLongDistanceMatching]: 1 }], + [large, { [c.ZSTD_c_strategy]: c.ZSTD_btopt, [c.ZSTD_c_windowLog]: 12 }], + ]; + expect(cases.map(([data, params]) => digest(zlib.zstdCompressSync(data, { params })))).toEqual([ + "89890 2e527bd77ff0248a", + "89813 167bd065007f9d2b", + "89828 46d69a88706923f0", + "89863 b3219cbfd91243e7", + "92097 87e40b8681136738", + "92667 ff08869afb33cf7b", + "90656 95d8a11469d222d9", + "57217 69a87c5b3e0a2b75", + ]); + const dictionary = data.subarray(0, 20_000); + const withDictionary = [3_000, 80_000].map(size => { + const params = { [c.ZSTD_c_compressionLevel]: 19 }; + const options = { dictionary, params, pledgedSrcSize: size }; + return digest(zlib.zstdCompressSync(data.subarray(20_000, 20_000 + size), options)); + }); + expect(withDictionary).toEqual(["2964 61e43e2a0511038c", "71053 b34a7f9afac77810"]); + }); + it("with parameters", () => { const cases: Record> = { checksum: { [c.ZSTD_c_checksumFlag]: 1 }, From 72f95e5f7a0b883331bfc216b8a1333ff2cc17db Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 23:46:38 +0000 Subject: [PATCH 161/199] zstd: long copies to the end of a large new buffer are made in parts glibc copies from 2 KB to about 1 MB with `rep movsb`, which is slow to write to pages for the first time. What is appended to a `Vec` with room for 512 KB or more, as the output of `decompress_alloc` is, now goes there 2 KB at a time. With the output mapped anew for every call, 8 MB at level 9, MB/s (C, before, after): noise 1772, 1772, 1966; a period of 100 KB 2016, 1731, 1963; a period of 1 KB 1965, 1851, 1926. --- src/zstd/util.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/zstd/util.rs b/src/zstd/util.rs index 5a47ad5a6cb2..9995e78537e7 100644 --- a/src/zstd/util.rs +++ b/src/zstd/util.rs @@ -141,6 +141,20 @@ fn is_slow_to_copy(distance: usize, size: usize) -> bool { && distance.wrapping_sub(1) % 4096 < 127 } +/// `rep movsb` is also slow to write to pages for the first time, as an append to what is new from the system does. +const NEW_FROM_SYSTEM: usize = 1 << 19; +/// From there on glibc copies in another way on some processors, faster than in parts. +const REP_MOVSB_END: usize = 1 << 20; + +/// [`is_slow_to_copy`], to the end of `to`. +#[inline(always)] +fn is_slow_to_append(to: &Vec, distance: usize, size: usize) -> bool { + is_slow_to_copy(distance, size) + || cfg!(all(target_arch = "x86_64", target_env = "gnu")) + && (MAX_COPY + 1..REP_MOVSB_END).contains(&size) + && to.capacity() >= NEW_FROM_SYSTEM +} + #[cold] #[inline(never)] fn copy_in_parts(to: &mut [T], from: &[T]) { @@ -168,7 +182,7 @@ pub(crate) fn append(to: &mut Vec, from: &[u8]) -> Result<()> { return Err(BUG); } let distance = (to.as_ptr_range().end.addr()).wrapping_sub(from.as_ptr().addr()); - if is_slow_to_copy(distance, from.len()) { + if is_slow_to_append(to, distance, from.len()) { from.chunks(MAX_COPY) .for_each(|part| to.extend_from_slice(part)); } else { @@ -184,7 +198,7 @@ pub(crate) fn append_within(to: &mut Vec, start: usize, len: usize) -> Resul if end > to.len() || len > to.capacity() - to.len() { return Err(BUG); } - if is_slow_to_copy(to.len() - start, len) { + if is_slow_to_append(to, to.len() - start, len) { (start..end) .step_by(MAX_COPY) .for_each(|part| to.extend_from_within(part..(part + MAX_COPY).min(end))); @@ -331,6 +345,12 @@ mod tests { within.clear(); (0..3).for_each(|_| within.extend_from_slice(&from)); } + + // To what is taken to be new from the system, whatever the distance. + let mut vec = Vec::with_capacity(NEW_FROM_SYSTEM); + append(&mut vec, &from)?; + append_within(&mut vec, 1, len)?; + assert!(*vec.sub(..from.len())? == *from && vec.sub(from.len()..)? == from.span(1, len)?); Ok(()) } } From d574f50166a9bda62dd284563a1091124a919ec4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 23:46:38 +0000 Subject: [PATCH 162/199] zstd: short matches from less than 16 bytes back have a loop of their own The loop that executes most sequences copies 16 bytes at a time, so it left every match from closer than that to the careful path. Where a few of them follow each other, a second instance of the loop now takes them too, 8 bytes at a time. `exec_batch` is one function for what was three, and the loop that alternates between it and `exec_others` is `exec_in_turns`. Decoding 128 KB of small integers compressed at level 19 goes from 1.17 to 0.97 of the time that C takes. Other data pays for the test in `exec_others`: from 0.5% (JSON, text) to 2.5% (sparse data at level 19). --- src/zstd/decompress/block.rs | 223 +++++++++++++++++++++++++--------- test/js/bun/util/zstd.test.ts | 26 ++++ 2 files changed, 190 insertions(+), 59 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 17a456e9bfbb..76303bb02b1d 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -1237,19 +1237,82 @@ const NO_EXT: u8 = 0; const EXT_DICT: u8 = 1; const EXT_WITHIN: u8 = 2; -/// Copies 16 bytes within `buf`, between positions that do not overflow. Whether they are in `buf`. +/// Copies `N` bytes within `buf`, between positions that do not overflow. Whether they are in `buf`. #[inline(always)] -fn copy_16(buf: &mut [u8], src: usize, dst: usize) -> bool { - let Some(Ok(chunk)) = buf.get(src..src + 16).map(<[u8; 16]>::try_from) else { +fn copy_chunk(buf: &mut [u8], src: usize, dst: usize) -> bool { + let Some(Ok(chunk)) = buf.get(src..src + N).map(<[u8; N]>::try_from) else { return false; }; - let Some(to) = buf.get_mut(dst..dst + 16) else { + let Some(to) = buf.get_mut(dst..dst + N) else { return false; }; to.copy_from_slice(&chunk); true } +/// The least offset value of a match that can be copied 16 bytes at a time. +const MIN_FAR_OFFSET_VALUE: u32 = 16 + 3; +/// How many literals a sequence can have for [`Executor::exec_batch`]. +const MAX_BATCH_LITERALS: u32 = 32; +/// How long a match from closer can be for [`copy_short_close_match`]. +const MAX_CLOSE_MATCH: u32 = 32; + +/// Whether `sequences[index]` and the few after it look like they are for [`Executor::exec_batch`] with `CLOSE`. +#[inline(always)] +fn starts_close_run(sequences: &[RawSequence], index: usize) -> bool { + let Some(run) = sequences.get(index..index + 4) else { + return false; + }; + (run.first()).is_some_and(|first| first.offset_value > 3) + && run.iter().all(|next| { + next.offset_value < MIN_FAR_OFFSET_VALUE + && next.lit_length <= MAX_BATCH_LITERALS + && next.match_length + MINMATCH <= MAX_CLOSE_MATCH + }) +} + +/// [`Executor::copy_close_match`] for what is short: `offset` is below 16, and up to 7 bytes more than `len` are written. +/// Whether all that is in `buf`. +#[inline(always)] +fn copy_short_close_match(buf: &mut [u8], dst: usize, offset: usize, len: usize) -> bool { + /// For each offset below 8: what leaves as many bytes, what repeats them, and how many of the 8 are whole periods. + const REPEAT: [(u64, u64, usize); 8] = [ + (0, 0, 8), + (0xFF, 0x0101_0101_0101_0101, 8), + (0xFFFF, 0x0001_0001_0001_0001, 8), + (0xFF_FFFF, 0x0001_0000_0100_0001, 6), + (0xFFFF_FFFF, 0x0000_0001_0000_0001, 8), + (0xFF_FFFF_FFFF, 1 | 1 << 40, 5), + (0xFFFF_FFFF_FFFF, 1 | 1 << 48, 6), + (0xFF_FFFF_FFFF_FFFF, 1 | 1 << 56, 7), + ]; + let Some(src) = dst.checked_sub(offset).filter(|_| offset != 0) else { + return false; + }; + let mut done = 0; + if let Some(&(mask, repeat, step)) = REPEAT.get(offset) { + let Some(Ok(bytes)) = buf.get(src..src + 8).map(<[u8; 8]>::try_from) else { + return false; + }; + let pattern = (u64::from_le_bytes(bytes) & mask).wrapping_mul(repeat); + while done < len { + let Some(to) = buf.get_mut(dst + done..dst + done + 8) else { + return false; + }; + to.copy_from_slice(&pattern.to_le_bytes()); + done += step; + } + } else { + while done < len { + if !copy_chunk::<8>(buf, src + done, dst + done) { + return false; + } + done += 8; + } + } + true +} + /// `buf[dst + i] = buf[src + i]`, one after the other, so that a match repeats itself where it overlaps its copy. /// `src` is less than `dst`. #[cold] @@ -1576,37 +1639,11 @@ impl<'a> Executor<'_, 'a> { /// it is 0 if that is not valid. /// /// The last offsets are those of the 3 sequences before, which are rewritten for that where it takes it. + /// + /// Matches in what comes before the output are of the common kind if `EXT` tells that it is `dict`, or `within` of the + /// buffer that the output goes to. So are, with `CLOSE`, short matches from less than 16 bytes back. #[inline(never)] - fn exec_batch(&mut self, sequences: &mut [RawSequence], index: usize) -> usize { - self.exec_batch_with::(sequences, index, &[], 0..0, 0) - } - - /// The same where matches in `dict`, which comes before the output, are of the common kind too. - #[inline(never)] - fn exec_batch_after_dict( - &mut self, - sequences: &mut [RawSequence], - index: usize, - dict: &[u8], - ) -> usize { - self.exec_batch_with::(sequences, index, dict, 0..0, 0) - } - - /// The same for `within` of the buffer that the output goes to. - #[inline(never)] - fn exec_batch_after_wrap( - &mut self, - sequences: &mut [RawSequence], - index: usize, - within: Range, - window: usize, - ) -> usize { - self.exec_batch_with::(sequences, index, &[], within, window) - } - - /// What comes before the output is nothing of the common kind, `dict` or `within`, as `EXT` tells. - #[inline(always)] - fn exec_batch_with( + fn exec_batch( &mut self, sequences: &mut [RawSequence], mut index: usize, @@ -1633,11 +1670,13 @@ impl<'a> Executor<'_, 'a> { let lit_length = raw.lit_length as usize; let match_length = raw.match_length as usize + MINMATCH as usize; let mut offset_value = raw.offset_value; - if offset_value < 16 + 3 { + if offset_value < MIN_FAR_OFFSET_VALUE { if offset_value <= 3 { offset_value = resolve_offset(sequences, index); } - if offset_value < 16 + 3 { + if offset_value < MIN_FAR_OFFSET_VALUE + && !(CLOSE && offset_value > 3 && match_length <= MAX_CLOSE_MATCH as usize) + { break; } } @@ -1647,8 +1686,9 @@ impl<'a> Executor<'_, 'a> { let lit_end = lit_pos as usize + lit_length; if o_match_end + WILDCOPY_OVERLENGTH > buf.len() || lit_end + WILDCOPY_OVERLENGTH > lits.len() - || lit_length > 32 - || (EXT == NO_EXT || match_length > 64) && offset_value as usize > o_lit_end + 3 + || lit_length > MAX_BATCH_LITERALS as usize + || (EXT == NO_EXT || match_length > 64 || offset_value < MIN_FAR_OFFSET_VALUE) + && offset_value as usize > o_lit_end + 3 { break; } @@ -1658,6 +1698,17 @@ impl<'a> Executor<'_, 'a> { ) else { break; }; + if CLOSE && offset_value < MIN_FAR_OFFSET_VALUE { + to.copy_from_slice(from); + let offset = offset_value as usize - 3; + if !copy_short_close_match(buf, o_lit_end, offset, match_length) { + break; + } + op = o_match_end as u32; + lit_pos = lit_end as u32; + index += 1; + continue; + } if EXT == EXT_DICT && offset_value as usize > o_lit_end + 3 { let back = offset_value as usize - 3 - o_lit_end; let Some(match_pos) = dict.len().checked_sub(back) else { @@ -1699,11 +1750,11 @@ impl<'a> Executor<'_, 'a> { let is_copied = if match_length > 64 { copy_match_exact(buf, match_pos, o_lit_end, match_length).is_ok() } else { - copy_16(buf, match_pos, o_lit_end) - && copy_16(buf, match_pos + 16, o_lit_end + 16) + copy_chunk::<16>(buf, match_pos, o_lit_end) + && copy_chunk::<16>(buf, match_pos + 16, o_lit_end + 16) && (match_length <= 32 - || copy_16(buf, match_pos + 32, o_lit_end + 32) - && copy_16(buf, match_pos + 48, o_lit_end + 48)) + || copy_chunk::<16>(buf, match_pos + 32, o_lit_end + 32) + && copy_chunk::<16>(buf, match_pos + 48, o_lit_end + 48)) }; if !is_copied { break; @@ -1720,7 +1771,7 @@ impl<'a> Executor<'_, 'a> { } /// Executes `sequences[index]` and those after it that are not for [`Self::exec_batch`]. Returns the index of the - /// first that is. + /// first that is, or that [`starts_close_run`]. #[inline(never)] fn exec_others(&mut self, sequences: &mut [RawSequence], first: usize) -> Result { let mut exec = self.fork(); @@ -1730,7 +1781,10 @@ impl<'a> Executor<'_, 'a> { if (1..=3).contains(&offset_value) { offset_value = resolve_offset(sequences, index); } - if index != first && raw.lit_length <= 32 && offset_value >= 16 + 3 { + if index != first + && raw.lit_length <= MAX_BATCH_LITERALS + && (offset_value >= MIN_FAR_OFFSET_VALUE || starts_close_run(sequences, index)) + { break; } exec.exec_sequence::(Sequence { @@ -1875,28 +1929,47 @@ impl<'a> Executor<'_, 'a> { for (last, offset) in sequences.iter_mut().zip(prev_offset.iter().rev()) { last.offset_value = *offset as u32 + 3; } - let mut index = 3; - loop { - index = match self.ext_dict { - ExtDict::Dict(dict) if !dict.is_empty() => { - self.exec_batch_after_dict(sequences, index, dict) - } - ExtDict::Within { end, len, window } if len != 0 => { - self.exec_batch_after_wrap(sequences, index, end - len..end, window) - } - _ => self.exec_batch(sequences, index), - }; - if index >= sequences.len() { - break; + match self.ext_dict { + ExtDict::Dict(dict) if !dict.is_empty() => { + self.exec_in_turns::(sequences, dict, 0..0, 0) } - index = self.exec_others(sequences, index)?; - } + ExtDict::Within { end, len, window } if len != 0 => { + self.exec_in_turns::(sequences, &[], end - len..end, window) + } + _ => self.exec_in_turns::(sequences, &[], 0..0, 0), + }?; for (last, offset) in sequences.iter().rev().zip(prev_offset) { *offset = last.offset_value as usize - 3; } Ok(()) } + /// Executes `sequences` but the first 3. What else it gets is for [`Self::exec_batch`]. + #[inline(always)] + fn exec_in_turns( + &mut self, + sequences: &mut [RawSequence], + dict: &[u8], + within: Range, + window: usize, + ) -> Result<()> { + let mut index = 3; + let mut close = false; + while index < sequences.len() { + index = if close { + self.exec_batch::(sequences, index, dict, within.clone(), window) + } else { + self.exec_batch::(sequences, index, dict, within.clone(), window) + }; + if index < sequences.len() { + index = self.exec_others(sequences, index)?; + close = (sequences.get(index)) + .is_some_and(|next| next.offset_value < MIN_FAR_OFFSET_VALUE); + } + } + Ok(()) + } + /// The literals after the last sequence. fn last_literals(&mut self) -> Result<()> { let extra = if self.split { self.lits_extra } else { &[] }; @@ -1917,3 +1990,35 @@ impl<'a> Executor<'_, 'a> { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn short_close_matches_repeat_what_is_before_them() -> Result<()> { + const DST: usize = 20; + let start: Vec = (0..DST as u8).collect(); + for offset in 1..16 { + for len in 1..=MAX_CLOSE_MATCH as usize { + let mut expected = start.clone(); + for i in 0..len { + expected.push(expected.at(DST - offset + i)?); + } + let mut buf = [start.as_slice(), &[0xEE; MAX_CLOSE_MATCH as usize + 8]].concat(); + assert!(copy_short_close_match(&mut buf, DST, offset, len)); + assert_eq!( + *buf.sub(..DST + len)?, + *expected, + "{len} bytes from {offset} back" + ); + assert!(buf.sub(DST + len + 7..)?.iter().all(|&byte| byte == 0xEE)); + let too_short = buf.sub_mut(..DST + len - 1)?; + assert!(!copy_short_close_match(too_short, DST, offset, len)); + } + } + assert!(!copy_short_close_match(&mut [0; 64], 3, 4, 5)); + assert!(!copy_short_close_match(&mut [0; 64], 3, 0, 5)); + Ok(()) + } +} diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index ebb62b332e5b..bc47d1ed4888 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -472,6 +472,32 @@ describe("decompressing frames whose size is known", () => { }, ); + it.each([1, 2, 8])("short matches from less than 16 bytes back, in one sequence out of %i", others => { + const data = noise(60_000); + // Where the first 24 bytes are a dictionary, these two reach into the part of it that matches are looked for in. + data.copyWithin(24, 9, 13); + data.copyWithin(29, 14, 18); + for (let at = 33, i = 0; at < data.length - 40; i++) { + at += 1 + (i % 7); + const offset = i % others === 0 || at < 320 ? 15 - (i % 15) : 16 + (i % 300); + for (const end = at + 3 + ((i + 2) % 31); at < end; at++) data[at] = data[at - offset]; + } + const frame = zstdCompressSync(data, { level: 19 }); + expect(blocks(frame)[0]).toBeGreaterThan(2_000); + expect(zstdDecompressSync(frame).equals(data)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + + // After a dictionary, and in a buffer that is written to from its start again. + const params = { [zlib.constants.ZSTD_c_compressionLevel]: 19, [zlib.constants.ZSTD_c_windowLog]: 10 }; + const dictionary = data.subarray(0, 24); + const afterDictionary = zlib.zstdCompressSync(data.subarray(24), { params, dictionary }); + expect(blocks(afterDictionary)[0]).toBeGreaterThan(32); + expect(zlib.zstdDecompressSync(afterDictionary, { dictionary }).equals(data.subarray(24))).toBe(true); + const inSmallWindow = zlib.zstdCompressSync(data, { params }); + expect(blocks(inSmallWindow)[1]).toBeGreaterThan(32); + expect(zlib.zstdDecompressSync(inSmallWindow).equals(data)).toBe(true); + }); + it("a frame that makes more than it says is refused", () => { for (const piece of pieces.filter(piece => piece.length >= 135_000)) { const frame = zstdCompressSync(piece.subarray(0, 135_000), { level: 1 }); From f629239bbf95ee587c81c73d5a6589c5c616cf31 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 23:46:38 +0000 Subject: [PATCH 163/199] zstd: a stream that has failed stays failed, and is not mixed with whole frames What `ZSTD_decompressStream` does after it has returned an error, or after `ZSTD_decompressDCtx` on the same context in the middle of a frame, is left undefined by the C library: it goes on with sizes that no longer belong together. Here these were subtractions that overflow. Now an error that a frame causes is returned again until `init_stream`, and the functions that take whole frames refuse with `StageWrong` while a stream is under way, as `load_dictionary` and `set_parameter` do. Nothing in Bun does either. --- src/zstd/decompress/mod.rs | 53 +++++++++++++++++++++++++---- src/zstd/lib.rs | 70 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index d12dda6ccfc9..c6677aa40077 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -484,6 +484,8 @@ enum DStreamStage { Read, Load, Flush, + /// The frame is given up. The C library leaves what a call does after one that has failed undefined. + Failed(Error), } /// What precedes `prefix_start`. @@ -622,11 +624,12 @@ impl DCtx { } /// `ZSTD_decompressDCtx`: decompresses all the frames of `src`, which has to end with one, to the start of `dst`. - /// Returns how many bytes that is. + /// Returns how many bytes that is. As the other functions that take whole frames, it refuses with `StageWrong` from + /// the first call of [`Self::decompress_stream`] for a frame until that frame ends or [`Self::init_stream`]. pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { let (size, dst) = (dst.len(), &mut OutputBuffer::Whole(dst)); self.state - .decompress_multi_frame(dst, size, 0, src, self.ddict.as_deref()) + .decompress_frames(dst, size, src, self.ddict.as_deref()) } /// The same to a new buffer of `capacity` bytes, little more of which is touched than is written. With `ddict`, if @@ -641,9 +644,7 @@ impl DCtx { dst.try_reserve_exact(capacity) .map_err(|_| Error::MemoryAllocation)?; let used = &mut OutputBuffer::Used(&mut dst); - let size = self - .state - .decompress_multi_frame(used, capacity, 0, src, ddict)?; + let size = self.state.decompress_frames(used, capacity, src, ddict)?; dst.truncate(size); Ok(dst) } @@ -656,8 +657,7 @@ impl DCtx { ddict: &DDict<'_>, ) -> Result { let (size, dst) = (dst.len(), &mut OutputBuffer::Whole(dst)); - self.state - .decompress_multi_frame(dst, size, 0, src, Some(ddict)) + self.state.decompress_frames(dst, size, src, Some(ddict)) } /// `ZSTD_DCtx_loadDictionary`: `dict` is copied, and used for every frame from now on. None if it is empty. @@ -702,6 +702,7 @@ impl DCtx { /// `ZSTD_decompressStream`: reads from `input` and writes to `output` as much as it can, and moves their positions. /// Returns 0 when a frame is complete and all of it is written, else a hint of how much input to give next. + /// Once a frame has made it fail, it fails in the same way until [`Self::init_stream`]. pub fn decompress_stream( &mut self, output: &mut OutBuffer<'_>, @@ -929,6 +930,21 @@ impl DState { Ok((op - ostart, ip)) } + /// [`Self::decompress_multi_frame`] to the start of `dst`, unless a stream is under way: the C library then goes on + /// with that stream, with what this leaves. + fn decompress_frames( + &mut self, + dst: &mut OutputBuffer<'_>, + dst_size: usize, + src: &[u8], + ddict: Option<&DDict<'_>>, + ) -> Result { + if self.stream_stage != DStreamStage::Init { + return Err(Error::StageWrong); + } + self.decompress_multi_frame(dst, dst_size, 0, src, ddict) + } + /// `ZSTD_decompressMultiFrame`: all the frames of `src`, to `dst` from `dst_start` on. fn decompress_multi_frame( &mut self, @@ -1178,6 +1194,29 @@ impl DState { if output.pos > output.dst.len() { return Err(Error::DstSizeTooSmall); } + self.decompress_stream_on(output, input, ddict) + .inspect_err(|&error| { + // These leave all as it was. + let can_go_on = matches!( + error, + Error::NoForwardProgressDestFull | Error::NoForwardProgressInputEmpty + ); + if !can_go_on { + self.stream_stage = DStreamStage::Failed(error); + } + }) + } + + /// [`Self::decompress_stream`], but for what becomes of the stream if it fails. + fn decompress_stream_on( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + ddict: Option<&DDict<'_>>, + ) -> Result { + if let DStreamStage::Failed(error) = self.stream_stage { + return Err(error); + } let src = input.src; let istart = input.pos; let iend = src.len(); diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 2e1a948d6277..05265a18b51d 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -448,6 +448,76 @@ mod tests { use super::*; use util::SliceExt; + #[test] + fn a_stream_that_has_failed_goes_on_failing() { + // 5 bytes are announced, and the last block has 3. + let frame = [ + 0x28, 0xB5, 0x2F, 0xFD, 0x20, 5, 0x19, 0, 0, b'a', b'b', b'c', + ]; + let mut stream = DCtx::new(); + let mut decompress = |src: &[u8]| { + let mut dst = [0; 64]; + let (mut output, mut input) = ( + OutBuffer { + dst: &mut dst, + pos: 0, + }, + InBuffer { src, pos: 0 }, + ); + stream.decompress_stream(&mut output, &mut input) + }; + let (start, end) = frame.split_at(9); + assert_eq!(decompress(start), Ok(3)); + assert_eq!(decompress(end), Err(Error::CorruptionDetected)); + for more in [&b"x"[..], b"", &frame] { + assert_eq!(decompress(more), Err(Error::CorruptionDetected)); + } + stream.init_stream(); + let (mut dst, src) = ([0; 64], [&frame[..5], &[3], &frame[6..]].concat()); + let (mut output, mut input) = ( + OutBuffer { + dst: &mut dst, + pos: 0, + }, + InBuffer { src: &src, pos: 0 }, + ); + assert_eq!(stream.decompress_stream(&mut output, &mut input), Ok(0)); + assert_eq!(output.pos, 3); + } + + #[test] + fn a_stream_under_way_goes_on_after_what_does_not_concern_its_frame() -> Result<()> { + let data = Vec::from_iter((0..300u32).map(|i| (i * i % 7) as u8)); + let frame = compress_alloc(&data, 3)?; + let mut dst = [0; 400]; + let mut stream = DCtx::new(); + let (start, end) = frame.split_at(frame.len() - 5); + let (mut output, mut input) = ( + OutBuffer { + dst: &mut dst, + pos: 0, + }, + InBuffer { src: start, pos: 0 }, + ); + assert_eq!(stream.decompress_stream(&mut output, &mut input), Ok(5)); + assert_eq!( + stream.decompress(&mut [0; 400], &frame), + Err(Error::StageWrong) + ); + let mut no_input = InBuffer { src: &[], pos: 0 }; + for _ in 0..15 { + assert_eq!(stream.decompress_stream(&mut output, &mut no_input), Ok(5)); + } + let result = stream.decompress_stream(&mut output, &mut no_input); + assert_eq!(result, Err(Error::NoForwardProgressInputEmpty)); + let mut input = InBuffer { src: end, pos: 0 }; + assert_eq!(stream.decompress_stream(&mut output, &mut input), Ok(0)); + assert_eq!(output.pos, data.len()); + assert_eq!(*dst.sub(..data.len())?, *data); + assert_eq!(stream.decompress(&mut dst, &frame), Ok(data.len())); + Ok(()) + } + #[test] fn little_more_is_zeroed_than_is_written() { const STEP: usize = 16; From a88148b930c7a156c73d6de55458379b23e6e43b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 00:49:13 +0000 Subject: [PATCH 164/199] zstd: what starts a new job resets a stream, and a compressor that has failed stays failed `zstd.h` says that a context is reset by what starts a new decompression job, `ZSTD_decompressDCtx` among them, and that is a way to use a context again whose stream has failed. So the functions that take whole frames no longer refuse while a stream is under way, which was new in the last commit: they give that stream up. The C library does not, and goes on with it. It also says that `ZSTD_compressStream2` after an error is undefined until the context is reset. Without workers the port went on with the frame as C does: later calls succeeded, and in the end one failed with the code for a bug. With workers both started a new frame, without what the failed one had consumed. Now the error is returned again ever after, and the workers are let go at once: nothing in Bun uses such a context again. A frame that could not start can still be tried again, as in C. --- src/zstd/compress/stream.rs | 30 ++++++++- src/zstd/decompress/mod.rs | 23 ++++--- src/zstd/lib.rs | 124 +++++++++++++++++++++--------------- 3 files changed, 113 insertions(+), 64 deletions(-) diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs index 60f163342dbd..8cdd0ed81afc 100644 --- a/src/zstd/compress/stream.rs +++ b/src/zstd/compress/stream.rs @@ -31,6 +31,9 @@ enum StreamStage { Init, Load, Flush, + /// The frame is given up. The C library then starts another if it has workers and else goes on with this one, and + /// `zstd.h` calls a call after one that has failed undefined behavior. + Failed(Error), } /// `ZSTD_CCtx` @@ -282,6 +285,7 @@ impl CCtx { loop { match self.stream_stage { StreamStage::Init => return Err(Error::InitMissing), + StreamStage::Failed(error) => return Err(error), StreamStage::Load => { if flush_mode == EndDirective::End @@ -413,7 +417,8 @@ impl CCtx { } /// `ZSTD_compressStream2`: returns the least number of bytes that are left to flush, which for - /// [`EndDirective::End`] is 0 once the frame is complete. + /// [`EndDirective::End`] is 0 once the frame is complete. Once a frame has made it fail, it fails in the same way + /// ever after. pub fn compress_stream2( &mut self, output: &mut OutBuffer<'_>, @@ -426,7 +431,26 @@ impl CCtx { if input.pos > input.src.len() { return Err(Error::SrcSizeWrong); } + self.compress_stream2_on(output, input, end_op) + .inspect_err(|&error| { + // A frame that could not start can be tried again, as with the C library. + if self.stream_stage != StreamStage::Init { + self.stream_stage = StreamStage::Failed(error); + self.mtctx = None; + } + }) + } + /// [`Self::compress_stream2`], but for what becomes of the stream if it fails. + fn compress_stream2_on( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + end_op: EndDirective, + ) -> Result { + if let StreamStage::Failed(error) = self.stream_stage { + return Err(error); + } if self.stream_stage == StreamStage::Init { self.init_compress_stream2(end_op, input.src.len() - input.pos)?; } @@ -446,12 +470,12 @@ impl CCtx { if pledged != 0 && (consumed > pledged || (is_complete && consumed != pledged)) { flush_min = flush_min.and(Err(Error::SrcSizeWrong)); } - if flush_min.is_err() || (end_op == EndDirective::End && flush_min == Ok(0)) { + let flush_min = flush_min?; + if end_op == EndDirective::End && flush_min == 0 { // `reset_session`, but `mtctx` is borrowed. self.stream_stage = StreamStage::Init; self.frame.pledged_src_size_plus_one = 0; } - let flush_min = flush_min?; if end_op == EndDirective::Continue { // Some progress is enough. Otherwise, wait for a job to make some. diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index c6677aa40077..32a4c20abd4f 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -624,8 +624,8 @@ impl DCtx { } /// `ZSTD_decompressDCtx`: decompresses all the frames of `src`, which has to end with one, to the start of `dst`. - /// Returns how many bytes that is. As the other functions that take whole frames, it refuses with `StageWrong` from - /// the first call of [`Self::decompress_stream`] for a frame until that frame ends or [`Self::init_stream`]. + /// Returns how many bytes that is. As the other functions that take whole frames, it gives up a stream that is under + /// way or has failed. pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { let (size, dst) = (dst.len(), &mut OutputBuffer::Whole(dst)); self.state @@ -695,14 +695,13 @@ impl DCtx { /// `ZSTD_initDStream`: forgets the frame in progress and the dictionary. pub fn init_stream(&mut self) { - self.state.stream_stage = DStreamStage::Init; - self.state.no_forward_progress = 0; + self.state.reset_session(); self.ddict = None; } /// `ZSTD_decompressStream`: reads from `input` and writes to `output` as much as it can, and moves their positions. /// Returns 0 when a frame is complete and all of it is written, else a hint of how much input to give next. - /// Once a frame has made it fail, it fails in the same way until [`Self::init_stream`]. + /// Once a frame has made it fail, it fails in the same way until the stream is given up. pub fn decompress_stream( &mut self, output: &mut OutBuffer<'_>, @@ -930,8 +929,14 @@ impl DState { Ok((op - ostart, ip)) } - /// [`Self::decompress_multi_frame`] to the start of `dst`, unless a stream is under way: the C library then goes on - /// with that stream, with what this leaves. + /// `ZSTD_DCtx_reset` with `ZSTD_reset_session_only` + fn reset_session(&mut self) { + self.stream_stage = DStreamStage::Init; + self.no_forward_progress = 0; + } + + /// [`Self::decompress_multi_frame`] to the start of `dst`. It resets the stream, as `zstd.h` says of what starts a new + /// job: the C library does not, and goes on with a stream that was under way, with what this leaves. fn decompress_frames( &mut self, dst: &mut OutputBuffer<'_>, @@ -939,9 +944,7 @@ impl DState { src: &[u8], ddict: Option<&DDict<'_>>, ) -> Result { - if self.stream_stage != DStreamStage::Init { - return Err(Error::StageWrong); - } + self.reset_session(); self.decompress_multi_frame(dst, dst_size, 0, src, ddict) } diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 05265a18b51d..d78582e71cdf 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -1,5 +1,5 @@ //! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the -//! same bytes in, in both directions, and the same errors. +//! same bytes in, in both directions, and the same errors. Where that is not so, a comment tells what the C library does. #![feature(hint_prefetch, mpmc_channel, vec_push_within_capacity)] #![forbid(unsafe_code)] // Frames come from anywhere and a panic aborts the process. @@ -448,73 +448,95 @@ mod tests { use super::*; use util::SliceExt; + fn feed(stream: &mut DCtx, output: &mut OutBuffer<'_>, src: &[u8]) -> Result { + stream.decompress_stream(output, &mut InBuffer { src, pos: 0 }) + } + #[test] fn a_stream_that_has_failed_goes_on_failing() { // 5 bytes are announced, and the last block has 3. let frame = [ 0x28, 0xB5, 0x2F, 0xFD, 0x20, 5, 0x19, 0, 0, b'a', b'b', b'c', ]; - let mut stream = DCtx::new(); - let mut decompress = |src: &[u8]| { - let mut dst = [0; 64]; - let (mut output, mut input) = ( - OutBuffer { - dst: &mut dst, - pos: 0, - }, - InBuffer { src, pos: 0 }, - ); - stream.decompress_stream(&mut output, &mut input) - }; - let (start, end) = frame.split_at(9); - assert_eq!(decompress(start), Ok(3)); - assert_eq!(decompress(end), Err(Error::CorruptionDetected)); - for more in [&b"x"[..], b"", &frame] { - assert_eq!(decompress(more), Err(Error::CorruptionDetected)); - } - stream.init_stream(); - let (mut dst, src) = ([0; 64], [&frame[..5], &[3], &frame[6..]].concat()); - let (mut output, mut input) = ( - OutBuffer { + let valid = [&frame[..5], &[3], &frame[6..]].concat(); + for with_whole_frame in [false, true] { + let (mut stream, mut dst) = (DCtx::new(), [0; 64]); + let output = &mut OutBuffer { dst: &mut dst, pos: 0, - }, - InBuffer { src: &src, pos: 0 }, - ); - assert_eq!(stream.decompress_stream(&mut output, &mut input), Ok(0)); - assert_eq!(output.pos, 3); + }; + assert_eq!(feed(&mut stream, output, &frame[..9]), Ok(3)); + for more in [&frame[9..], b"x", b"", &valid] { + let result = feed(&mut stream, output, more); + assert_eq!(result, Err(Error::CorruptionDetected)); + } + assert_eq!(stream.load_dictionary(&[]), Err(Error::StageWrong)); + if with_whole_frame { + assert_eq!(stream.decompress(&mut [0; 64], &valid), Ok(3)); + } else { + stream.init_stream(); + } + assert_eq!(feed(&mut stream, output, &valid), Ok(0)); + assert_eq!(output.pos, 3); + } } #[test] - fn a_stream_under_way_goes_on_after_what_does_not_concern_its_frame() -> Result<()> { + fn a_stream_goes_on_after_a_lack_of_progress_but_not_after_a_whole_frame() -> Result<()> { let data = Vec::from_iter((0..300u32).map(|i| (i * i % 7) as u8)); let frame = compress_alloc(&data, 3)?; - let mut dst = [0; 400]; - let mut stream = DCtx::new(); let (start, end) = frame.split_at(frame.len() - 5); - let (mut output, mut input) = ( - OutBuffer { - dst: &mut dst, - pos: 0, - }, - InBuffer { src: start, pos: 0 }, - ); - assert_eq!(stream.decompress_stream(&mut output, &mut input), Ok(5)); - assert_eq!( - stream.decompress(&mut [0; 400], &frame), - Err(Error::StageWrong) - ); - let mut no_input = InBuffer { src: &[], pos: 0 }; + let (mut stream, mut dst) = (DCtx::new(), [0; 400]); + let output = &mut OutBuffer { + dst: &mut dst, + pos: 0, + }; + assert_eq!(feed(&mut stream, output, start), Ok(5)); for _ in 0..15 { - assert_eq!(stream.decompress_stream(&mut output, &mut no_input), Ok(5)); + assert_eq!(feed(&mut stream, output, &[]), Ok(5)); } - let result = stream.decompress_stream(&mut output, &mut no_input); + let result = feed(&mut stream, output, &[]); assert_eq!(result, Err(Error::NoForwardProgressInputEmpty)); - let mut input = InBuffer { src: end, pos: 0 }; - assert_eq!(stream.decompress_stream(&mut output, &mut input), Ok(0)); - assert_eq!(output.pos, data.len()); - assert_eq!(*dst.sub(..data.len())?, *data); - assert_eq!(stream.decompress(&mut dst, &frame), Ok(data.len())); + assert_eq!(feed(&mut stream, output, end), Ok(0)); + assert_eq!(*output.dst.sub(..output.pos)?, *data); + + assert_eq!(feed(&mut stream, output, start), Ok(5)); + assert_eq!(stream.decompress(&mut [0; 400], &frame), Ok(data.len())); + assert_eq!(feed(&mut stream, output, end), Err(Error::PrefixUnknown)); + Ok(()) + } + + #[test] + fn a_stream_that_has_failed_to_compress_goes_on_failing() -> Result<()> { + let data = Vec::from_iter((0..1500u32).map(|i| (i * i % 251) as u8)); + for nb_workers in [0, 1] { + let mut cctx = CCtx::new(); + cctx.set_parameter(c_param::COMPRESSION_LEVEL, 1)?; + cctx.set_parameter(c_param::NB_WORKERS, nb_workers)?; + let mut dst = vec![0; compress_bound(data.len())]; + let output = &mut OutBuffer { + dst: &mut dst, + pos: 0, + }; + let mut compress = |cctx: &mut CCtx, src, end| { + cctx.compress_stream2(output, &mut InBuffer { src, pos: 0 }, end) + }; + + // The magic number of a dictionary, and nothing that goes with it: no frame starts. + cctx.load_dictionary(&[0x37, 0xA4, 0x30, 0xEC, 1, 2, 3, 4, 5])?; + let result = compress(&mut cctx, &data, EndDirective::Continue); + assert_eq!(result, Err(Error::MemoryAllocation)); + cctx.load_dictionary(&[])?; + + // Enough for the workers to be used. + cctx.set_pledged_src_size(1 << 20)?; + compress(&mut cctx, &[], EndDirective::Continue)?; + for end in [EndDirective::End, EndDirective::End, EndDirective::Flush] { + let result = compress(&mut cctx, &data, end); + assert_eq!(result, Err(Error::SrcSizeWrong), "{nb_workers} workers"); + } + assert_eq!(cctx.set_pledged_src_size(1), Err(Error::StageWrong)); + } Ok(()) } From d106dc84f1cc8454695c7f85e09a8b6a919be24e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 00:49:14 +0000 Subject: [PATCH 165/199] zstd: what `exec_batch` does before its loop is a function of its own It is the same for all six instances of `exec_batch`, which is what the `mordant` job pointed out. --- src/zstd/decompress/block.rs | 61 ++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 76303bb02b1d..a8acee75d41d 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -1367,6 +1367,18 @@ fn resolve_offset(sequences: &mut [RawSequence], index: usize) -> u32 { this.offset_value } +/// What [`Executor::exec_batch`] works on. Positions in these fit in 32 bits, so that the compiler sees that nothing added +/// to them overflows. +struct BatchBuffers<'buf, 'a, 'dict> { + /// Where in the output buffer `buf` starts. + start: usize, + buf: &'buf mut [u8], + lits: &'a [u8], + /// What comes right before `buf`, if anything of the common kind does: this or, of `buf`, `within`. + dict: &'dict [u8], + within: Range, +} + /// Turns sequences into output. struct Executor<'buf, 'a> { /// What is beyond `oend` is room for copies in chunks. @@ -1634,6 +1646,33 @@ impl<'a> Executor<'_, 'a> { } } + /// `within` is of the whole output buffer. + #[inline(always)] + fn batch_buffers<'dict>( + &mut self, + dict: &'dict [u8], + within: Range, + ) -> BatchBuffers<'_, 'a, 'dict> { + const MAX_BACK: usize = 3 << 30; + let start = self.prefix_start.max(self.op.saturating_sub(MAX_BACK)); + let (dict, within) = if start == self.prefix_start && within.start >= start { + (dict, within.start - start..within.end - start) + } else { + (&[][..], 0..0) + }; + let lits = self.lits.get(..u32::MAX as usize).unwrap_or(self.lits); + let buf = self.out.bytes(); + let end = buf.len().min(start + u32::MAX as usize); + let buf = buf.get_mut(start..end).unwrap_or_default(); + BatchBuffers { + start, + buf, + lits, + dict, + within, + } + } + /// Executes `sequences` from `index` on, for as long as nothing is close to its end and they are of the common kind. /// Returns the index of the first that is not, whose offset value then is no reference to one of the last offsets: /// it is 0 if that is not valid. @@ -1651,19 +1690,15 @@ impl<'a> Executor<'_, 'a> { within: Range, window: usize, ) -> usize { - // Positions fit in 32 bits, so that the compiler sees that nothing added to them overflows. - const MAX_BACK: usize = 3 << 30; - let start = self.prefix_start.max(self.op.saturating_sub(MAX_BACK)); - let (dict, within) = if start == self.prefix_start && within.start >= start { - (dict, within.start - start..within.end - start) - } else { - (&[][..], 0..0) - }; - let buf = self.out.bytes(); - let end = buf.len().min(start + u32::MAX as usize); - let buf = buf.get_mut(start..end).unwrap_or_default(); - let mut op = self.op.wrapping_sub(start) as u32; - let lits = self.lits.get(..u32::MAX as usize).unwrap_or(self.lits); + let op = self.op; + let BatchBuffers { + start, + buf, + lits, + dict, + within, + } = self.batch_buffers(dict, within); + let mut op = op.wrapping_sub(start) as u32; let mut lit_pos = 0u32; while let Some(raw) = sequences.get(index) { From e48dc2dc314adb9fe04ce85f0ac4ba7ab8239ea4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 01:36:19 +0000 Subject: [PATCH 166/199] zstd: matches in a dictionary are of the common kind wherever the output starts Whether the loop for common sequences kept the dictionary hung on a test that is about the other kind of history, on a placeholder for it, and that failed whenever the output did not start at the start of its buffer: in every frame but the first of a call, and in a frame that a stream decodes in one go behind earlier output. Their matches in the dictionary took the careful path. `batch_buffers` now takes what comes before the output from the executor, so there are no placeholders, and three arguments less to three functions. 20 frames of 100 KB made of pieces of 8 to 16 bytes of a dictionary, decoded in one call: 1.24 ms, from 2.59 (C: 1.64). A test calls the loop itself, at the start of a buffer and further on, and tells how far it gets. Two more decode frames one after the other, since the path is new to them. --- src/zstd/decompress/block.rs | 112 +++++++++++++++++++++++----------- src/zstd/lib.rs | 34 +++++++++++ test/js/bun/util/zstd.test.ts | 11 ++++ 3 files changed, 122 insertions(+), 35 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index a8acee75d41d..115869bde3bf 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -1369,14 +1369,16 @@ fn resolve_offset(sequences: &mut [RawSequence], index: usize) -> u32 { /// What [`Executor::exec_batch`] works on. Positions in these fit in 32 bits, so that the compiler sees that nothing added /// to them overflows. -struct BatchBuffers<'buf, 'a, 'dict> { +struct BatchBuffers<'buf, 'a> { /// Where in the output buffer `buf` starts. start: usize, buf: &'buf mut [u8], lits: &'a [u8], /// What comes right before `buf`, if anything of the common kind does: this or, of `buf`, `within`. - dict: &'dict [u8], + dict: &'a [u8], within: Range, + /// [`ExtDict::Within::window`] + window: usize, } /// Turns sequences into output. @@ -1646,19 +1648,17 @@ impl<'a> Executor<'_, 'a> { } } - /// `within` is of the whole output buffer. #[inline(always)] - fn batch_buffers<'dict>( - &mut self, - dict: &'dict [u8], - within: Range, - ) -> BatchBuffers<'_, 'a, 'dict> { + fn batch_buffers(&mut self) -> BatchBuffers<'_, 'a> { const MAX_BACK: usize = 3 << 30; let start = self.prefix_start.max(self.op.saturating_sub(MAX_BACK)); - let (dict, within) = if start == self.prefix_start && within.start >= start { - (dict, within.start - start..within.end - start) - } else { - (&[][..], 0..0) + let is_whole = start == self.prefix_start; + let (dict, within, window) = match self.ext_dict { + ExtDict::Dict(dict) if is_whole => (dict, 0..0, 0), + ExtDict::Within { end, len, window } if is_whole && end - len >= start => { + (&[][..], end - len - start..end - start, window) + } + _ => (&[][..], 0..0, 0), }; let lits = self.lits.get(..u32::MAX as usize).unwrap_or(self.lits); let buf = self.out.bytes(); @@ -1670,6 +1670,7 @@ impl<'a> Executor<'_, 'a> { lits, dict, within, + window, } } @@ -1679,16 +1680,13 @@ impl<'a> Executor<'_, 'a> { /// /// The last offsets are those of the 3 sequences before, which are rewritten for that where it takes it. /// - /// Matches in what comes before the output are of the common kind if `EXT` tells that it is `dict`, or `within` of the - /// buffer that the output goes to. So are, with `CLOSE`, short matches from less than 16 bytes back. + /// `EXT` tells which kind [`Self::ext_dict`] is of, if matches in it are of the common kind. So are, with `CLOSE`, short + /// matches from less than 16 bytes back. #[inline(never)] fn exec_batch( &mut self, sequences: &mut [RawSequence], mut index: usize, - dict: &[u8], - within: Range, - window: usize, ) -> usize { let op = self.op; let BatchBuffers { @@ -1697,7 +1695,8 @@ impl<'a> Executor<'_, 'a> { lits, dict, within, - } = self.batch_buffers(dict, within); + window, + } = self.batch_buffers(); let mut op = op.wrapping_sub(start) as u32; let mut lit_pos = 0u32; @@ -1965,13 +1964,9 @@ impl<'a> Executor<'_, 'a> { last.offset_value = *offset as u32 + 3; } match self.ext_dict { - ExtDict::Dict(dict) if !dict.is_empty() => { - self.exec_in_turns::(sequences, dict, 0..0, 0) - } - ExtDict::Within { end, len, window } if len != 0 => { - self.exec_in_turns::(sequences, &[], end - len..end, window) - } - _ => self.exec_in_turns::(sequences, &[], 0..0, 0), + ExtDict::Dict(dict) if !dict.is_empty() => self.exec_in_turns::(sequences), + ExtDict::Within { len, .. } if len != 0 => self.exec_in_turns::(sequences), + _ => self.exec_in_turns::(sequences), }?; for (last, offset) in sequences.iter().rev().zip(prev_offset) { *offset = last.offset_value as usize - 3; @@ -1979,22 +1974,16 @@ impl<'a> Executor<'_, 'a> { Ok(()) } - /// Executes `sequences` but the first 3. What else it gets is for [`Self::exec_batch`]. + /// Executes `sequences` but the first 3. `EXT` is for [`Self::exec_batch`]. #[inline(always)] - fn exec_in_turns( - &mut self, - sequences: &mut [RawSequence], - dict: &[u8], - within: Range, - window: usize, - ) -> Result<()> { + fn exec_in_turns(&mut self, sequences: &mut [RawSequence]) -> Result<()> { let mut index = 3; let mut close = false; while index < sequences.len() { index = if close { - self.exec_batch::(sequences, index, dict, within.clone(), window) + self.exec_batch::(sequences, index) } else { - self.exec_batch::(sequences, index, dict, within.clone(), window) + self.exec_batch::(sequences, index) }; if index < sequences.len() { index = self.exec_others(sequences, index)?; @@ -2056,4 +2045,57 @@ mod tests { assert!(!copy_short_close_match(&mut [0; 64], 3, 0, 5)); Ok(()) } + + #[test] + fn matches_in_a_dictionary_are_of_the_common_kind_wherever_the_output_starts() -> Result<()> { + let dict: Vec = (0..200).collect(); + let lits = [b'l'; 64]; + let sequence = |lit_length, match_length, offset: u32| RawSequence { + offset_value: offset + 3, + lit_length, + match_length: match_length - MINMATCH, + }; + for prefix_start in [0, 50] { + let mut sequences = [ + RawSequence::EMPTY, + RawSequence::EMPTY, + RawSequence::EMPTY, + sequence(0, 10, 150), + sequence(2, 40, 12 + 100), + sequence(1, 5, 4), + sequence(0, 8, 58 + 200), + ]; + let expected = [ + dict.sub(50..60)?, + b"ll", + dict.sub(100..140)?, + b"l", + &[137, 138, 139, b'l', 137], + dict.sub(..8)?, + ] + .concat(); + let mut buf = [0xEE; 250]; + let mut exec = Executor { + out: OutputBuffer::Whole(&mut buf), + in_one_call: true, + oend: 250, + op: prefix_start, + prefix_start, + ext_dict: ExtDict::Dict(&dict), + lits: &lits, + split: false, + lits_extra: &[], + lit_end_in_dst: 0, + }; + assert_eq!(exec.exec_batch::(&mut sequences, 3), 5); + assert_eq!(exec.exec_batch::(&mut sequences, 5), 7); + assert_eq!(exec.op, prefix_start + expected.len()); + assert_eq!( + *buf.sub(prefix_start..prefix_start + expected.len())?, + *expected + ); + assert!(buf.sub(..prefix_start)?.iter().all(|&byte| byte == 0xEE)); + } + Ok(()) + } } diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index d78582e71cdf..d32aa80e9757 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -725,6 +725,40 @@ mod tests { Ok(()) } + #[test] + fn matches_in_a_dictionary_are_found_from_every_frame_of_a_call() -> Result<()> { + let dict = Vec::from_iter((0..3000u32).map(|i| ((i * i % 251) ^ (i >> 3)) as u8)); + // Short and long, from all over the dictionary and from its end on into the output, with a literal between. + let mut src = Vec::new(); + for i in 0..if cfg!(miri) { 60 } else { 600 } { + // Where most are long, sequences are not executed in batches. + let len = if i % 11 == 0 { 70 } else { 3 + i * 7 % 45 }; + let at = if i % 5 == 0 { + dict.len() - len / 2 + } else { + i * 613 % (dict.len() - len) + }; + src.extend_from_slice(dict.sub(at..dict.len().min(at + len))?); + let beyond = (at + len).saturating_sub(dict.len()).min(src.len()); + let from_start = src.sub(..beyond)?.to_vec(); + src.extend_from_slice(&from_start); + src.push(i as u8); + } + let mut cctx = CCtx::new(); + cctx.load_dictionary(&dict)?; + let frame = frame_of(&mut cctx, &[&src])?; + assert!(frame.len() < src.len() / 3); + + let (frames, all) = (frame.repeat(3), src.repeat(3)); + let ddict = DDict::by_reference(&dict)?; + let mut there = vec![0; all.len()]; + let written = DCtx::new().decompress_using_ddict(&mut there, &frames, &ddict); + assert!(written == Ok(all.len()) && there == all); + let appended = DCtx::new().decompress_to_vec(all.len(), &frames, Some(&ddict))?; + assert!(appended == all); + Ok(()) + } + #[test] fn output_that_is_appended_is_what_output_that_is_there_is() -> Result<()> { let mut state = 1u32; diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index bc47d1ed4888..129926405e83 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -498,6 +498,17 @@ describe("decompressing frames whose size is known", () => { expect(zlib.zstdDecompressSync(inSmallWindow).equals(data)).toBe(true); }); + it("frames one after the other whose matches are in a dictionary", () => { + const dictionary = noise(20_000); + const piece = (i: number) => dictionary.subarray((i * 7919) % 19_000).subarray(0, 8 + (i % 9)); + const data = Buffer.concat(Array.from({ length: 1_500 }, (_, i) => piece(i))); + const frame = zlib.zstdCompressSync(data, { dictionary }); + expect(blocks(frame)[0]).toBeGreaterThan(1_000); + // There is room for all three, so each is decoded in one go to where the one before it ends. + const decoded = zlib.zstdDecompressSync(Buffer.concat([frame, frame, frame]), { dictionary, chunkSize: 1 << 17 }); + expect(decoded.equals(Buffer.concat([data, data, data]))).toBe(true); + }); + it("a frame that makes more than it says is refused", () => { for (const piece of pieces.filter(piece => piece.length >= 135_000)) { const frame = zstdCompressSync(piece.subarray(0, 135_000), { level: 1 }); From 39b1e9fae62f581192c1854b1191dba8c6617769 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 03:11:21 +0000 Subject: [PATCH 167/199] zstd: sequences are executed in batches also while literals would be in the output Of more than 64 KB of literals, the C library keeps what is above 64 KB in the output buffer if that has no room for all of them: in a stream, and in the last blocks of a frame that is decoded to a buffer of its size. It then refuses a sequence whose output catches up with them. For that one error, sequences were decoded and executed one at a time until those literals were used up, which in a block that ends with 64 KB of literals is never. The literals are not really there, so all it takes is that the loop for common sequences stops short of where they would be. They only move on, so where they are when a batch starts will do. What is closer than that is left to the code that does what the C library does, as before. No valid block gets there. The buffer that the loop works on ends there, so there is nothing more to check, unless the older part of a stream's buffer is after that. Time to decode, of what it was: - 128 KB, half text and half noise, level 1: 0.79 (0.91 of C, from 1.15) - noise of 6 bits with a short match every 25 bytes, as a stream with a window of 128 KB: 0.86 if they are from 70 to 125 KB back, 0.93 from 200 to 3000, 1.016 from 16 to 55, where the loop is slower than what it stands in for - 1 MB of hex digits as a stream: 0.92 at level 3, 0.88 at level 9 with that window - 128 KB of integers, whose literals come 90 at a time: 1.006 - an executable as a stream: 1.005 to 1.014; text: 1.006; JSON, logs: the same A unit test has blocks made by hand, in one call and in a stream whose buffer is written to from its start again: valid ones, of which one uses up the literals in question and one has few sequences, blocks that make too much, and blocks whose 125th sequence the C library refuses. Each of the four conditions was taken out once to see it fail. --- src/zstd/decompress/block.rs | 77 +++++++++++++++++++++------------- src/zstd/lib.rs | 65 +++++++++++++++++++++++++++++ test/js/bun/util/zstd.test.ts | 78 ++++++++++++++++++++++++++++++----- 3 files changed, 180 insertions(+), 40 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 115869bde3bf..51c3800576f4 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -1379,6 +1379,9 @@ struct BatchBuffers<'buf, 'a> { within: Range, /// [`ExtDict::Within::window`] window: usize, + /// Where in `buf` the literals would be, which the output must not catch up with. What gets close to them is not of + /// the common kind. That is where `buf` ends, unless `within` is after it. + lit_ptr: usize, } /// Turns sequences into output. @@ -1421,6 +1424,12 @@ impl<'a> Executor<'_, 'a> { self.lits.len().saturating_sub(WILDCOPY_OVERLENGTH) } + /// `litPtr`, if `split`: where in the output buffer `lits` would start. + #[inline(always)] + fn lit_ptr(&self) -> usize { + self.lit_end_in_dst.wrapping_sub(self.lits_left()) + } + /// The same, for a while. A loop works on a copy that is a local variable and whose address nothing takes, which /// can stay in registers. What is behind a reference is written and read again for every sequence. #[inline(always)] @@ -1462,7 +1471,7 @@ impl<'a> Executor<'_, 'a> { return Err(Error::CorruptionDetected); } if self.split { - let lit_ptr = self.lit_end_in_dst.wrapping_sub(self.lits_left()); + let lit_ptr = self.lit_ptr(); if op > lit_ptr && op < lit_ptr.wrapping_add(lit_length) { return Err(Error::DstSizeTooSmall); } @@ -1506,6 +1515,7 @@ impl<'a> Executor<'_, 'a> { let match_length = sequence.match_length as usize; let offset = sequence.offset; let op = self.op; + let lit_ptr = self.lit_ptr(); let room = self.out.bytes().get_mut(op..)?; if lit_length + match_length + WILDCOPY_OVERLENGTH > room.len() @@ -1515,9 +1525,7 @@ impl<'a> Executor<'_, 'a> { } if SPLIT { // The output must not catch up with the literals. - let i_lit_end = (self.lit_end_in_dst) - .wrapping_sub(self.lits.len() - WILDCOPY_OVERLENGTH) - .wrapping_add(lit_length); + let i_lit_end = lit_ptr.wrapping_add(lit_length); if op + lit_length + match_length + WILDCOPY_OVERLENGTH > i_lit_end { return None; } @@ -1661,8 +1669,18 @@ impl<'a> Executor<'_, 'a> { _ => (&[][..], 0..0, 0), }; let lits = self.lits.get(..u32::MAX as usize).unwrap_or(self.lits); + let lit_ptr = if self.split { + self.lit_ptr() + } else { + usize::MAX + }; + let end = if within.is_empty() { + lit_ptr + } else { + usize::MAX + }; let buf = self.out.bytes(); - let end = buf.len().min(start + u32::MAX as usize); + let end = buf.len().min(start + u32::MAX as usize).min(end); let buf = buf.get_mut(start..end).unwrap_or_default(); BatchBuffers { start, @@ -1671,6 +1689,7 @@ impl<'a> Executor<'_, 'a> { dict, within, window, + lit_ptr: lit_ptr.saturating_sub(start), } } @@ -1696,6 +1715,7 @@ impl<'a> Executor<'_, 'a> { dict, within, window, + lit_ptr, } = self.batch_buffers(); let mut op = op.wrapping_sub(start) as u32; let mut lit_pos = 0u32; @@ -1719,6 +1739,7 @@ impl<'a> Executor<'_, 'a> { let o_match_end = o_lit_end + match_length; let lit_end = lit_pos as usize + lit_length; if o_match_end + WILDCOPY_OVERLENGTH > buf.len() + || EXT == EXT_WITHIN && o_match_end + WILDCOPY_OVERLENGTH > lit_ptr || lit_end + WILDCOPY_OVERLENGTH > lits.len() || lit_length > MAX_BATCH_LITERALS as usize || (EXT == NO_EXT || match_length > 64 || offset_value < MIN_FAR_OFFSET_VALUE) @@ -1821,7 +1842,7 @@ impl<'a> Executor<'_, 'a> { { break; } - exec.exec_sequence::(Sequence { + exec.exec_sequence_maybe_split(Sequence { lit_length: raw.lit_length, match_length: raw.match_length + MINMATCH, offset: (offset_value as usize).wrapping_sub(3), @@ -1854,7 +1875,7 @@ impl<'a> Executor<'_, 'a> { let mut remaining = nb_seq; // Long copies go on while the next sequences are decoded, if that is what comes next. And a batch of a few - // sequences is not worth it. + // sequences is not worth it. This loop knows nothing of literals that would be in the output. if (seq_state.tables.matches_are_long() || nb_seq <= MAX_ONE_BY_ONE && !LONG) && !self.split { let mut exec = self.fork(); @@ -1870,28 +1891,26 @@ impl<'a> Executor<'_, 'a> { let mut left_at = None; while remaining > ahead.len() { let mut count = 0; - if !self.split { - if PaddedStart::fits(&seq_state.bit_d) - && let Some(start) = start.take() - && let Some(bit_d) = start.insert(PaddedStart::new()).set(&seq_state.bit_d) - { - left_at = Some((seq_state, remaining)); - seq_state.bit_d = bit_d; - } - // The last sequence is followed by no state. - let room = (batch.len() - 4).min(remaining - ahead.len().max(1)); - if room > 0 { - let history = &mut self.history(); - let room = batch.span_mut(3, room)?; - count = seq_state.decode_batch(LONG, more_bits, room, history)?; - } - if left_at.is_some() - && count + 1 == remaining - && let Some(last) = seq_state.decode_last() - { - batch.put(3 + count, last)?; - count += 1; - } + if PaddedStart::fits(&seq_state.bit_d) + && let Some(start) = start.take() + && let Some(bit_d) = start.insert(PaddedStart::new()).set(&seq_state.bit_d) + { + left_at = Some((seq_state, remaining)); + seq_state.bit_d = bit_d; + } + // The last sequence is followed by no state. + let room = (batch.len() - 4).min(remaining - ahead.len().max(1)); + if room > 0 { + let history = &mut self.history(); + let room = batch.span_mut(3, room)?; + count = seq_state.decode_batch(LONG, more_bits, room, history)?; + } + if left_at.is_some() + && count + 1 == remaining + && let Some(last) = seq_state.decode_last() + { + batch.put(3 + count, last)?; + count += 1; } let mut sequence = None; if count == 0 { diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index d32aa80e9757..163ece33bc73 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -759,6 +759,71 @@ mod tests { Ok(()) } + #[test] + fn output_does_not_catch_up_with_literals_that_would_be_in_its_buffer() { + const MAGIC: [u8; 4] = common::ZSTD_MAGICNUMBER.to_le_bytes(); + let le24 = |value: usize| [value as u8, (value >> 8) as u8, (value >> 16) as u8]; + let size = |more: usize, nb_seq: usize| (1 << 16) + more + 1027 * nb_seq; + // A last block: 64 KB and `more` times a byte, and sequences of 16 literals and 1027 bytes from 16 back. + let block = |more: usize, nb_seq: usize| { + // For each sequence 4 bits of the offset, 10 of the match length and one of the literal length. After the + // last, a bit set. + let mut bits = vec![0u8; 15 * nb_seq / 8 + 1]; + let of_offsets = (0..nb_seq).flat_map(|i| [15 * i + 11, 15 * i + 12]); + for bit in of_offsets.chain([15 * nb_seq]) { + if let Some(byte) = bits.get_mut(bit / 8) { + *byte |= 1 << (bit % 8); + } + } + let literals = le24(1 | 3 << 2 | ((1 << 16) + more) << 4); + // All three codes are one symbol. + let sequences = [nb_seq as u8, 0x54, 16, 4, 46]; + let header = le24(1 | 2 << 1 | (4 + sequences.len() + bits.len()) << 3); + [&header[..], &literals, b"a", &sequences, &bits].concat() + }; + let in_one_call = |more: usize, nb_seq: usize| { + let content_size = (size(more, nb_seq) as u32).to_le_bytes(); + let frame = [&MAGIC[..], &[0xA0], &content_size, &block(more, nb_seq)].concat(); + let mut dst = vec![0; size(more, nb_seq)]; + let written = DCtx::new().decompress(&mut dst, &frame)?; + dst.truncate(written); + Ok((written, dst == vec![b'a'; written])) + }; + // With a window of 128 KB, after 3 blocks of as many times the byte, which fill the buffer of the stream. + let as_a_stream = |more: usize, nb_seq: usize| { + let filler = [&le24(1 << 1 | 1 << 17 << 3)[..], b"a"].concat().repeat(3); + let frame = [&MAGIC[..], &[0, 7 << 3], &filler, &block(more, nb_seq)].concat(); + let mut dst = vec![0; 600_000]; + let mut output = OutBuffer { + dst: &mut dst, + pos: 0, + }; + assert_eq!(feed(&mut DCtx::new(), &mut output, &frame)?, 0); + let written = output.pos; + dst.truncate(written); + Ok((written - (3 << 17), dst == vec![b'a'; written])) + }; + // Where there is too little room for the C library to keep all the literals in it, it keeps the first `more`, + // which end 32 bytes before the 128 KB that a block can make. + let cases = [ + (500, 60, Ok((size(500, 60), true))), + // The 125th starts 6 bytes after where its literals do, and ends before where the last of them does. + (3698, 125, Err(Error::DstSizeTooSmall)), + (3698, 126, Err(Error::DstSizeTooSmall)), + (3698, 124, Err(Error::CorruptionDetected)), + (3698, 61, Err(Error::CorruptionDetected)), + (3698, 60, Ok((size(3698, 60), true))), + // Too few for a batch where the literals are elsewhere. + (50, 5, Ok((size(50, 5), true))), + ]; + for (more, nb_seq, expected) in cases.into_iter().take(if cfg!(miri) { 2 } else { 7 }) { + assert_eq!(in_one_call(more, nb_seq), expected, "{more}, {nb_seq}"); + if !cfg!(miri) { + assert_eq!(as_a_stream(more, nb_seq), expected, "{more}, {nb_seq}"); + } + } + } + #[test] fn output_that_is_appended_is_what_output_that_is_there_is() -> Result<()> { let mut state = 1u32; diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 129926405e83..fbd0fdd5dc8e 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -367,8 +367,8 @@ describe("decompressing frames whose size is known", () => { ]; const whole = Buffer.concat(pieces); - // "raw", "rle", or how many sequences a compressed block has. Or else how its literals are written. - function blocks(frame: Uint8Array, ofLiterals = false) { + // "raw", "rle", or how many sequences a compressed block has. Or else how its literals are written, or how many it has. + function blocks(frame: Uint8Array, ofLiterals: boolean | "count" = false) { const descriptor = frame[4]; const isSingleSegment = (descriptor >> 5) & 1; let at = 5 + (1 - isSingleSegment) + [0, 1, 2, 4][descriptor & 3] + [isSingleSegment, 2, 4, 8][descriptor >> 6]; @@ -387,22 +387,26 @@ describe("decompressing frames whose size is known", () => { frame[at + 3] * 2 ** 24 + frame[at + 4] * 2 ** 32; let literals: number; + let count: number; if ((frame[at] & 3) < 2) { - const size = format === 1 ? (bits >> 4) & 0xfff : format === 3 ? (bits >> 4) & 0xfffff : frame[at] >> 3; - literals = [1, 2, 1, 3][format] + ((frame[at] & 3) === 0 ? size : 1); + count = format === 1 ? (bits >> 4) & 0xfff : format === 3 ? (bits >> 4) & 0xfffff : frame[at] >> 3; + literals = [1, 2, 1, 3][format] + ((frame[at] & 3) === 0 ? count : 1); } else { const width = [10, 10, 14, 18][format]; + count = Math.floor(bits / 2 ** 4) % 2 ** width; literals = [3, 3, 4, 5][format] + (Math.floor(bits / 2 ** (4 + width)) % 2 ** width); } const [first, second, third] = frame.subarray(at + literals); kinds.push( - ofLiterals - ? ["raw", "rle", "code", "same code"][frame[at] & 3] - : first < 128 - ? first - : first < 255 - ? ((first - 128) << 8) + second - : second + (third << 8) + 0x7f00, + ofLiterals === "count" + ? count + : ofLiterals + ? ["raw", "rle", "code", "same code"][frame[at] & 3] + : first < 128 + ? first + : first < 255 + ? ((first - 128) << 8) + second + : second + (third << 8) + 0x7f00, ); } else { kinds.push(["raw", "rle"][type]); @@ -509,6 +513,58 @@ describe("decompressing frames whose size is known", () => { expect(decoded.equals(Buffer.concat([data, data, data]))).toBe(true); }); + // The C library keeps some of them in the output buffer, unless that has room for all of them. + describe("blocks of more than 64 KB of literals", () => { + // Noise of 6 bits, and after every 6 to 30 bytes of it, up to `end`, 5 to 8 bytes from `nearest` to `furthest` back. + function literalsAndMatches(length: number, end: number, nearest: number, furthest: number) { + const data = Buffer.alloc(length); + for (let at = 0, x = 0x9e3779b9; at < length; at++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + data[at] = x & 63; + } + for (let at = furthest, i = 0; at < end - 64; i++) { + at += 6 + (i % 25); + const from = at - nearest - ((i * 7919) % (furthest - nearest)); + data.copyWithin(at, from, from + 5 + (i % 4)); + at += 5 + (i % 4); + } + return data; + } + const params = { [zlib.constants.ZSTD_c_compressionLevel]: 6, [zlib.constants.ZSTD_c_windowLog]: 17 }; + + it.each([128_000, 40_000])("with matches in the first %i bytes", end => { + const data = literalsAndMatches(128_000, end, 16, 56); + const frame = zstdCompressSync(data, { level: 3 }); + expect(blocks(frame)).toEqual([expect.any(Number)]); + expect(blocks(frame)[0]).toBeGreaterThan(end / 40); + expect(blocks(frame, "count")[0]).toBeGreaterThan(1 << 16); + expect(zstdDecompressSync(frame).equals(data)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + }); + + it("with matches in the older part of a buffer that is written to from its start again", () => { + const data = literalsAndMatches(900_000, 900_000, 70_000, 125_000); + const frame = zlib.zstdCompressSync(data, { params }); + expect(blocks(frame).length).toBe(7); + // The buffer holds 3 blocks. + for (const sequences of blocks(frame).slice(3)) expect(sequences).toBeGreaterThan(3_000); + for (const literals of blocks(frame, "count")) expect(literals).toBeGreaterThan(1 << 16); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + }); + + it("with matches in a dictionary", () => { + const all = literalsAndMatches(250_000, 250_000, 70_000, 125_000); + const [dictionary, data] = [all.subarray(0, 125_000), all.subarray(125_000)]; + const frame = zlib.zstdCompressSync(data, { params, dictionary }); + expect(blocks(frame)).toEqual([expect.any(Number)]); + expect(blocks(frame)[0]).toBeGreaterThan(3_000); + expect(blocks(frame, "count")[0]).toBeGreaterThan(1 << 16); + expect(zlib.zstdDecompressSync(frame, { dictionary }).equals(data)).toBe(true); + }); + }); + it("a frame that makes more than it says is refused", () => { for (const piece of pieces.filter(piece => piece.length >= 135_000)) { const frame = zstdCompressSync(piece.subarray(0, 135_000), { level: 1 }); From 68fca8c91d1888204af6e86acfb854bd55708b5c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 03:18:07 +0000 Subject: [PATCH 168/199] build: the C decoder of zstd stays for the `jsc` shell that reads a prebuilt ICU On main the `jsc` shell of a WebKit that is built here (`--webkit=local`) with the ICU of a prebuilt one (`BUN_ICU_ROOT`, Linux) is now compiled with zstd's headers and linked with zstd: the data of that ICU has a zstd frame an item, and the hook that reads them is in WebKit's jsc.cpp, which calls the C library. Merged with this branch, which had no such dependency any more, the build scripts did not type-check, and that configuration would have failed to configure. The dependency is defined again, for that program alone: only its decoder, only in that configuration, and apart from the dependencies of bun, so that nothing of it is offered to bun's compiles or link. No CI lane has that configuration: with a prebuilt WebKit the program comes out of the tarball. jsc.cpp is now compiled after the sources are fetched, which nothing said. Checked: both type checks of the lint job; without `BUN_ICU_ROOT` the ninja file has nothing of vendor/zstd; with it, the 13 objects are fetched and compiled, they define the seven functions that jsc.cpp calls and need nothing else, and `jsc` is the only link that takes them. Not checked: the link of `jsc` itself, which takes a prebuilt ICU that cannot be downloaded here. --- scripts/build/bun.ts | 8 +++++- scripts/build/deps/zstd.ts | 52 ++++++++++++++++++++++++++++++++++++++ scripts/build/flags.ts | 11 +++++++- scripts/build/source.ts | 3 ++- 4 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 scripts/build/deps/zstd.ts diff --git a/scripts/build/bun.ts b/scripts/build/bun.ts index e7088433e2b2..5e7e65b69f27 100644 --- a/scripts/build/bun.ts +++ b/scripts/build/bun.ts @@ -23,6 +23,7 @@ import { allDeps } from "./deps/index.ts"; import { lolhtml } from "./deps/lolhtml.ts"; import { rustArgon2 } from "./deps/rust-argon2.ts"; import { jscPrograms, linuxIcuRoot, webkitEngineSources } from "./deps/webkit.ts"; +import { zstd } from "./deps/zstd.ts"; import { assert } from "./error.ts"; import { bunIncludes, @@ -197,6 +198,9 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput { depsByName.set(dep.name, resolved); } } + // For a program beside bun, so not one of `deps`. + const zstdDep = resolveDep(n, cfg, zstd, depsByName); + if (zstdDep !== null) depsByName.set(zstd.name, zstdDep); // Collect all dep lib paths, include dirs, output stamps, and directly- // compiled source files (deps like picohttpparser that provide .c files @@ -498,11 +502,13 @@ function emitJscPrograms( }); prefixes.unshift({ prefix, files }); } + // Their headers are there once they are fetched. + const depsFetched = (program.deps ?? []).flatMap(name => deps.get(name)!.outputs); const objects = program.sources.map(src => { const prefix = prefixes.find(({ files }) => files === undefined || files.test(slash(src)))?.prefix; return cxx(n, cfg, src, { flags: program.cxxflags, - orderOnlyInputs: [headersReady], + orderOnlyInputs: [headersReady, ...depsFetched], ...(prefix !== undefined && { pch: prefix.pch, pchHeader: prefix.wrapperHeader, pchBases: prefix.bases }), }); }); diff --git a/scripts/build/deps/zstd.ts b/scripts/build/deps/zstd.ts new file mode 100644 index 000000000000..50ae5f61d45a --- /dev/null +++ b/scripts/build/deps/zstd.ts @@ -0,0 +1,52 @@ +/** + * Zstandard's decoder, in C, for one program that is not bun: the `jsc` shell + * of a WebKit that is built here (`--webkit=local`) with the ICU of a prebuilt + * one (`BUN_ICU_ROOT`). The data of that ICU has a zstd frame an item, and + * WebKit's jsc.cpp reads them with this library, as in WebKit's own build + * (shell/CMakeLists.txt, BUN_ICU_ZSTD_SOURCE_DIR). + * + * No other build fetches or compiles it, and it is not in `allDeps`: nothing of + * it reaches bun, whose zstd is src/zstd. + */ + +import type { Dependency } from "../source.ts"; +import { linuxIcuRoot } from "./webkit.ts"; + +const ZSTD_COMMIT = "f8745da6ff1ad1e7bab384bd1f9d742439278e99"; + +// prettier-ignore +const SOURCES = [ + "common/debug", "common/entropy_common", "common/error_private", + "common/fse_decompress", "common/pool", "common/threading", "common/xxhash", + "common/zstd_common", + "decompress/huf_decompress", "decompress/zstd_ddict", + "decompress/zstd_decompress", "decompress/zstd_decompress_block", +].map(s => `lib/${s}.c`); + +export const zstd: Dependency = { + name: "zstd", + + enabled: cfg => cfg.webkit === "local" && linuxIcuRoot(cfg) !== undefined, + + source: () => ({ + kind: "github-archive", + repo: "facebook/zstd", + commit: ZSTD_COMMIT, + }), + + build: cfg => ({ + kind: "direct", + // The Huffman kernel for amd64 is a .S file that clang assembles. + sources: [...SOURCES, ...(cfg.x64 ? ["lib/decompress/huf_decompress_amd64.S"] : [])], + defines: { ZSTD_LEGACY_SUPPORT: 0, ...(!cfg.x64 && { ZSTD_DISABLE_ASM: 1 }) }, + includes: ["lib", "lib/common"], + pic: true, + // A bare token that xxhash pastes onto its names: `defines` would quote it. + cflags: ["-DXXH_NAMESPACE=ZSTD_"], + }), + + provides: () => ({ + libs: [], + includes: [], + }), +}; diff --git a/scripts/build/flags.ts b/scripts/build/flags.ts index 03406ebcbd9c..9338cadf5334 100644 --- a/scripts/build/flags.ts +++ b/scripts/build/flags.ts @@ -18,7 +18,16 @@ import { quote, slash } from "./shell.ts"; import { loadFeatures } from "./trim/features.ts"; import { ucrtServicingLibDir } from "./winsysroot.ts"; -export const safeStackDependencies = ["brotli", "libdeflate", "libjpeg-turbo", "libspng", "libwebp", "lshpack", "zlib"]; +export const safeStackDependencies = [ + "brotli", + "libdeflate", + "libjpeg-turbo", + "libspng", + "libwebp", + "lshpack", + "zlib", + "zstd", +]; export function usesSafeStack(c: Pick): boolean { return c.hardened && c.x64 && c.abi === "gnu" && !c.asan; diff --git a/scripts/build/source.ts b/scripts/build/source.ts index 94f11e88bf5a..b6ab18c12177 100644 --- a/scripts/build/source.ts +++ b/scripts/build/source.ts @@ -487,7 +487,8 @@ export type DepName = | "rust-argon2" | "sqlite" | "tinycc" - | "zlib"; + | "zlib" + | "zstd"; /** * A vendored dependency definition. Lives in scripts/build/deps/.ts. From c788848768b4323f9e0502d476afe228de6e6d62 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 04:15:16 +0000 Subject: [PATCH 169/199] zstd: lazy and lazy2 share their code The loop of the lazy parsers was compiled for each depth, with each match finder and each kind of dictionary. Above 0 the depth only matters in one comparison, so it is now an argument: 14 instances and 83 KB of code less. `test/bundler/compile-aot-size.test.ts` had found that an executable maps more than it may. Time of before, 16 KB and 128 KB of three kinds of data: levels 6 to 8 take 0.97 to 0.99 at 128 KB and 0.97 to 1.01 at 16 KB; levels 5 and 13 are the same. --- src/zstd/compress/lazy.rs | 56 ++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 1215ec4ed23c..e1288bdf958a 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -2001,11 +2001,12 @@ impl Start { } } -/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `DEPTH` is how many positions -/// further a better match is looked for, once there is one. +/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `depth` is how many positions +/// further a better match is looked for, once there is one, and `LAZY` whether that is any. #[inline(never)] -fn compress_block_lazy_generic<'a, F: Finder<'a>, const DEPTH: u32, const DICT_MODE: u8>( +fn compress_block_lazy_generic<'a, F: Finder<'a>, const LAZY: bool, const DICT_MODE: u8>( args: LazyArgs<'a>, + depth: u32, ) -> Result { let LazyArgs { ms, @@ -2050,14 +2051,14 @@ fn compress_block_lazy_generic<'a, F: Finder<'a>, const DEPTH: u32, const DICT_M } // Check the repeat offset. - let mut best = Found::repeat(if DEPTH == 0 && no_repeat { + let mut best = Found::repeat(if !LAZY && no_repeat { 0 } else { rep_length::(block, ip + 1, offset_1)? }); let mut start = ip + 1; - if DEPTH > 0 || best.len == 0 { + if LAZY || best.len == 0 { // How many positions were searched since the last match that was taken, that one included. let mut since_best = 0; let mut wanted = Wanted { @@ -2067,13 +2068,13 @@ fn compress_block_lazy_generic<'a, F: Finder<'a>, const DEPTH: u32, const DICT_M loop { since_best += 1; - let found = if DEPTH > 0 { + let found = if LAZY { finder.find::(ip, wanted, skipping)? } else { finder.find::(ip, wanted, skipping)? }; if let Some(found) = found - && (DEPTH == 0 || found.gain() > wanted.gain) + && (!LAZY || found.gain() > wanted.gain) { best = found; start = ip; @@ -2095,7 +2096,7 @@ fn compress_block_lazy_generic<'a, F: Finder<'a>, const DEPTH: u32, const DICT_M } // Look for something better at the next positions. - if since_best > DEPTH || ip >= ilimit { + if !LAZY || since_best > depth || ip >= ilimit { break; } ip += 1; @@ -2189,16 +2190,19 @@ struct LazyArgs<'a> { /// [`compress_block_lazy_generic`], with the hash chains or the rows. #[inline(always)] -fn compress_block_lazy_hashed( +fn compress_block_lazy_hashed( args: LazyArgs<'_>, + depth: u32, use_row: bool, ) -> Result { if !use_row { if DICT_MODE == NO_DICT && chains_are_simple(args.ms, &args.block) { - return compress_block_lazy_generic::, DEPTH, NO_DICT>(args); + return compress_block_lazy_generic::, LAZY, NO_DICT>( + args, depth, + ); } - return compress_block_lazy_generic::, DEPTH, DICT_MODE>( - args, + return compress_block_lazy_generic::, LAZY, DICT_MODE>( + args, depth, ); } // With a dictionary, the second tags are given up: how many candidates there are matters if it is searched next. @@ -2206,17 +2210,19 @@ fn compress_block_lazy_hashed( row_flatten_tags(&mut args.ms.row_tags, &mut args.ms.tag_table)?; } match (args.ms.c_params.search_log.clamp(4, 6), args.ms.row_tags) { - (4, _) => { - compress_block_lazy_generic::, DEPTH, DICT_MODE>(args) - } - (5, _) => { - compress_block_lazy_generic::, DEPTH, DICT_MODE>(args) - } + (4, _) => compress_block_lazy_generic::, LAZY, DICT_MODE>( + args, depth, + ), + (5, _) => compress_block_lazy_generic::, LAZY, DICT_MODE>( + args, depth, + ), (_, RowTags::Flat) => { - compress_block_lazy_generic::, DEPTH, DICT_MODE>(args) + compress_block_lazy_generic::, LAZY, DICT_MODE>( + args, depth, + ) } (_, RowTags::Second(_)) => { - compress_block_lazy_generic::, DEPTH, NO_DICT>(args) + compress_block_lazy_generic::, LAZY, NO_DICT>(args, depth) } } } @@ -2229,14 +2235,14 @@ fn compress_block_lazy_any_strategy( use_row: bool, ) -> Result { match strategy { - Strategy::Greedy => compress_block_lazy_hashed::<0, DICT_MODE>(args, use_row), - Strategy::Lazy => compress_block_lazy_hashed::<1, DICT_MODE>(args, use_row), - Strategy::Lazy2 => compress_block_lazy_hashed::<2, DICT_MODE>(args, use_row), + Strategy::Greedy => compress_block_lazy_hashed::(args, 0, use_row), + Strategy::Lazy => compress_block_lazy_hashed::(args, 1, use_row), + Strategy::Lazy2 => compress_block_lazy_hashed::(args, 2, use_row), Strategy::BtLazy2 if !use_row && TreeInput::is_far(args.block.near) => { - compress_block_lazy_generic::, 2, DICT_MODE>(args) + compress_block_lazy_generic::, true, DICT_MODE>(args, 2) } Strategy::BtLazy2 if !use_row => { - compress_block_lazy_generic::, 2, DICT_MODE>(args) + compress_block_lazy_generic::, true, DICT_MODE>(args, 2) } _ => Err(BUG), } From 9b6d330381d9705456becb6861421628d93db21b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 04:15:16 +0000 Subject: [PATCH 170/199] zstd: fast and dfast are compiled for the sizes of tables only They were also compiled for each least length of a match. That only decides the factor of the hash, which can as well be in a register: what makes the instances faster is that the size of a table is known. 16 instances and 75 KB of code less, and a least length that no level comes with now gets the faster code too. Time of before: 0.998 over 36 rows of levels 1 to 4 (0.98 to 1.015), and 0.98 to 1.015 with a dictionary that is searched in place. --- src/zstd/compress/double_fast.rs | 73 ++++++++++++------------------- src/zstd/compress/fast.rs | 75 ++++++++++++-------------------- src/zstd/compress/internal.rs | 6 +++ test/js/bun/util/zstd.test.ts | 39 +++++++++++++++++ 4 files changed, 102 insertions(+), 91 deletions(-) diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index b0a4a032caee..219a00153617 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -90,9 +90,9 @@ impl Finder { /// The C code computes the long hash and reads its table one position ahead. Here that is left to the processor: a /// turn of the inner loop only takes the position and the 8 bytes there, `bytes0`, from the one before. /// - /// `MLS`, `LONG` and `SMALL` are what `small` and `long` are made with, or all 0 for any: see [`table_of_size`]. + /// `LONG` and `SMALL` are how many bits `long` and `small` give, or both 0 for any: see [`table_of_size`]. #[inline(never)] - fn find_matches( + fn find_matches( &mut self, hash_long: &mut [u32], hash_small: &mut [u32], @@ -109,11 +109,7 @@ impl Finder { } = *self; let near = sink.near; let long = if LONG > 0 { Hasher::new(LONG, 8) } else { long }; - let small = if SMALL > 0 { - Hasher::new(SMALL, MLS) - } else { - small - }; + let small = small.of_size::(); let Some(hash_long) = table_of_size::(hash_long) else { return; }; @@ -264,21 +260,19 @@ pub(crate) fn compress_block_double_fast( let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; // What the levels come with, for an input of any size. - match (ms.c_params.min_match, h_bits_l, h_bits_s) { - (5, 18, 18) => finder.find_matches::<5, 18, 18>(hash_long, hash_small, &mut sink), - (5, 17, 16) => finder.find_matches::<5, 17, 16>(hash_long, hash_small, &mut sink), - (5, 16, 15) => finder.find_matches::<5, 16, 15>(hash_long, hash_small, &mut sink), - (5, 14, 14) => finder.find_matches::<5, 14, 14>(hash_long, hash_small, &mut sink), - (4, 17, 17) => finder.find_matches::<4, 17, 17>(hash_long, hash_small, &mut sink), - (4, 17, 16) => finder.find_matches::<4, 17, 16>(hash_long, hash_small, &mut sink), - (4, 16, 16) => finder.find_matches::<4, 16, 16>(hash_long, hash_small, &mut sink), - (4, 16, 15) => finder.find_matches::<4, 16, 15>(hash_long, hash_small, &mut sink), - (4, 15, 14) => finder.find_matches::<4, 15, 14>(hash_long, hash_small, &mut sink), - (4, 14, 13) => finder.find_matches::<4, 14, 13>(hash_long, hash_small, &mut sink), - (4, 13, 12) => finder.find_matches::<4, 13, 12>(hash_long, hash_small, &mut sink), - (4, 12, 11) => finder.find_matches::<4, 12, 11>(hash_long, hash_small, &mut sink), - (4, 11, 10) => finder.find_matches::<4, 11, 10>(hash_long, hash_small, &mut sink), - _ => finder.find_matches::<0, 0, 0>(hash_long, hash_small, &mut sink), + match (h_bits_l, h_bits_s) { + (18, 18) => finder.find_matches::<18, 18>(hash_long, hash_small, &mut sink), + (17, 17) => finder.find_matches::<17, 17>(hash_long, hash_small, &mut sink), + (17, 16) => finder.find_matches::<17, 16>(hash_long, hash_small, &mut sink), + (16, 16) => finder.find_matches::<16, 16>(hash_long, hash_small, &mut sink), + (16, 15) => finder.find_matches::<16, 15>(hash_long, hash_small, &mut sink), + (15, 14) => finder.find_matches::<15, 14>(hash_long, hash_small, &mut sink), + (14, 14) => finder.find_matches::<14, 14>(hash_long, hash_small, &mut sink), + (14, 13) => finder.find_matches::<14, 13>(hash_long, hash_small, &mut sink), + (13, 12) => finder.find_matches::<13, 12>(hash_long, hash_small, &mut sink), + (12, 11) => finder.find_matches::<12, 11>(hash_long, hash_small, &mut sink), + (11, 10) => finder.find_matches::<11, 10>(hash_long, hash_small, &mut sink), + _ => finder.find_matches::<0, 0>(hash_long, hash_small, &mut sink), } let last_literals = sink.last_literals()?; restore_rep( @@ -294,7 +288,7 @@ pub(crate) fn compress_block_double_fast( impl Finder { /// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` #[inline(never)] - fn find_matches_dict( + fn find_matches_dict( &mut self, hash_long: &mut [u32], hash_small: &mut [u32], @@ -311,8 +305,8 @@ impl Finder { mut offset_2, } = *self; let near = window.near; + // As it is made, which the compiler does not see. let long = long.with_mls(8); - let small = small.with_mls(MLS); if hash_long.is_empty() || hash_small.is_empty() { return; } @@ -334,7 +328,7 @@ impl Finder { if near.has::<{ u64::MAX }>(match_long, bytes0) { break 'found window.found_near(ip, sink.anchor, match_long, 8); } - if let Some(place) = dict_tables.long.get::<8>(bytes0) + if let Some(place) = dict_tables.long.get(long, bytes0) && window.older.read64(place) == Some(bytes0) { break 'found window.found_older(ip, sink.anchor, place, 8); @@ -345,7 +339,7 @@ impl Finder { let pos = match_index_s - base_index; (near.read32(pos) == Some(bytes0 as u32)).then(|| window.place_of(pos)) } else { - (dict_tables.short.get::(bytes0)) + (dict_tables.short.get(small, bytes0)) .filter(|&place| window.older.read32(place) == Some(bytes0 as u32)) }; let Some(place_short) = short else { @@ -361,7 +355,7 @@ impl Finder { ip += 1; break 'found window.found_near(ip, sink.anchor, match_long1, 8); } - if let Some(place) = dict_tables.long.get::<8>(bytes1) + if let Some(place) = dict_tables.long.get(long, bytes1) && window.older.read64(place) == Some(bytes1) { ip += 1; @@ -459,12 +453,7 @@ pub(crate) fn compress_block_double_fast_dict_match_state( let sink = &mut MatchSink::with_near(seq_store, block.window.near, block.istart); let long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let small = ms.chain_table.sub_mut(..1 << h_bits_s)?; - match ms.c_params.min_match { - 5 => finder.find_matches_dict::<5>(long, small, sink, block.window, tables), - 6 => finder.find_matches_dict::<6>(long, small, sink, block.window, tables), - 7 => finder.find_matches_dict::<7>(long, small, sink, block.window, tables), - _ => finder.find_matches_dict::<4>(long, small, sink, block.window, tables), - } + finder.find_matches_dict(long, small, sink, block.window, tables); let last_literals = sink.last_literals()?; let [rep0, rep1, _] = rep; *rep0 = finder.offset_1; @@ -473,9 +462,9 @@ pub(crate) fn compress_block_double_fast_dict_match_state( } impl Finder { - /// `ZSTD_compressBlock_doubleFast_extDict_generic`. `MLS`, `LONG` and `SMALL` are as for [`Self::find_matches`]. + /// `ZSTD_compressBlock_doubleFast_extDict_generic`. `LONG` and `SMALL` are as for [`Self::find_matches`]. #[inline(never)] - fn find_matches_ext_dict( + fn find_matches_ext_dict( &mut self, hash_long: &mut [u32], hash_small: &mut [u32], @@ -495,11 +484,7 @@ impl Finder { // The index of the first place. let origin_index = base_index.wrapping_sub(older_len); let long = if LONG > 0 { Hasher::new(LONG, 8) } else { long }; - let small = if SMALL > 0 { - Hasher::new(SMALL, MLS) - } else { - small - }; + let small = small.of_size::(); let Some(hash_long) = table_of_size::(hash_long) else { return; }; @@ -623,10 +608,10 @@ pub(crate) fn compress_block_double_fast_ext_dict( let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; let (long, small, sink) = (hash_long, hash_small, &mut sink); // What the levels come with, for an input that is large enough to get here. - match (ms.c_params.min_match, h_bits_l, h_bits_s) { - (5, 18, 18) => finder.find_matches_ext_dict::<5, 18, 18>(long, small, sink, block.window), - (5, 17, 16) => finder.find_matches_ext_dict::<5, 17, 16>(long, small, sink, block.window), - _ => finder.find_matches_ext_dict::<0, 0, 0>(long, small, sink, block.window), + match (h_bits_l, h_bits_s) { + (18, 18) => finder.find_matches_ext_dict::<18, 18>(long, small, sink, block.window), + (17, 16) => finder.find_matches_ext_dict::<17, 16>(long, small, sink, block.window), + _ => finder.find_matches_ext_dict::<0, 0>(long, small, sink, block.window), } let last_literals = sink.last_literals()?; let [rep0, rep1, _] = rep; diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index eda9a2b24ae6..728fb6831d98 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -578,9 +578,9 @@ impl Finder { /// at `ip2` for a repeat. The C code computes hashes and reads the table one position ahead. Here that is left to /// the processor: a turn only takes the position and the 8 bytes there, `bytes0`, from the one before. /// - /// `MLS` and `HLOG` are what `hasher` is made with, or both 0 for any: see [`table_of_size`]. + /// `HLOG` is how many bits `hasher` gives, or 0 for any: see [`table_of_size`]. #[inline(never)] - fn find_matches( + fn find_matches( &mut self, hash_table: &mut [u32], sink: &mut MatchSink<'_, '_>, @@ -595,11 +595,7 @@ impl Finder { mut rep_offset2, } = *self; let near = sink.near; - let hasher = if HLOG > 0 { - Hasher::new(HLOG, MLS) - } else { - hasher - }; + let hasher = hasher.of_size::(); let Some(hash_table) = table_of_size::(hash_table) else { return; }; @@ -708,7 +704,7 @@ impl Finder { impl Finder { /// `ZSTD_compressBlock_fast_extDict_generic`, which goes about it as [`Self::find_matches`] does. #[inline(never)] - fn find_matches_ext_dict( + fn find_matches_ext_dict( &mut self, hash_table: &mut [u32], sink: &mut MatchSink<'_, '_>, @@ -727,11 +723,7 @@ impl Finder { let older_len = window.older.len(); // The index of the first place. let origin_index = base_index.wrapping_sub(older_len); - let hasher = if HLOG > 0 { - Hasher::new(HLOG, MLS) - } else { - hasher - }; + let hasher = hasher.of_size::(); let Some(hash_table) = table_of_size::(hash_table) else { return; }; @@ -863,22 +855,14 @@ pub(crate) fn compress_block_fast( let mut sink = MatchSink::new(seq_store, &block); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; // What the levels come with, for an input of any size. - match (ms.c_params.min_match, hlog) { - (7, 14) => finder.find_matches::<7, 14>(hash_table, &mut sink), - (6, 16) => finder.find_matches::<6, 16>(hash_table, &mut sink), - (6, 14) => finder.find_matches::<6, 14>(hash_table, &mut sink), - (6, 13) => finder.find_matches::<6, 13>(hash_table, &mut sink), - (5, 15) => finder.find_matches::<5, 15>(hash_table, &mut sink), - (5, 14) => finder.find_matches::<5, 14>(hash_table, &mut sink), - (5, 13) => finder.find_matches::<5, 13>(hash_table, &mut sink), - (5, 12) => finder.find_matches::<5, 12>(hash_table, &mut sink), - (5, 11) => finder.find_matches::<5, 11>(hash_table, &mut sink), - (4, 15) => finder.find_matches::<4, 15>(hash_table, &mut sink), - (4, 14) => finder.find_matches::<4, 14>(hash_table, &mut sink), - (4, 13) => finder.find_matches::<4, 13>(hash_table, &mut sink), - (4, 12) => finder.find_matches::<4, 12>(hash_table, &mut sink), - (4, 11) => finder.find_matches::<4, 11>(hash_table, &mut sink), - _ => finder.find_matches::<0, 0>(hash_table, &mut sink), + match hlog { + 16 => finder.find_matches::<16>(hash_table, &mut sink), + 15 => finder.find_matches::<15>(hash_table, &mut sink), + 14 => finder.find_matches::<14>(hash_table, &mut sink), + 13 => finder.find_matches::<13>(hash_table, &mut sink), + 12 => finder.find_matches::<12>(hash_table, &mut sink), + 11 => finder.find_matches::<11>(hash_table, &mut sink), + _ => finder.find_matches::<0>(hash_table, &mut sink), } let last_literals = sink.last_literals()?; restore_rep( @@ -943,11 +927,14 @@ impl<'a> TaggedTable<'a> { }) } - /// The place that the table has for the `MLS` first bytes of `bytes`, as many as it was made for. `None` if the tag - /// tells that it is for something else, or if it is the first place, which stands for none. + /// The place that the table has for `bytes`. `None` if the tag tells that it is for something else, or if it is the + /// first place, which stands for none. + /// + /// `hasher` takes as many bytes as the table was made for. It is that of the table of the input, so that one + /// multiplication does for both. #[inline(always)] - pub(super) fn get(self, bytes: u64) -> Option { - let hash_and_tag = Hasher::new(self.h_bits, MLS).hash(bytes); + pub(super) fn get(self, hasher: Hasher, bytes: u64) -> Option { + let hash_and_tag = hasher.with_log(self.h_bits).hash(bytes); let index_and_tag = hashed_get(self.table, hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS); if !compare_packed_tags(index_and_tag as usize, hash_and_tag) { return None; @@ -962,7 +949,7 @@ impl<'a> TaggedTable<'a> { impl Finder { /// `ZSTD_compressBlock_fast_dictMatchState_generic` #[inline(never)] - fn find_matches_dict( + fn find_matches_dict( &mut self, hash_table: &mut [u32], sink: &mut MatchSink<'_, '_>, @@ -979,7 +966,6 @@ impl Finder { mut rep_offset2, } = *self; let near = window.near; - let hasher = hasher.with_mls(MLS); if hash_table.is_empty() || dict_table.table.is_empty() { return; } @@ -1008,7 +994,7 @@ impl Finder { } // A match in the dictionary is only taken if there is no candidate in the prefix. - if let Some(place) = dict_table.get::(bytes0) + if let Some(place) = dict_table.get(hasher, bytes0) && window.older.read32(place) == Some(bytes0 as u32) && match_index <= base_index { @@ -1098,12 +1084,7 @@ pub(crate) fn compress_block_fast_dict_match_state( }; let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; - match ms.c_params.min_match { - 5 => finder.find_matches_dict::<5>(hash_table, &mut sink, block.window, dict_table), - 6 => finder.find_matches_dict::<6>(hash_table, &mut sink, block.window, dict_table), - 7 => finder.find_matches_dict::<7>(hash_table, &mut sink, block.window, dict_table), - _ => finder.find_matches_dict::<4>(hash_table, &mut sink, block.window, dict_table), - } + finder.find_matches_dict(hash_table, &mut sink, block.window, dict_table); let last_literals = sink.last_literals()?; let [rep0, rep1, _] = rep; *rep0 = finder.rep_offset1; @@ -1134,11 +1115,11 @@ pub(crate) fn compress_block_fast_ext_dict( let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; // What the levels come with, for an input that is large enough to get here. - match (ms.c_params.min_match, hlog) { - (7, 14) => finder.find_matches_ext_dict::<7, 14>(hash_table, &mut sink, block.window), - (6, 16) => finder.find_matches_ext_dict::<6, 16>(hash_table, &mut sink, block.window), - (6, 13) => finder.find_matches_ext_dict::<6, 13>(hash_table, &mut sink, block.window), - _ => finder.find_matches_ext_dict::<0, 0>(hash_table, &mut sink, block.window), + match hlog { + 16 => finder.find_matches_ext_dict::<16>(hash_table, &mut sink, block.window), + 14 => finder.find_matches_ext_dict::<14>(hash_table, &mut sink, block.window), + 13 => finder.find_matches_ext_dict::<13>(hash_table, &mut sink, block.window), + _ => finder.find_matches_ext_dict::<0>(hash_table, &mut sink, block.window), } let last_literals = sink.last_literals()?; restore_rep( diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index d0b5b9bd8ca7..0b0fadb8ce02 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1643,6 +1643,12 @@ impl Hasher { } } + /// The same for what [`table_of_size`] gives. + #[inline(always)] + pub(crate) const fn of_size(self) -> Self { + if BITS > 0 { self.with_log(BITS) } else { self } + } + /// The same of `mls` bytes. pub(crate) const fn with_mls(self, mls: u32) -> Self { Self { diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index fbd0fdd5dc8e..06211e075849 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1620,6 +1620,45 @@ describe("the output is that of zstd 1.5.7", () => { `); }); + it("with every least length of a match, at the strategies that hash as many bytes", () => { + const dictionary = sample(20_000, 4); + const short = corpus.slice(400_000, 405_000); + short.set(dictionary.subarray(-600, -100), 1_000); + const results: Record = {}; + for (const [name, strategy] of [ + ["fast", c.ZSTD_fast], + ["dfast", c.ZSTD_dfast], + ] as const) { + for (const minMatch of [3, 4, 5, 6, 7]) { + const params = { [c.ZSTD_c_strategy]: strategy, [c.ZSTD_c_minMatch]: minMatch }; + const tables = (hashLog: number) => ({ [c.ZSTD_c_hashLog]: hashLog, [c.ZSTD_c_chainLog]: hashLog - 1 }); + const compressed = [ + zlib.zstdCompressSync(large, { params: { ...params, ...tables(16) } }), + zlib.zstdCompressSync(large, { params: { ...params, ...tables(10) } }), + zlib.zstdCompressSync(short, { params, dictionary, pledgedSrcSize: short.length }), + ]; + results[`${name}, ${minMatch}`] = compressed.map(digest).join(", "); + expect(zlib.zstdDecompressSync(compressed[0]).equals(large)).toBe(true); + expect(zlib.zstdDecompressSync(compressed[1]).equals(large)).toBe(true); + expect(zlib.zstdDecompressSync(compressed[2], { dictionary }).equals(short)).toBe(true); + } + } + expect(results).toMatchInlineSnapshot(` + { + "dfast, 3": "44487 25e0622d3e12bfbd, 48050 558295cad240a785, 1005 858a418a2405d05a", + "dfast, 4": "44487 25e0622d3e12bfbd, 48050 558295cad240a785, 1005 858a418a2405d05a", + "dfast, 5": "44922 5901d333a7330ff9, 48936 4abcce7cd20aaf96, 1028 6aaf30fe0e09dd18", + "dfast, 6": "45343 386c1fe73dd038a9, 49635 2fa5fd5697de4709, 1032 46565d1807a0eaa6", + "dfast, 7": "46545 39a718fcbe68cb38, 51913 6367517af191ced2, 1089 8dde7dc61d61525b", + "fast, 3": "45605 bb8d5e0a2a6d89bc, 48410 b143a25a8491bf48, 1013 c8b68aa258a78281", + "fast, 4": "45605 bb8d5e0a2a6d89bc, 48410 b143a25a8491bf48, 1013 c8b68aa258a78281", + "fast, 5": "45771 fe9b40f28db9fd9, 49021 56c10ab35e811607, 1036 c3e106a367a73897", + "fast, 6": "46011 8660df15f934f2e1, 49559 90160f0d372cf91, 1040 78faf9395332bbf", + "fast, 7": "46936 560777669662bd37, 52350 c186154a65c5f997, 1088 538a72fd960bac79", + } + `); + }); + it("where a block is one match, which is too short for the optimal parser to take at once", () => { const digests = (data: Buffer, windowLog: number) => [16, 19, 22].map(level => { From 33a1c7b64d4a3ac177d8d3bbebf1d0f1ed1f616d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 05:47:47 +0000 Subject: [PATCH 171/199] zstd: literals that are stored or one byte are not written to a buffer that was just cleared A decoder clears the part that its buffer for literals gets longer by. For literals that are copied there or that are one byte many times, that part is now appended instead. Literals with a Huffman code are decoded into a slice, so for them nothing changes. It matters to a new decoder whose first block has many such literals and next to no sequences: 128 KB of noise with a period of 100 KB take 6.2 to 6.7 us to decode, from 7.0 to 7.5 (C 6.0 to 6.2), 256 KB 9.5 to 12.5, from 10.5 to 13.9 (C 9.1 to 10.3). Nothing else moves. --- src/zstd/decompress/block.rs | 42 ++++++++++++++++++++++++++---------- src/zstd/lib.rs | 39 +++++++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index 51c3800576f4..bf683537f68c 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -222,8 +222,8 @@ impl OutputBuffer<'_> { pub(crate) enum Growth { /// Zeros up to there, and somewhat further if `limit` allows. Zeros { limit: usize }, - /// What is written becomes its end, without being zeroed first. Only for output in one call, where nothing that - /// comes after what is being written is of any use. + /// What is written becomes its end, without being zeroed first. Only where nothing that comes after what is being + /// written is of any use, as in output in one call. Append, } @@ -331,19 +331,36 @@ impl Scratch { /// A thread keeps no more than that for its next decoder. const MAX_SPARE_LIT_BUFFER: usize = 1 << 14; - fn lit_buffer(&mut self, lit_size: usize) -> Result<&mut [u8]> { - let needed = lit_size + WILDCOPY_OVERLENGTH; + fn reserve_literals(&mut self, lit_size: usize) -> Result<()> { make_room( &mut self.lit_buffer, - needed, + lit_size + WILDCOPY_OVERLENGTH, ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH, - )?; + ) + } + + /// Makes `lit_buffer` long enough for `lit_size` literals. + fn extend_lit_buffer(&mut self, lit_size: usize) -> Result<()> { + let needed = lit_size + WILDCOPY_OVERLENGTH; + self.reserve_literals(lit_size)?; if self.lit_buffer.len() < needed { self.lit_buffer.resize(needed, 0); } // Literals are copied by more than there are: this much can be read along with the last ones. self.lit_buffer.sub_mut(lit_size..needed)?.fill(0); - self.lit_buffer.sub_mut(..lit_size) + Ok(()) + } + + /// [`Self::extend_lit_buffer`], for literals that `write` writes from the start of the buffer on, all `lit_size` of + /// them: what the buffer gets longer by for them is then not cleared first. + fn write_literals( + &mut self, + lit_size: usize, + write: impl FnOnce(OutputBuffer<'_>, Growth) -> Result<()>, + ) -> Result<()> { + self.reserve_literals(lit_size)?; + write(OutputBuffer::Used(&mut self.lit_buffer), Growth::Append)?; + self.extend_lit_buffer(lit_size) } } @@ -449,7 +466,7 @@ impl BlockDecoder { let huf_source = self.huf_source; let scratch = self.scratch(); - scratch.lit_buffer(lit_size)?; + scratch.extend_lit_buffer(lit_size)?; let Scratch { entropy, lit_buffer, @@ -525,9 +542,9 @@ impl BlockDecoder { if lit_size + lh_size > src.len() { return Err(Error::CorruptionDetected); } + let literals = src.span(lh_size, lit_size)?; self.scratch() - .lit_buffer(lit_size)? - .write_at(0, src.span(lh_size, lit_size)?)?; + .write_literals(lit_size, |mut to, growth| to.write(0, literals, growth))?; return Ok(Literals { c_size: lh_size + lit_size, size: lit_size, @@ -565,7 +582,10 @@ impl BlockDecoder { if expected_write_size < lit_size { return Err(Error::DstSizeTooSmall); } - self.scratch().lit_buffer(lit_size)?.fill(src.at(lh_size)?); + let byte = src.at(lh_size)?; + self.scratch().write_literals(lit_size, |mut to, growth| { + to.fill(0, byte, lit_size, growth) + })?; Ok(Literals { c_size: lh_size + 1, size: lit_size, diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 163ece33bc73..8a3d89b1f7b1 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -759,10 +759,14 @@ mod tests { Ok(()) } + const MAGIC: [u8; 4] = common::ZSTD_MAGICNUMBER.to_le_bytes(); + + fn le24(value: usize) -> [u8; 3] { + [value as u8, (value >> 8) as u8, (value >> 16) as u8] + } + #[test] fn output_does_not_catch_up_with_literals_that_would_be_in_its_buffer() { - const MAGIC: [u8; 4] = common::ZSTD_MAGICNUMBER.to_le_bytes(); - let le24 = |value: usize| [value as u8, (value >> 8) as u8, (value >> 16) as u8]; let size = |more: usize, nb_seq: usize| (1 << 16) + more + 1027 * nb_seq; // A last block: 64 KB and `more` times a byte, and sequences of 16 literals and 1027 bytes from 16 back. let block = |more: usize, nb_seq: usize| { @@ -824,6 +828,37 @@ mod tests { } } + #[test] + fn literals_of_any_size_follow_literals_of_any_size() -> Result<()> { + let noise: Vec = (0..8_000u32) + .map(|i| (i.wrapping_mul(i) >> 5) as u8) + .collect(); + // The buffer for them gets a new allocation of 72 bytes, of 532 and of 1064. It then has 552 bytes, which 540 + // fit in but for what is read along with them, and room for 800. The last one is of 7032 bytes, few enough for + // the next decoder of the thread to start with it, empty. + let sizes = [40, 500, 520, 540, 800, 7_000, 9, 100]; + let mut dst = vec![0; 1 << 13]; + for stored in [[true, true], [false, false], [true, false]] { + let mut dctx = DCtx::new(); + for ((&size, stored), nth) in sizes.iter().zip(stored.into_iter().cycle()).zip(1..) { + let expected = if stored { + noise.span(100 * nth, size)?.to_vec() + } else { + vec![size as u8; size] + }; + let bytes = expected.sub(..if stored { size } else { 1 })?; + // A window of 128 KB, and a last block without a sequence. + let literals = le24(usize::from(!stored) | 3 << 2 | size << 4); + let header = le24(1 | 2 << 1 | (literals.len() + bytes.len() + 1) << 3); + let frame = [&MAGIC[..], &[0, 7 << 3], &header, &literals, bytes, &[0]].concat(); + dst.fill(0xAA); + let written = dctx.decompress(&mut dst, &frame)?; + assert!(dst.sub(..written)? == expected, "{size}, {stored}"); + } + } + Ok(()) + } + #[test] fn output_that_is_appended_is_what_output_that_is_there_is() -> Result<()> { let mut state = 1u32; From a7fb6ebb7cc1ec3522ad7af11dbd59f502eb312c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 08:06:25 +0000 Subject: [PATCH 172/199] zstd: a round trip that fails is not taken for a parameter that is accepted `attempt` in zstd.test.ts gave the code of whatever was thrown, which a failed expectation has none of, and "the least and the most that parameters can be" took all that is not a string for success. Errors without a code are thrown again. From the same review: a stream is destroyed also if the expectation before fails, and a test looks at how much input the call that ends a frame takes. Two comments say where the port does as the C library does and a reader would not expect it: the shift by -1 for a code of 12 bits of a dictionary, and the limit on windows that a frame decoded in one go is not held to. --- src/zstd/compress/opt.rs | 3 ++- src/zstd/decompress/mod.rs | 3 ++- src/zstd/lib.rs | 4 +++- test/js/bun/util/zstd.test.ts | 10 +++++++--- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index abefa5abf7cf..7fd458c19481 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -120,7 +120,8 @@ fn freqs_from_bit_costs( let mut sum = 0u32; for (symbol, freq) in (0u32..).zip(freqs.sub_mut(..=last_elt_index as usize)?) { let bit_cost = bit_cost(symbol); - // At least 1, to have a cost. + // At least 1, to have a cost. A code of a dictionary can have one bit more than `scale_log`: the C code then + // shifts by -1, which x86-64 and aarch64 take for 31. *freq = if bit_cost != 0 { 1 << (scale_log.wrapping_sub(bit_cost) & 31) } else { diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs index 32a4c20abd4f..7259d5dffb11 100644 --- a/src/zstd/decompress/mod.rs +++ b/src/zstd/decompress/mod.rs @@ -1265,7 +1265,8 @@ impl DState { } } - // A frame that is all there, with room for all of its output, needs no buffer. + // A frame that is all there, with room for all of its output, needs no buffer, so the limit on windows + // does not apply to it, as in the C code. if self.f_params.frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN && self.f_params.frame_type != FrameType::Skippable && (oend - op) as u64 >= self.f_params.frame_content_size diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 8a3d89b1f7b1..7f102e7823f0 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -497,7 +497,9 @@ mod tests { } let result = feed(&mut stream, output, &[]); assert_eq!(result, Err(Error::NoForwardProgressInputEmpty)); - assert_eq!(feed(&mut stream, output, end), Ok(0)); + let mut input = InBuffer { src: end, pos: 0 }; + assert_eq!(stream.decompress_stream(output, &mut input), Ok(0)); + assert_eq!(input.pos, end.len()); assert_eq!(*output.dst.sub(..output.pos)?, *data); assert_eq!(feed(&mut stream, output, start), Ok(5)); diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 06211e075849..f7cf96d7dfde 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1989,6 +1989,7 @@ describe("bugs that zstd has had", () => { try { return Buffer.from(decompress()); } catch (error: any) { + if (typeof error.code !== "string") throw error; return error.code as string; } }; @@ -2184,9 +2185,12 @@ describe("bugs that zstd has had", () => { const { promise, resolve, reject } = Promise.withResolvers(); stream.once("data", chunk => resolve(chunk.length)).on("error", reject); stream.on("close", () => reject(new Error("closed before any data"))); - stream.write(frame.subarray(0, 5000), () => stream.flush()); - expect(await promise).toBeWithin(4000, 5000); - stream.destroy(); + try { + stream.write(frame.subarray(0, 5000), () => stream.flush()); + expect(await promise).toBeWithin(4000, 5000); + } finally { + stream.destroy(); + } }); const roundTrip = (input: Buffer, params: Record, dictionary?: Buffer) => { From 51386a66bf1b31090a640fe75472ac280c20531c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 08:39:46 +0000 Subject: [PATCH 173/199] build: --local-deps=zstd is taken again where zstd is built zstd left `allDeps` with bun's own use of it, and the check of the names given to --local-deps reads that list, so the flag was refused as an unknown dep in the one build that still compiles zstd: the jsc shell of --webkit=local with BUN_ICU_ROOT. `programDeps` lists what only programs beside bun take. The graph, the check of --local-deps and `clean deep` read it beside `allDeps`, and jsc.cpp finds zstd's headers where its sources are compiled from. zstd is no longer compiled with SafeStack: only bun has the second stack, so a hardened jsc could not be linked with it. --- scripts/build/CLAUDE.md | 4 +-- scripts/build/bun.ts | 12 ++++----- scripts/build/clean.ts | 8 +++--- scripts/build/config.ts | 2 +- scripts/build/deps/README.md | 4 +-- scripts/build/deps/index.ts | 11 +++++++- scripts/build/deps/webkit.ts | 4 +-- scripts/build/deps/zstd.ts | 4 +-- scripts/build/flags.ts | 11 +------- test/internal/build-hardened-flags.test.ts | 6 ++++- test/internal/build-local-webkit.test.ts | 29 ++++++++++++++++++++++ 11 files changed, 64 insertions(+), 31 deletions(-) diff --git a/scripts/build/CLAUDE.md b/scripts/build/CLAUDE.md index 7db4f9552380..73ae8554b158 100644 --- a/scripts/build/CLAUDE.md +++ b/scripts/build/CLAUDE.md @@ -174,7 +174,7 @@ For `mode: "full"` (the normal case): 1. **Codegen** — `emitCodegen(n, cfg, sources, engine)` emits ~20 generation steps (bindgen, bundled modules, LUTs). Returns grouped outputs. 2. **Rust** — `emitRust(n, cfg, {...})` emits one rustc edge per crate and returns the crates' rlibs, `bun_runtime`'s and std's included, which the link takes beside the C/C++ objects (after resolving the vendored path deps, lolhtml and rust-argon2). cargo plans, ninja executes: the `rust_plan` edge runs `cargo build … --unit-graph` + `cargo metadata` for exactly the arguments `cargoBuildInvocation()` computes and writes `rust/plan.json`; `build.ninja` depends on that file, so a new plan (lockfile/manifest/toolchain change) reconfigures and ninja restarts. `rust/units.ts` turns each unit into a rustc argv/env (cargo's rules, transcribed and diffed against `cargo -vv`), written to `rust/units/-.json`; `rust/emit.ts` emits the edges; `rust/run.ts` executes them. Every edge is one process from start to exit, and a crate is one edge: one rustc with the `.rlib` and the `.rmeta` as outputs. Dependent libraries name only the `.rmeta`, which rustc writes long before it has generated code; the edge carries `early_output_prefix`, which asks ninja to release an output when the running command announces it. oven-sh/ninja (what the driver runs, see `ninja-release.ts`) exports the prefix to the command as `NINJA_EARLY_OUTPUT_PREFIX`, `rust/run.ts` prints it with the `.rmeta`'s path when rustc reports the file written, and dependents start while rustc goes on — cargo's pipelining. A stock ninja ignores the binding and releases both outputs at exit: the same graph, built correctly, without the overlap. Build scripts are a compile edge plus a `rust_build_script` run edge whose parsed directives (`output.json`, restat) feed the package's rustc edges. A Windows target has a second graph built the same way, under `rust/shim/`: the `.bin/` launcher (`src/install/windows-shim`), planned with `--profile shim` and `-Zbuild-std`, with a `bin` root that `run.ts` also writes to `/bun-shim-impl.exe`, where `bun_install` embeds it from (every edge of that package waits for it). Codegen and Rust are emitted before the deps on purpose: with no `.ninja_log` (every CI build) ninja weighs each edge as 1 and runs the longest remaining chain first, ties in emission order. -3. **Deps** — loop `allDeps`, call `resolveDep(n, cfg, dep)`. Each emits fetch → direct cc/cxx edges, or fetch → cargo, or prebuilt download. Collects objects, lib paths, include dirs, outputs. +3. **Deps** — loop `allDeps`, call `resolveDep(n, cfg, dep)`. Each emits fetch → direct cc/cxx edges, or fetch → cargo, or prebuilt download. Collects objects, lib paths, include dirs, outputs. `programDeps` are resolved the same way, but nothing of them is collected for bun: only the programs beside it that name them take them. 4. **Flags** — `computeFlags(cfg)` evaluates flag tables → cflags/cxxflags/defines/ldflags/stripflags. 5. **PCH** — compile `root-pch.h` → PCH. 6. **Compile** — loop sources, `cxx()`/`cc()` per file. @@ -259,7 +259,7 @@ It is configured by `configureCodegen()`, not `configure()`, and resolves a `Cod | `glob-sources.ts` (parent dir) | Source glob patterns + CLI to print them | | `ci-images/spec.ts` | CI's machines in one file: images, version pins, locations, every tool, and the generator of the bake scripts; `bun run ci:images` | | `deps/*.ts` | One `Dependency` object per vendored dep | -| `deps/index.ts` | `allDeps` array — fetch order + link order | +| `deps/index.ts` | `allDeps` array — fetch order + link order; `programDeps` | | `shims/*.c` | Platform workaround sources | | `trim/*.ts` | The trimmable link: section graph, partition by feature, layout, the trim table (`link.ts` is the entry) | diff --git a/scripts/build/bun.ts b/scripts/build/bun.ts index 5e7e65b69f27..a4b90196818e 100644 --- a/scripts/build/bun.ts +++ b/scripts/build/bun.ts @@ -19,11 +19,10 @@ import { emitCodegen, type CodegenOutputs } from "./codegen.ts"; import { cc, cxx, link, pch, type PrecompiledHeader } from "./compile.ts"; import { bunExeName, shouldStrip, type Config } from "./config.ts"; import { generateDepVersionsHeader } from "./depVersionsHeader.ts"; -import { allDeps } from "./deps/index.ts"; +import { allDeps, programDeps } from "./deps/index.ts"; import { lolhtml } from "./deps/lolhtml.ts"; import { rustArgon2 } from "./deps/rust-argon2.ts"; import { jscPrograms, linuxIcuRoot, webkitEngineSources } from "./deps/webkit.ts"; -import { zstd } from "./deps/zstd.ts"; import { assert } from "./error.ts"; import { bunIncludes, @@ -198,9 +197,10 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput { depsByName.set(dep.name, resolved); } } - // For a program beside bun, so not one of `deps`. - const zstdDep = resolveDep(n, cfg, zstd, depsByName); - if (zstdDep !== null) depsByName.set(zstd.name, zstdDep); + for (const dep of programDeps) { + const resolved = resolveDep(n, cfg, dep, depsByName); + if (resolved !== null) depsByName.set(dep.name, resolved); + } // Collect all dep lib paths, include dirs, output stamps, and directly- // compiled source files (deps like picohttpparser that provide .c files @@ -1132,7 +1132,7 @@ export function validateBunConfig(cfg: Config): void { // --local-deps names must match a dep — a typo would otherwise silently // build the pinned tarball while the banner claims `local:`. // Keyed by string: these names are the user's. - const depsByName = new Map(allDeps.map(d => [d.name, d])); + const depsByName = new Map([...allDeps, ...programDeps].map(d => [d.name, d])); for (const [name, path] of Object.entries(cfg.localDeps)) { const dep = depsByName.get(name); assert(dep !== undefined, `--local-deps: unknown dep '${name}'`, { diff --git a/scripts/build/clean.ts b/scripts/build/clean.ts index 5d31524245bc..325f2ca5511c 100644 --- a/scripts/build/clean.ts +++ b/scripts/build/clean.ts @@ -10,7 +10,7 @@ import { rm } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { sharedCacheDir as resolveSharedCacheDir } from "./config.ts"; -import { allDeps } from "./deps/index.ts"; +import { allDeps, programDeps } from "./deps/index.ts"; const cwd = resolve(dirname(fileURLToPath(import.meta.url)), "..", ".."); // Machine-shared cache (ccache/cargo/tarballs/webkit): resolveConfig()'s @@ -56,8 +56,8 @@ function buildProfiles(): string[] { const profile = (name: string) => () => [resolve(cwd, "build", name)]; // Deps whose vendor// dir is user-managed (manual clone, not fetched -// by the build system). `deep` skips these; everything else in allDeps gets -// its vendor dir nuked. +// by the build system). `deep` skips these; every other dep gets its vendor +// dir nuked. const userManagedDeps = new Set(["WebKit"]); const presets: Record string[]> = { @@ -81,7 +81,7 @@ const presets: Record string[]> = { deep: () => [ resolve(cwd, "build"), resolve(cwd, "target"), - ...allDeps.filter(d => !userManagedDeps.has(d.name)).map(d => resolve(cwd, "vendor", d.name)), + ...[...allDeps, ...programDeps].filter(d => !userManagedDeps.has(d.name)).map(d => resolve(cwd, "vendor", d.name)), ], }; diff --git a/scripts/build/config.ts b/scripts/build/config.ts index 74db7251f04a..541a2a2f22be 100644 --- a/scripts/build/config.ts +++ b/scripts/build/config.ts @@ -1751,7 +1751,7 @@ export function findRepoRoot(): string { /** * Parse `--local-deps=name=path[,name=path...]` into name → absolute path. - * Names are checked against `allDeps` later (bun.ts) where the dep list is + * Names are checked against the dep lists later (bun.ts) where they are * in scope; here we only validate shape and resolve paths. */ function parseLocalDeps(spec: string | undefined, cwd: string): Record { diff --git a/scripts/build/deps/README.md b/scripts/build/deps/README.md index d723d28870ea..bb758b224670 100644 --- a/scripts/build/deps/README.md +++ b/scripts/build/deps/README.md @@ -8,7 +8,7 @@ libraries/headers it provides. 1. Copy `libdeflate.ts` (the simplest direct dep) to `.ts` 2. Fill in `name`, `repo`, `commit`, `sources`, `includes`, `provides.includes` -3. Add `import { } from "./.ts"` + entry in `allDeps` array in `index.ts` +3. Add `import { } from "./.ts"` + entry in `allDeps` array in `index.ts` (`programDeps` if only a program beside bun takes it, which then names it in its `JSCProgram.deps`) 4. Add the name to `DepName` in `../source.ts` 5. `bun run scripts/build/phase3-test.ts` to verify it builds @@ -28,7 +28,7 @@ Case-sensitive filesystems enforce this. ## Removing a dependency 1. Delete `.ts` -2. Remove from `allDeps` in `index.ts`, and from `DepName` in `../source.ts` +2. Remove from `allDeps` (or `programDeps`) in `index.ts`, and from `DepName` in `../source.ts` 3. If any other dep has `fetchDeps: [""]`, remove that reference ## Updating a commit diff --git a/scripts/build/deps/index.ts b/scripts/build/deps/index.ts index 766a89d65b11..80568e10db90 100644 --- a/scripts/build/deps/index.ts +++ b/scripts/build/deps/index.ts @@ -32,9 +32,10 @@ import { sqlite } from "./sqlite.ts"; import { tinycc } from "./tinycc.ts"; import { webkit } from "./webkit.ts"; import { zlib } from "./zlib.ts"; +import { zstd } from "./zstd.ts"; /** - * All deps in dependency-resolution + link order. + * All deps of bun in dependency-resolution + link order. * * zlib FIRST — libspng's and lsquic's fetchDeps reference it. * brotli libs in internal dep order (common last on link line). @@ -72,6 +73,13 @@ export const allDeps: readonly Dependency[] = [ webkit, ]; +/** + * What only programs beside bun take (`JSCProgram.deps`). Fetched, compiled and + * redirected (`--local-deps`) like the others, but no part of bun: not linked + * into it, no include dir of its sources, no entry of its versions. + */ +export const programDeps: readonly Dependency[] = [zstd]; + // Re-export individuals for direct import when needed. export { boringssl, @@ -95,4 +103,5 @@ export { tinycc, webkit, zlib, + zstd, }; diff --git a/scripts/build/deps/webkit.ts b/scripts/build/deps/webkit.ts index 0ce4b50106cb..6647c535addf 100644 --- a/scripts/build/deps/webkit.ts +++ b/scripts/build/deps/webkit.ts @@ -1903,7 +1903,7 @@ export interface JSCProgram { cxxflags: string[]; ldflags: string[]; libs?: string[]; - /** Dependencies of bun that it is linked with, beside WebKit and mimalloc. */ + /** Of `programDeps`: what it is linked with, beside WebKit and mimalloc. */ deps?: Array<"zstd">; /** * Headers to precompile for the program's sources, with its flags, each on top of the one before. A source takes @@ -1998,7 +1998,7 @@ export function jscPrograms(cfg: Config): JSCProgram[] { // The data of the ICU of a prebuilt WebKit is a zstd frame an item (icu/compress-data.ts). jsc.cpp has the hook that // reads it, as shell/CMakeLists.txt builds it with BUN_ICU_ZSTD_SOURCE_DIR. if (linuxIcuRoot(cfg) !== undefined) { - jscShell.cxxflags = [...jscShell.cxxflags, "-DBUN_ICU_ZSTD=1", `-I${join(cfg.vendorDir, "zstd", "lib")}`]; + jscShell.cxxflags = [...jscShell.cxxflags, "-DBUN_ICU_ZSTD=1", `-I${wk.q(join(depSourceDir(cfg, "zstd"), "lib"))}`]; jscShell.deps = ["zstd"]; } return [ diff --git a/scripts/build/deps/zstd.ts b/scripts/build/deps/zstd.ts index 50ae5f61d45a..886ef5480637 100644 --- a/scripts/build/deps/zstd.ts +++ b/scripts/build/deps/zstd.ts @@ -5,8 +5,8 @@ * WebKit's jsc.cpp reads them with this library, as in WebKit's own build * (shell/CMakeLists.txt, BUN_ICU_ZSTD_SOURCE_DIR). * - * No other build fetches or compiles it, and it is not in `allDeps`: nothing of - * it reaches bun, whose zstd is src/zstd. + * No other build fetches or compiles it, and it is one of `programDeps`, not of + * `allDeps`: nothing of it reaches bun, whose zstd is src/zstd. */ import type { Dependency } from "../source.ts"; diff --git a/scripts/build/flags.ts b/scripts/build/flags.ts index 9338cadf5334..03406ebcbd9c 100644 --- a/scripts/build/flags.ts +++ b/scripts/build/flags.ts @@ -18,16 +18,7 @@ import { quote, slash } from "./shell.ts"; import { loadFeatures } from "./trim/features.ts"; import { ucrtServicingLibDir } from "./winsysroot.ts"; -export const safeStackDependencies = [ - "brotli", - "libdeflate", - "libjpeg-turbo", - "libspng", - "libwebp", - "lshpack", - "zlib", - "zstd", -]; +export const safeStackDependencies = ["brotli", "libdeflate", "libjpeg-turbo", "libspng", "libwebp", "lshpack", "zlib"]; export function usesSafeStack(c: Pick): boolean { return c.hardened && c.x64 && c.abi === "gnu" && !c.asan; diff --git a/test/internal/build-hardened-flags.test.ts b/test/internal/build-hardened-flags.test.ts index 1113a93ebb60..ee0d1b7b0ad5 100644 --- a/test/internal/build-hardened-flags.test.ts +++ b/test/internal/build-hardened-flags.test.ts @@ -4,6 +4,7 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; import { resolveConfig, type Config, type PartialConfig, type Toolchain } from "../../scripts/build/config.ts"; +import { programDeps } from "../../scripts/build/deps/index.ts"; import { mimalloc } from "../../scripts/build/deps/mimalloc.ts"; import { webkit } from "../../scripts/build/deps/webkit.ts"; import { fetchCliPath } from "../../scripts/build/fetch-cli.ts"; @@ -295,7 +296,10 @@ test("the libraries that never call back into JavaScript keep their arrays on a expect(safeStackFlags(on, name)).toEqual(["-fsanitize=safe-stack"]); expect(safeStackFlags(off, name)).toEqual([]); } - for (const name of [...librariesThatCallBackIntoJavaScript, "mimalloc", "WebKit"]) + // The programs beside bun have no second stack. + const forOtherPrograms = programDeps.map(dep => dep.name); + expect(forOtherPrograms).toContain("zstd"); + for (const name of [...librariesThatCallBackIntoJavaScript, "mimalloc", "WebKit", ...forOtherPrograms]) expect(safeStackFlags(on, name)).toEqual([]); expect(everyFlagOf(on).cxx).toContain("-DBUN_SAFE_STACK=1"); expect(everyFlagOf(on).dependencyC).not.toContain("-fsanitize=safe-stack"); diff --git a/test/internal/build-local-webkit.test.ts b/test/internal/build-local-webkit.test.ts index 141a7bb9c67e..3c61965302b2 100644 --- a/test/internal/build-local-webkit.test.ts +++ b/test/internal/build-local-webkit.test.ts @@ -8,6 +8,7 @@ import { isWindows, tempDir } from "harness"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join, relative, resolve } from "node:path"; +import { validateBunConfig } from "../../scripts/build/bun.ts"; import { cxx } from "../../scripts/build/compile.ts"; import { resolveConfig, type Config, type PartialConfig, type Toolchain } from "../../scripts/build/config.ts"; import { @@ -20,10 +21,12 @@ import { webkitTestAotPath, type WebKitAotFile, } from "../../scripts/build/deps/webkit.ts"; +import { zstd } from "../../scripts/build/deps/zstd.ts"; import { BuildError } from "../../scripts/build/error.ts"; import { Ninja, readManifest } from "../../scripts/build/ninja.ts"; import { getProfile } from "../../scripts/build/profiles.ts"; import { registerAllRules } from "../../scripts/build/rules.ts"; +import { quote } from "../../scripts/build/shell.ts"; import { depSource, resolveDep, @@ -733,6 +736,32 @@ describe.skipIf(Bun.which(python) === null)("the graph of a checkout", () => { for (const source of program.sources) cxx(n, cfg, source, { flags: program.cxxflags }); } }); + + test("--local-deps=zstd= is for the jsc that reads the ICU of a prebuilt WebKit, as only that needs zstd", () => { + using local = checkOut({}, checkout); + const path = resolve("/else where", "zstd"); + const { buildDir, localDeps } = local.cfg; + const cfg = linux({ localDeps: `WebKit=${localDeps.WebKit},zstd=${path}`, buildDir }); + const flagsOfJsc = () => jscPrograms(cfg).find(program => program.name === "jsc")!.cxxflags; + const headersOf = (sourceDir: string) => `-I${quote(join(sourceDir, "lib"), isWindows)}`; + const saved = process.env.BUN_ICU_ROOT; + try { + process.env.BUN_ICU_ROOT = join("/somewhere", "icu"); + validateBunConfig(cfg); + expect(depSource(cfg, zstd)).toMatchObject({ kind: "local", path }); + expect(flagsOfJsc()).toContain(headersOf(path)); + expect(flagsOfJsc()).not.toContain(headersOf(join(cfg.vendorDir, "zstd"))); + expect(() => validateBunConfig(linux({ localDeps: `zstd=${path}`, buildDir }))).toThrow( + /--local-deps: zstd is disabled/, + ); + delete process.env.BUN_ICU_ROOT; + expect(() => validateBunConfig(cfg)).toThrow(/--local-deps: zstd is disabled/); + expect(flagsOfJsc()).not.toContain(headersOf(path)); + } finally { + if (saved === undefined) delete process.env.BUN_ICU_ROOT; + else process.env.BUN_ICU_ROOT = saved; + } + }); }); describe("DirectBuild groups and steps", () => { From dd89d787607494139b8645fd16a45553ffcb9dd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 08:52:23 +0000 Subject: [PATCH 174/199] zstd: the contexts of workers are in a pool, as in the C code Each worker had a context of its own, on the ground that which context a job goes to is a matter of timing in the C library. It is not where jobs do not overlap: ZSTDMT_getCCtx takes the context that was given back last, and a job gives its context back before it reports that it is over. So frames of one job, a stream that is flushed after each write, and anything with a single worker go through one and the same context. What that context has been through shows once about 3.4 GB have gone through it: where its indices stand decides where they are brought down, which lets go of an attached dictionary. The frames were valid, but not those of C: with one worker after a change of parameters that makes C allocate a new workspace, and with any number of workers where jobs do not overlap, as each context had seen only a share of the input. The contexts are now in a pool of the MtCtx: last given back, first taken, at the start of a job; given back before the job reports its end; kept when there are fewer workers, new when there are more than ever. They keep the books of the C workspace like any other context, so that their indices start again where C makes a new workspace, and the rule that workers had of their own for tables that are too large is gone. Small jobs all go to the context that had the first job of the frame, as in C, so a larger job than those before still does not clear the tables. It does with a dictionary, whose first job has the small tables of the dictionary, as in C: 300 flushes that grow, at level 19, take 290 ms where they took 20, and 16 s in C. --- src/zstd/compress/cctx.rs | 166 ++++++++++++++++++------------- src/zstd/compress/internal.rs | 4 - src/zstd/compress/mt.rs | 178 +++++++++++++++++++++++++--------- 3 files changed, 230 insertions(+), 118 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 171dbddb698f..157048c23a4f 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -13,7 +13,7 @@ use super::fse::FseRepeat; use super::huf::{self, HufRepeat}; use super::internal::{ BlockSrc, BufId, Buffers, CompressedBlockState, CompressionParameters, DictMatchState, - EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, Match, MatchState, OptState, + EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, MatchState, OptState, ParamSwitch, Prefix, PriceTable, RawSeq, RawSeqStore, RowTags, SeqChunk, SeqStore, Strategy, Table, TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, block_header, min_gain, @@ -34,7 +34,7 @@ use crate::common::{ BlockType, LITBITS, LL_FSE_LOG, MAX_LL, MAX_ML, MAX_OFF, MIN_CBLOCK_SIZE, ML_FSE_LOG, OFF_FSE_LOG, ZSTD_BLOCKHEADERSIZE, ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_FRAMEHEADERSIZE_MAX, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGICNUMBER, ZSTD_REP_NUM, - ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, + ZSTD_WINDOWLOG_ABSOLUTEMIN, }; use crate::error::{Error, Result}; use crate::util::{ @@ -280,34 +280,6 @@ fn nb_opt_positions(block_size: usize) -> usize { ZSTD_OPT_SIZE.min(block_size + 2) } -/// What the tables of the optimal parser take for a position: a `Match`, and 7 numbers in the `PriceTable`. -const OPT_POSITION_SIZE: usize = size_of::() + 7 * size_of::(); - -/// How many bytes the tables of a context take. -fn tables_size( - c_params: &CompressionParameters, - use_row_match_finder: ParamSwitch, - block_size: usize, -) -> usize { - let sizes = MatchStateSizes::new(c_params, use_row_match_finder, true); - let mut size = (sizes.h_size + sizes.chain_size + sizes.h3_size) * size_of::(); - if row_match_finder_used(c_params.strat(), use_row_match_finder) { - size += sizes.h_size; - } - if c_params.strat() >= Strategy::BtOpt { - size += nb_opt_positions(block_size) * OPT_POSITION_SIZE; - } - size -} - -/// How many bytes the tables of `ms` hold on to, in use or not. -fn tables_capacity(ms: &MatchState) -> usize { - (ms.hash_table.capacity() + ms.chain_table.capacity() + ms.hash_table3.capacity()) - * size_of::() - + ms.tag_table.capacity() - + ms.opt.match_table.capacity() * OPT_POSITION_SIZE -} - /// `ZSTD_indexTooCloseToMax`: starting a frame this close to the limit would mean bringing indices down during it. fn index_too_close_to_max(w: &Window) -> bool { const ZSTD_INDEXOVERFLOW_MARGIN: usize = 16 << 20; @@ -857,10 +829,7 @@ pub(crate) struct FrameCompressor { pub(crate) applied_params: CCtxParams, pub(crate) dict_id: u32, pub(crate) dict_content_size: usize, - /// `None` for a worker, see [`Self::for_worker`]. - workspace: Option, - /// For a worker: for how many frames in a row the tables have been much larger than it takes. - oversized_duration: usize, + workspace: Workspace, pub(crate) block_size_max: usize, pub(crate) pledged_src_size_plus_one: u64, pub(crate) consumed_src_size: u64, @@ -906,8 +875,7 @@ impl FrameCompressor { applied_params: CCtxParams::new(0), dict_id: 0, dict_content_size: 0, - workspace: Some(Workspace::new()), - oversized_duration: 0, + workspace: Workspace::new(), block_size_max: 0, pledged_src_size_plus_one: 0, consumed_src_size: 0, @@ -925,13 +893,9 @@ impl FrameCompressor { } } - /// For the jobs of one worker. Which context a job goes to is a matter of timing in the C code, so what its contexts - /// have been through cannot show in the output. + /// For jobs of workers, which stop if `cancelled` is set. pub(crate) fn for_worker(cancelled: Arc) -> Self { - let mut worker = Self { - workspace: None, - ..Self::new() - }; + let mut worker = Self::new(); worker.match_state.cancelled = Some(cancelled); worker } @@ -981,18 +945,16 @@ impl FrameCompressor { let index_too_close = index_too_close_to_max(&self.match_state.window); let dict_too_big = loaded_dict_size > ZSTD_CHUNKSIZE_MAX as usize; // Where indices start shows in where they get to the limit, which is where a dictionary is let go of. - let new_workspace = self.workspace.as_mut().is_some_and(|workspace| { - if !initialized { - *workspace = Workspace::new(); - } - workspace.reset(&WorkspaceNeeds { - c_params: ¶ms.c_params, - ldm_params: ¶ms.ldm_params, - use_row_match_finder: params.use_row_match_finder, - buff_in_size: sizes.buff_in_size, - buff_out_size: sizes.buff_out_size, - block_size, - }) + if !initialized { + self.workspace = Workspace::new(); + } + let new_workspace = self.workspace.reset(&WorkspaceNeeds { + c_params: ¶ms.c_params, + ldm_params: ¶ms.ldm_params, + use_row_match_finder: params.use_row_match_finder, + buff_in_size: sizes.buff_in_size, + buff_out_size: sizes.buff_out_size, + block_size, }); let needs_index_reset = if index_too_close || dict_too_big || !initialized || new_workspace { @@ -1002,21 +964,7 @@ impl FrameCompressor { }; // A context does not hold on for long to much more than it uses. - let new_tables = if self.workspace.is_some() { - new_workspace - } else { - let needed = tables_size(¶ms.c_params, params.use_row_match_finder, block_size); - let oversized = tables_capacity(&self.match_state) - >= needed.saturating_mul(ZSTD_WORKSPACETOOLARGE_FACTOR); - self.oversized_duration = if oversized { - self.oversized_duration + 1 - } else { - 0 - }; - self.oversized_duration > ZSTD_WORKSPACETOOLARGE_MAXDURATION - }; - if new_tables { - self.oversized_duration = 0; + if new_workspace { let ms = &mut self.match_state; ms.hash_table = Table::new(); ms.chain_table = Table::new(); @@ -1925,3 +1873,83 @@ impl FrameCompressor { Ok(c_size + end_result) } } + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::common::ZSTD_WORKSPACETOOLARGE_MAXDURATION; + use crate::compress::params::{FrameParameters, Parameters}; + + /// With tables that are as small as can be, but for the hash table. + pub(crate) fn small_params(strategy: Strategy, hash_log: u32) -> CCtxParams { + let params = Parameters { + c_params: CompressionParameters { + window_log: 10, + chain_log: 6, + hash_log, + search_log: 1, + min_match: 4, + target_length: 0, + strategy: Some(strategy), + }, + f_params: FrameParameters { + content_size_flag: false, + checksum_flag: false, + no_dict_id_flag: false, + }, + }; + CCtxParams::from_params(¶ms, 0) + } + + /// It takes 3.5 GB through a context for where its indices stand to show in a frame. + #[test] + fn indices_start_again_where_the_c_code_makes_a_new_workspace() -> Result<()> { + const JOB_SIZE: u32 = 64; + let data = [0u8; JOB_SIZE as usize]; + let input = Input { + bufs: Buffers { + in_buff: &[], + user: &data, + dict: &[], + }, + cdict: None, + }; + let src = Span { + buf: BufId::User, + start: 0, + end: data.len(), + }; + let begin = |cctx: &mut FrameCompressor, hash_log| { + let params = small_params(Strategy::Fast, hash_log); + cctx.begin(&input, None, ¶ms, u64::from(JOB_SIZE), false) + }; + // Returns the index that the next byte would get. + let compress = |cctx: &mut FrameCompressor, hash_log| -> Result { + begin(cctx, hash_log)?; + cctx.compress_continue(&mut [0u8; 128], &input, src)?; + let window = &cctx.match_state.window; + Ok(window.index_of(window.next_src)) + }; + let after_jobs = |jobs: u32| ZSTD_WINDOW_START_INDEX + jobs * JOB_SIZE; + + let worker = FrameCompressor::for_worker(Arc::new(AtomicBool::new(false))); + for mut cctx in [FrameCompressor::new(), worker] { + assert_eq!(compress(&mut cctx, 6)?, after_jobs(1)); + assert_eq!(compress(&mut cctx, 6)?, after_jobs(2)); + // Too small. + assert_eq!(compress(&mut cctx, 15)?, after_jobs(1)); + // Much too large, for too long. + for _ in 1..ZSTD_WORKSPACETOOLARGE_MAXDURATION { + begin(&mut cctx, 6)?; + } + assert_eq!(compress(&mut cctx, 6)?, after_jobs(2)); + assert_eq!(compress(&mut cctx, 6)?, after_jobs(1)); + // As large as it takes, for however long. + for _ in 0..=ZSTD_WORKSPACETOOLARGE_MAXDURATION { + begin(&mut cctx, 6)?; + } + assert_eq!(compress(&mut cctx, 6)?, after_jobs(2)); + } + Ok(()) + } +} diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 0b0fadb8ce02..c122a3c44d30 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -2091,10 +2091,6 @@ impl Table { self.storage.truncate(self.start + self.len); } - pub(crate) fn capacity(&self) -> usize { - self.storage.capacity() - } - #[inline] pub(crate) fn as_slice(&self) -> &[T] { let table = self.storage.get(self.start..self.start + self.len); diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 9891ed621ae1..0c98cc88cf10 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -123,6 +123,49 @@ impl SerialState { } } +/// `ZSTDMT_CCtxPool`. A job takes the context that was given back last, so jobs that come one after the other all have +/// the same one, and what it has been through shows in their output, see [`Workspace`](super::params::Workspace). +#[derive(Clone)] +struct CCtxPool { + /// `totalCCtx` + total: usize, + /// Holds the contexts that are free, but for while someone looks at them. + free: (Sender>, Receiver>), +} + +impl CCtxPool { + /// `ZSTDMT_createCCtxPool` + fn new(total: usize) -> Result { + let mut contexts = Vec::new(); + (contexts.try_reserve_exact(total)).map_err(|_| Error::MemoryAllocation)?; + let free = channel(); + let _ = free.0.send(contexts); + Ok(Self { total, free }) + } + + fn with_free(&self, look: impl FnOnce(&mut Vec) -> R) -> R { + let mut contexts = self.free.1.recv().unwrap_or_default(); + let result = look(&mut contexts); + let _ = self.free.0.send(contexts); + result + } + + /// `ZSTDMT_getCCtx` + fn take(&self, cancelled: &Arc) -> FrameCompressor { + self.with_free(Vec::pop) + .unwrap_or_else(|| FrameCompressor::for_worker(Arc::clone(cancelled))) + } + + /// `ZSTDMT_releaseCCtx` + fn give_back(&self, cctx: FrameCompressor) { + self.with_free(|contexts| { + if contexts.len() < self.total { + contexts.push(cctx); + } + }); + } +} + /// What a job tells about itself. enum Progress { /// Some more output, and how much of the input is consumed by now. @@ -155,15 +198,19 @@ struct Job { /// What a worker keeps from one job to the next. struct WorkerState { - cctx: FrameCompressor, seq: Vec, dst: Vec, } const CHUNK_SIZE: usize = 4 * ZSTD_BLOCKSIZE_MAX; -/// `ZSTDMT_compressionJob` -fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) -> Result<()> { +/// `ZSTDMT_compressionJob`, up to `_endJob`. Returns the end of the output. +fn compression_job( + state: &mut WorkerState, + cctx: &mut FrameCompressor, + job: &Job, + cancelled: &AtomicBool, +) -> Result> { let mut job_params = job.params; let src_size = job.data.len() - job.prefix_size; let src = job.data.sub(job.prefix_size..)?; @@ -205,7 +252,6 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - }, cdict, }; - let cctx = &mut state.cctx; if cdict.is_some() { cctx.begin(&input, None, &job_params, job.full_frame_size, false)?; } else { @@ -272,20 +318,27 @@ fn compression_job(state: &mut WorkerState, job: &Job, cancelled: &AtomicBool) - cctx.compress_continue(&mut state.dst, &input, chunk)? }; } - send(state.dst.sub(..last_c_block_size)?, src_size) + try_copy(state.dst.sub(..last_c_block_size)?) } -fn worker_main(jobs: &Receiver, cancelled: &Arc) { +fn worker_main(jobs: &Receiver, cctx_pool: &CCtxPool, cancelled: &Arc) { let mut state = WorkerState { - cctx: FrameCompressor::for_worker(Arc::clone(cancelled)), seq: Vec::new(), dst: Vec::new(), }; while let Ok(job) = jobs.recv() { - if let Err(code) = compression_job(&mut state, &job, cancelled) { - let _ = job.progress.send(Progress::Failed(code)); - } - state.seq = state.cctx.take_external_sequences(); + let mut cctx = cctx_pool.take(cancelled); + let end = compression_job(&mut state, &mut cctx, &job, cancelled); + state.seq = cctx.take_external_sequences(); + // First, so that a job that is created once this one is known to be over finds its context. + cctx_pool.give_back(cctx); + let _ = job.progress.send(match end { + Ok(data) => Progress::Output { + data, + consumed: job.data.len() - job.prefix_size, + }, + Err(code) => Progress::Failed(code), + }); } } @@ -356,12 +409,12 @@ impl PendingJob { /// `ZSTDMT_CCtx` pub(crate) struct MtCtx { - /// They are on their own: they share nothing, and leave once nothing can be sent to their queue. + /// The workers have it too. Before `queue`, so that it is the last of them to leave that frees the contexts. + cctx_pool: CCtxPool, + /// They are on their own, and leave once nothing can be sent to their queue. nb_workers: usize, /// Whichever worker is free takes the next job, so that a job does not wait for one thread to be scheduled. queue: Sender, - /// The other end, for workers to come. - queue_end: Receiver, cancelled: Arc, /// The jobs from `doneJobID` to `nextJobID`. jobs: VecDeque, @@ -426,11 +479,10 @@ fn compute_overlap_size(params: &CCtxParams) -> usize { impl MtCtx { /// `ZSTDMT_createCCtx_advanced` pub(crate) fn new(nb_workers: u32) -> Result { - let (queue, queue_end) = channel(); let mut mtctx = Self { + cctx_pool: CCtxPool::new(0)?, nb_workers: 0, - queue, - queue_end, + queue: channel().0, cancelled: Arc::new(AtomicBool::new(false)), jobs: VecDeque::new(), params: CCtxParams::new(0), @@ -454,18 +506,24 @@ impl MtCtx { /// `ZSTDMT_resize`: no job is under way. fn resize(&mut self, nb_workers: usize) -> Result<()> { - if nb_workers < self.nb_workers { - // They leave when nothing can be sent to the queue they have. - (self.queue, self.queue_end) = channel(); - self.nb_workers = 0; - } + // `ZSTDMT_expandCCtxPool` + if nb_workers > self.cctx_pool.total { + // Now, and not when the last of the workers has left: the new ones are to have the memory. + self.cctx_pool.with_free(Vec::clear); + self.cctx_pool = CCtxPool::new(nb_workers)?; + } + // The workers there are leave, as nothing can be sent to the queue they have. + let queue_end; + (self.queue, queue_end) = channel(); + self.nb_workers = 0; while self.nb_workers < nb_workers { - let queue_end = self.queue_end.clone(); + let queue_end = queue_end.clone(); + let cctx_pool = self.cctx_pool.clone(); let cancelled = Arc::clone(&self.cancelled); std::thread::Builder::new() .name("zstd".into()) .stack_size(2 << 20) - .spawn(move || worker_main(&queue_end, &cancelled)) + .spawn(move || worker_main(&queue_end, &cctx_pool, &cancelled)) .map_err(|_| Error::MemoryAllocation)?; self.nb_workers += 1; } @@ -798,30 +856,60 @@ impl Drop for MtCtx { #[cfg(test)] mod tests { use super::*; - use crate::compress::internal::{CompressionParameters, Strategy}; - use crate::compress::params::{FrameParameters, Parameters}; + use crate::common::ZSTD_CONTENTSIZE_UNKNOWN; + use crate::compress::cctx::tests::small_params; + + #[test] + fn the_context_that_was_given_back_last_is_taken_first() -> Result<()> { + let cancelled = Arc::new(AtomicBool::new(false)); + let pool = CCtxPool::new(2)?; + let mut contexts = [(); 3].map(|()| pool.take(&cancelled)); + for (dict_id, cctx) in (1..).zip(&mut contexts) { + cctx.dict_id = dict_id; + } + // One more than there are workers, which does not happen. + contexts.into_iter().for_each(|cctx| pool.give_back(cctx)); + assert_eq!([(); 3].map(|()| pool.take(&cancelled).dict_id), [2, 1, 0]); + Ok(()) + } + + #[test] + fn jobs_that_come_one_after_the_other_have_the_same_context() -> Result<()> { + let mut mtctx = MtCtx::new(1)?; + // A frame in three jobs, each of which is over before the next one is created. + let compress = |mtctx: &mut MtCtx, nb_workers| -> Result<()> { + let mut params = small_params(Strategy::Fast, 6); + params.nb_workers = nb_workers; + mtctx.init_c_stream_internal(None, params, ZSTD_CONTENTSIZE_UNKNOWN)?; + for end_op in [EndDirective::Flush, EndDirective::Flush, EndDirective::End] { + let (mut dst, src) = ([0u8; 64], [0u8; 16]); + let mut output = OutBuffer { + dst: &mut dst, + pos: 0, + }; + let mut input = InBuffer { src: &src, pos: 0 }; + while mtctx.compress_stream_generic(&mut output, &mut input, end_op)? != 0 {} + } + Ok(()) + }; + let nb_contexts = |mtctx: &MtCtx| mtctx.cctx_pool.with_free(|contexts| contexts.len()); + + compress(&mut mtctx, 4)?; + compress(&mut mtctx, 4)?; + assert_eq!(nb_contexts(&mtctx), 1); + // With fewer workers the C code keeps its contexts, with more than ever it starts with new ones. + compress(&mut mtctx, 2)?; + assert_eq!((nb_contexts(&mtctx), mtctx.cctx_pool.total), (1, 4)); + let mut params = small_params(Strategy::Fast, 6); + params.nb_workers = 5; + mtctx.init_c_stream_internal(None, params, 0)?; + assert_eq!((nb_contexts(&mtctx), mtctx.cctx_pool.total), (0, 5)); + Ok(()) + } #[test] fn a_worker_stops_at_a_prefix_or_a_block_of_a_frame_that_is_given_up() { - let params = CCtxParams::from_params( - &Parameters { - c_params: CompressionParameters { - window_log: 10, - chain_log: 6, - hash_log: 6, - search_log: 1, - min_match: 4, - target_length: 0, - strategy: Some(Strategy::BtOpt), - }, - f_params: FrameParameters { - content_size_flag: false, - checksum_flag: false, - no_dict_id_flag: false, - }, - }, - 0, - ); + let params = small_params(Strategy::BtOpt, 6); let data = [0u8; 64]; let input = Input { bufs: Buffers { From 79995be3f35cad0ad4bb61fcbeb02c6deba71fad Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 09:28:40 +0000 Subject: [PATCH 175/199] test: one shape of build-scaling grows 2.58 times in memory The change of #47329, as it is there, so that it merges with it. test/aot/build-scaling/build-scaling-2.test.ts failed on macOS 26 arm64 with "twice as many functions cost 2.59 times the peakBytes; allowed: 2.5". It does on main too (2.56 to 2.67 in the four attempts of build 135281), where it otherwise passes with 2.40 to 2.48. On Linux, where the growth in memory is not judged, this branch and the commit of main that it starts from take the same instructions to 0.01% and the same memory. --- test/aot/build-scaling/budgets.linux-x64.json | 1 + test/aot/build-scaling/build-scaling-cases.ts | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/aot/build-scaling/budgets.linux-x64.json b/test/aot/build-scaling/budgets.linux-x64.json index a85a8eb4fe23..168d20d8311b 100644 --- a/test/aot/build-scaling/budgets.linux-x64.json +++ b/test/aot/build-scaling/budgets.linux-x64.json @@ -349,6 +349,7 @@ "megabytes": 164, "growth": { "instructions": 3.31, + "peakBytes": 2.58, "counts": { "HandedOnReferencesRead": 4, "FunctionsBelowEntrancesInCyclesVisited": 4 diff --git a/test/aot/build-scaling/build-scaling-cases.ts b/test/aot/build-scaling/build-scaling-cases.ts index cc068cc7dd8a..21e938a6883b 100644 --- a/test/aot/build-scaling/build-scaling-cases.ts +++ b/test/aot/build-scaling/build-scaling-cases.ts @@ -331,9 +331,12 @@ export const buildScaling = (part: number, parts: number) => const fastCounts = growthOfCounts.flatMap(([name, times]) => times > mostGrowth ? [[name, rounded(times)]] : [], ); + // The bounds hold on every platform, so a growth that this machine does not judge stays as the file has it. + const overTheBoundOrKept = (what: keyof typeof growth) => + growth[what] === undefined ? budget?.growth?.[what] : overTheBound(growth[what]); const fast = { - instructions: overTheBound(growth.instructions), - peakBytes: overTheBound(growth.peakBytes), + instructions: overTheBoundOrKept("instructions"), + peakBytes: overTheBoundOrKept("peakBytes"), counts: fastCounts.length ? (Object.fromEntries(fastCounts) as Record) : undefined, }; const orNothing = (growth: NonNullable) => From 41fad94a3bad11c226983c9e01d58f26436bf49e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 16:24:54 +0000 Subject: [PATCH 176/199] zstd: records that differ in a field no longer have every candidate compared in full A search beside a match that the parser already has looks at a candidate 4 bytes at a time first, where a match that is good enough would end. With records that differ in one field, that is past the field, where all records are the same again: every candidate passed and was then compared from its start to the field, up to 64 of them for each search. The C code compares the first one, and from then on looks where that one ended. The first candidate of 16 bytes or more that is not long enough is now taken all the same, which puts the 4 bytes where the C code has them. The parser drops it: by its gain, or, in the first search at a position, for the repeat at the next position, which is at least as long. Time against the C library, before and after: records of 30 bytes and a counter, 1 MB at level 12: 1.81, 0.99; of 50 bytes, 16 KB at level 8: 1.48, 0.92; of 100 bytes, 1 MB at level 12: 1.87, 1.05; of 1000 bytes: 2.19, 0.86. Text, JSON and binaries of 16 KB at levels 5 to 8 take 1 to 2% longer than before (0.70 to 0.95 of the C library), larger ones the same. --- src/zstd/compress/lazy.rs | 99 +++++++++++++++++++++++++++++------ test/js/bun/util/zstd.test.ts | 61 +++++++++++++++++++++ 2 files changed, 144 insertions(+), 16 deletions(-) diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index e1288bdf958a..077e969f348b 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -1,6 +1,6 @@ //! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. -use core::hint::{Locality, prefetch_read}; +use core::hint::{Locality, cold_path, prefetch_read}; use core::num::NonZeroU32; use core::ops::Range; @@ -568,7 +568,7 @@ impl Found { } } -/// What a match has to be like for the parser to take it. +/// What a match has to be like to be better than what the parser has. #[derive(Clone, Copy)] struct Wanted { /// It is longer than this, which is at least 3. @@ -621,6 +621,10 @@ impl Best { } } +/// What [`Near::count_short`] compares at first. From there on, it took more than a glance to find out that a candidate +/// is not long enough. +const LONG_WAY: u32 = 16; + /// The longest match so far for a position, while candidates are compared with it. struct Longest<'a> { /// The current segment, from the lowest index that a match can be at or from its start. @@ -629,7 +633,7 @@ struct Longest<'a> { start_index: u32, /// The position that matches are for, in `segment`. ip: u32, - /// A match has to be more than 3 bytes longer than this. + /// A match has to be more than 3 bytes longer than this, unless it is the first and [`LONG_WAY`] bytes. tail_start: u32, /// `segment` from `tail_start` on. tails: Near<'a>, @@ -660,20 +664,33 @@ impl<'a> Longest<'a> { #[inline(always)] fn offer_match(&mut self, len: u32, off_base: u32) -> Offered { if len > self.tail_start + 3 { - self.tail_start = len - 3; - self.off_base = off_base; - let (Some(tails), Some(tail)) = ( - self.segment.tail(self.tail_start), - self.segment.read32(self.ip + self.tail_start), - ) else { - return Offered::Longest; - }; - self.tails = tails; - self.tail = tail; + return self.take(len, off_base); + } + // Records that differ in one field are the same again after it. If that is where the tail is, it tells none of + // them from a longer match. Where this one ends does, as in the C code: it is taken, short as it is. + if len >= LONG_WAY && self.off_base == 0 { + cold_path(); + return self.take(len, off_base); } Offered::Considered } + /// From now on, a match has to be longer than `len`. + #[inline(always)] + fn take(&mut self, len: u32, off_base: u32) -> Offered { + self.tail_start = len - 3; + self.off_base = off_base; + let (Some(tails), Some(tail)) = ( + self.segment.tail(self.tail_start), + self.segment.read32(self.ip + self.tail_start), + ) else { + return Offered::Longest; + }; + self.tails = tails; + self.tail = tail; + Offered::Considered + } + /// There may be a match for the position `ip` of `block` at `match_pos` in its dictionary. #[inline(always)] fn offer_in_dict( @@ -717,6 +734,7 @@ impl<'a> Longest<'a> { self.offer_match(len, offset_to_offbase(self.ip.wrapping_sub(match_pos))) } + /// It may be shorter than what [`Self::new`] was given. #[inline(always)] const fn best(&self) -> Best { Best { @@ -752,9 +770,9 @@ trait Finder<'a>: Sized { /// `block` is at least `MARGIN` bytes. fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result; - /// The longest match for `ip`, the most recent one of that length, unless it is not as `wanted`. Then it may be - /// `None` just as well. Positions are searched in increasing order. While `skipping`, those in between are not - /// indexed. + /// The longest match for `ip`, the most recent one of that length, if it is as `wanted`. If it is not, `None` or any + /// match of at least 4 bytes that is not either. Positions are searched in increasing order. While `skipping`, those + /// in between are not indexed. /// `BY_GAIN` is whether `wanted.gain` may be something else than [`Wanted::ANY_GAIN`]. fn find( &mut self, @@ -2073,9 +2091,14 @@ fn compress_block_lazy_generic<'a, F: Finder<'a>, const LAZY: bool, const DICT_M } else { finder.find::(ip, wanted, skipping)? }; + // At first any gain will do, that of a match that is not wanted too: one that is no longer than the repeat + // at the next position. It loses to that repeat at the next step. if let Some(found) = found && (!LAZY || found.gain() > wanted.gain) { + debug_assert!( + found.len > wanted.longer_than || (LAZY && depth > 0 && ip < ilimit) + ); best = found; start = ip; since_best = 1; @@ -2276,3 +2299,47 @@ pub(crate) fn compress_block_lazy( } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_long_candidate_that_falls_short_is_what_the_next_ones_have_to_beat() -> Result<()> { + const RECORD: u32 = 200; + let best = |longest: &Longest<'_>| (longest.best().len, longest.best().off_base); + + for field in [LONG_WAY - 1, LONG_WAY, 100] { + let mut bytes: Vec = (0..5 * RECORD).map(|n| (n % RECORD) as u8).collect(); + for (record, value) in (0..).zip([250, 251, 252, 250, 253]) { + *bytes.at_mut((record * RECORD + field) as usize)? = value; + } + // The tail is after the field, where all records are the same. + let floor = Best { + len: field + 50, + off_base: 0, + }; + let mut longest = Longest::new(Near::new(&bytes)?, 0, 3 * RECORD, floor).ok_or(BUG)?; + let up_to_the_field = if field >= LONG_WAY { + (field, offset_to_offbase(RECORD)) + } else { + (floor.len, 0) + }; + for record in [2, 1] { + assert!(longest.offer::(record * RECORD) == Offered::Considered); + assert_eq!(best(&longest), up_to_the_field); + } + if field > LONG_WAY { + assert!(longest.offer_match(field - 1, 1) == Offered::Considered); + assert_eq!(best(&longest), up_to_the_field); + } + + let past_the_field = (RECORD + field, offset_to_offbase(3 * RECORD)); + assert!(longest.offer::(0) == Offered::Considered); + assert_eq!(best(&longest), past_the_field); + assert!(longest.offer_match(RECORD, 1) == Offered::Considered); + assert_eq!(best(&longest), past_the_field); + } + Ok(()) + } +} diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index f7cf96d7dfde..883bec619959 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1501,6 +1501,67 @@ describe("the output is that of zstd 1.5.7", () => { ]); }); + it("where records differ in a few fields", () => { + // What is before a position ends at a field, and may be the same again after it. + const piece = (n: number, size: number) => corpus.subarray(500_000 + n * size, 500_000 + (n + 1) * size); + const counted = (size: number, count: number) => + Buffer.concat( + Array.from({ length: count }, (_, i) => [Buffer.from(`${i}`.padStart(8, "0")), piece(0, size)]).flat(), + ); + const flags: Uint8Array[] = []; + for (let i = 0, x = 7; i < 2_400; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + flags.push(Buffer.from(`${x & 1}`), piece(i % 4, 80)); + } + const flagged = Buffer.concat(flags); + const level = (level: number) => ({ [c.ZSTD_c_compressionLevel]: level }); + const dictionary = flagged.subarray(0, 30_000); + const cases: [Buffer, zlib.ZstdOptions][] = [ + // Hash chains. + [flagged.subarray(0, 16_000), { params: level(5) }], + [flagged.subarray(0, 16_000), { params: level(6) }], + [flagged.subarray(0, 16_000), { params: level(7) }], + [counted(100, 148), { params: level(8) }], + // Rows. From level 11 on, it is binary trees at these sizes. + [flagged, { params: level(6) }], + [flagged, { params: level(7) }], + [flagged, { params: level(9) }], + [flagged, { params: level(10) }], + [counted(100, 2_000), { params: level(10) }], + [counted(1_000, 200), { params: level(10) }], + [flagged, { params: { [c.ZSTD_c_strategy]: c.ZSTD_greedy } }], + // Rows of 64 that fill up. + [flagged, { params: { [c.ZSTD_c_strategy]: c.ZSTD_lazy2, [c.ZSTD_c_searchLog]: 6, [c.ZSTD_c_hashLog]: 12 } }], + // What is before the window is left behind, and what is in the dictionary is not: hash chains, then rows. + [flagged, { params: { ...level(8), [c.ZSTD_c_windowLog]: 12 } }], + [flagged, { params: { ...level(8), [c.ZSTD_c_windowLog]: 15 } }], + [flagged.subarray(30_000, 40_000), { params: { ...level(7), [c.ZSTD_c_windowLog]: 14 }, dictionary }], + [flagged.subarray(30_000, 40_000), { params: level(7), dictionary }], + ]; + expect(cases.map(([data, options]) => digest(zlib.zstdCompressSync(data, options)))).toMatchInlineSnapshot(` + [ + "249 f907d3f525b7aa81", + "243 de9085e2cdfac856", + "235 d670b708be78ef4c", + "225 854a885c1d8be4a5", + "1692 1de698fb782b56f9", + "1616 66239976b21d3320", + "1517 f00cc0a5a2e99559", + "1476 a94b01a6e016ad19", + "2460 ecb935f7ffd006fb", + "626 ee78b0ff57f42d3e", + "1497 d88328c87f134e60", + "1540 80ad275d9b5cea5a", + "2457 a55760737c90d6d0", + "1727 77731bfed55e726", + "104 61304d0540ed07df", + "115 f5c7d2c89a55ace1", + ] + `); + }); + it("where matches are far between, for the optimal parsers", () => { const data = Buffer.alloc(100_000); let x = 9; From 605aa6e576ce4702e154712eed136970ac37d636 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 17:45:46 +0000 Subject: [PATCH 177/199] zstd: a single Huffman stream of short codes is no longer left to the careful decoder The quick decoder of a single stream stopped 16 bytes before the start of the stream, and what is left has to fit 128 symbols. A valid stream of 129 to 183 symbols of about a bit each was too short for the one and too long for the other, so it went to the decoder that mirrors the C code step by step, which fills a table of its own for every block. Blocks of records with few literals are like that. The loop now goes on for as long as there are 8 bytes to read, which leaves 119 bits at most; a constant assertion ties that to the 128 symbols. The careful decoder's table is now one array of the kind that is asked for, filled in place: 8 or 16 KB and not 24 KB that were copied twice, and a table can only be given to the decoder that it was made for. Decoding, time of the port / time of C zstd 1.5.7, before -> after: records of 1 KB, 1 MB: level 6 2.19 -> 1.20, level 17 2.45 -> 1.22 records of 100 B, 128 KB: level 13 1.86 -> 1.08 records of 100 B, 1 MB: level 17 1.95 -> 1.22 Text of 1 to 16 KB is as before (0.88 to 0.95). --- src/zstd/decompress/huf.rs | 104 +++++++++++++++++++++++-- src/zstd/decompress/huf_exact.rs | 125 ++++++++++++++++++------------- 2 files changed, 169 insertions(+), 60 deletions(-) diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs index f17f549463c6..05e3970b21dc 100644 --- a/src/zstd/decompress/huf.rs +++ b/src/zstd/decompress/huf.rs @@ -632,6 +632,7 @@ impl Tail { /// No less than a round can read, and 8 bytes. const PADDING: usize = 16; const MAX_SRC: usize = 64; + /// No less than [`Stream::decode_most`] can leave of a stream that is valid, at a bit a symbol. const MAX_SYMBOLS: usize = 128; const fn new() -> Self { @@ -818,6 +819,9 @@ impl<'a> Stream<'a> { /// Decodes until the stream is close to its start, or its output to its end. fn decode_most(&mut self, table: &[Entry; TABLE_SIZE], log: u32) { const LOOKUPS: usize = 4; + /// How many bytes are read at a time. + const READ: usize = size_of::(); + const { assert!(8 * (READ - 1) + 63 <= Tail::MAX_SYMBOLS) }; if self.bits == 0 { return; } @@ -827,7 +831,8 @@ impl<'a> Stream<'a> { let consumed = self.bits.trailing_zeros(); // Where the bytes after those that `bits` was read from end. let mut next = self.ip as usize; - let Some(Ok(container)) = self.src.get(next..next + 8).map(<[u8; 8]>::try_from) else { + let Some(Ok(container)) = (self.src.get(next..next + READ)).map(<[u8; READ]>::try_from) + else { return; }; // The next bits, from the highest on: `valid` of them, or somewhat more. @@ -836,8 +841,8 @@ impl<'a> Stream<'a> { let mut op = self.op as usize; while let Some(window) = (self.out.get_mut(op..op + 3 * LOOKUPS + 1)) .and_then(|window| window.as_mut_array::<{ 3 * LOOKUPS + 1 }>()) - && next >= 16 - && let Some(Ok(more)) = self.src.get(next - 8..next).map(<[u8; 8]>::try_from) + && next >= READ + && let Some(Ok(more)) = (self.src.get(next - READ..next)).map(<[u8; READ]>::try_from) { bits |= u64::from_le_bytes(more) >> (valid & 63); let nb_bytes = (63 - (valid & 63)) >> 3; @@ -857,11 +862,9 @@ impl<'a> Stream<'a> { op += done; } let position = 8 * next + valid as usize; - let end = position.div_ceil(8); - if let Some(Ok(container)) = - (self.src.get(end.wrapping_sub(8)..end)).map(<[u8; 8]>::try_from) - { - self.ip = (end - 8) as u32; + let end = position.div_ceil(8).max(READ); + if let Some(Ok(container)) = (self.src.get(end - READ..end)).map(<[u8; READ]>::try_from) { + self.ip = (end - READ) as u32; self.bits = (u64::from_le_bytes(container) | 1) << ((8 * end - position) & 63); self.op = op as u32; } @@ -1255,4 +1258,89 @@ mod tests { assert!(changes.as_flattened().iter().all(|&count| count > 0)); Ok(()) } + + #[test] + fn a_single_stream_of_short_codes_is_not_left_to_the_careful_way() -> Result<()> { + // Up to as many as the header of a block can tell of a single stream. + let sizes: &[usize] = if cfg!(miri) { + &[129, 255] + } else { + &Vec::from_iter(1..=1023) + }; + // The first symbol takes a bit. The last ones take 3 bits, or 11. + let codes: [&[u32]; 2] = [ + &[4000, 3, 2, 1], + &[1024, 512, 256, 128, 64, 32, 16, 8, 4, 2, 1, 1], + ]; + for counts in codes { + let max_symbol = counts.len() - 1; + let mut ctable = HufCTable::new(); + let table_log = build_ctable(&mut ctable, counts, max_symbol as u32, 11)?; + let mut description = [0; NB_SYMBOLS]; + let size = write_ctable( + &mut description, + &ctable, + max_symbol as u32, + table_log as u32, + )?; + let description = description.sub(..size)?; + let mut dtable = HufDTable::new(); + let read = dtable.read(description, TableType::X1, 255, false, &mut Spread::new()); + assert_eq!(read, Ok(description.len())); + + for (log, single) in [ + (dtable.log, false), + (dtable.log, true), + (FAST_LOG + 1, false), + ] { + dtable.build(log, single)?; + for (&nb_literals, rarity) in sizes + .iter() + .flat_map(|size| [(size, usize::MAX), (size, 50)]) + { + let literals: Vec = (1..=nb_literals) + .map(|n| { + if n % rarity == 0 { + (n % max_symbol) as u8 + 1 + } else { + 0 + } + }) + .collect(); + let mut stream = vec![0; 2 * nb_literals + 8]; + let size = compress_1x_using_ctable(&mut stream, &literals, &ctable)?; + stream.truncate(size); + let mut decoded = vec![0; nb_literals]; + assert!(dtable.decode_1x(&mut decoded, &stream), "{nb_literals}"); + assert_eq!(decoded, literals); + + if cfg!(miri) { + continue; + } + // Streams that do not end where they should. + let step = if nb_literals < 300 { 5 } else { 8 * size }; + for bit in (0..8 * size).step_by(step) { + let mut damaged = stream.clone(); + *damaged.at_mut(bit / 8)? ^= 1 << (bit % 8); + for table_type in [TableType::X1, TableType::X2] { + dtable.table_type = table_type; + let mut exact = vec![0; nb_literals]; + let result = huf_exact::decompress_1x( + &mut exact, + &damaged, + &dtable.code, + table_type, + ); + assert_eq!( + decompress_1x_using_dtable(&mut decoded, &damaged, &dtable), + result + ); + assert!(result.is_err() || decoded == exact); + } + } + } + } + } + Ok(()) + } } diff --git a/src/zstd/decompress/huf_exact.rs b/src/zstd/decompress/huf_exact.rs index 1effda19066f..c5b9be77c3e3 100644 --- a/src/zstd/decompress/huf_exact.rs +++ b/src/zstd/decompress/huf_exact.rs @@ -22,22 +22,18 @@ pub(super) enum TableType { } /// `HUF_DTable`, of capacity `ZSTD_HUFFDTABLE_CAPACITY_LOG`. -struct ExactTable { - table_type: TableType, +struct ExactTable { table_log: u32, - /// `HUF_DEltX1`: the number of bits in the low byte, and the symbol above it. - x1: [u16; TABLE_SIZE], - /// `HUF_DEltX2`: two symbols, then the number of bits, then how many of the symbols count. - x2: [u32; TABLE_SIZE], + /// - `u16`, `HUF_DEltX1`: the number of bits in the low byte, and the symbol above it. + /// - `u32`, `HUF_DEltX2`: two symbols, then the number of bits, then how many of the symbols count. + entries: [Entry; TABLE_SIZE], } -impl ExactTable { - const fn new() -> Self { +impl ExactTable { + fn new() -> Self { Self { - table_type: TableType::X1, table_log: 0, - x1: [0; TABLE_SIZE], - x2: [0; TABLE_SIZE], + entries: [Entry::default(); TABLE_SIZE], } } } @@ -234,7 +230,7 @@ fn rescale_stats( } /// `HUF_readDTableX1_wksp`, once the description of the table is read. -fn build_dtable_x1(dtable: &mut ExactTable, stats: &Code) -> Result<()> { +fn build_dtable_x1(dtable: &mut ExactTable, stats: &Code) -> Result<()> { let mut rank_val = stats.rank_stats; let mut rank_start = [0u32; NB_RANKS]; let mut symbols = [0u8; NB_SYMBOLS]; @@ -248,7 +244,6 @@ fn build_dtable_x1(dtable: &mut ExactTable, stats: &Code) -> Result<()> { if table_log > max_table_log { return Err(Error::TableLogTooLarge); } - dtable.table_type = TableType::X1; dtable.table_log = table_log; // Sort the symbols by weight. @@ -276,7 +271,7 @@ fn build_dtable_x1(dtable: &mut ExactTable, stats: &Code) -> Result<()> { let nb_bits = (table_log + 1 - w) as u16; for &byte in symbols.span(symbol, symbol_count)? { dtable - .x1 + .entries .span_mut(u_start, length)? .fill((u16::from(byte) << 8) | nb_bits); u_start += length; @@ -322,9 +317,13 @@ fn decode_stream_x1( } /// `HUF_decompress1X1_usingDTable_internal_body` -fn decompress_1x1_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { +fn decompress_1x1_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result { let mut bit_d = BitDStream::new(src)?; - decode_stream_x1(dst, &mut bit_d, &dtable.x1, dtable.table_log); + decode_stream_x1(dst, &mut bit_d, &dtable.entries, dtable.table_log); if !bit_d.is_at_end() { return Err(Error::CorruptionDetected); } @@ -332,7 +331,7 @@ fn decompress_1x1_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) } /// `HUF_decompress4X1_usingDTable_internal_body` -fn decompress_4x1_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { +fn decompress_4x1_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { if src.len() < 10 { return Err(Error::CorruptionDetected); } @@ -340,7 +339,7 @@ fn decompress_4x1_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Resul return Err(Error::CorruptionDetected); } let dst_size = dst.len(); - let dt = &dtable.x1; + let dt = &dtable.entries; let dt_log = dtable.table_log; let streams = Streams::new(src)?; let [_, _, _, length4] = streams.length; @@ -447,22 +446,30 @@ fn fast_loop_4x1( } /// `HUF_decompress4X1_usingDTable_internal_fast`. `None` if the input has to be decoded by the other loop. -fn decompress_4x1_fast(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result> { +fn decompress_4x1_fast( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result> { let Some(mut args) = FastArgs::new(dst.len(), src, dtable.table_log)? else { return Ok(None); }; - fast_loop_4x1(&mut args, dst, src, &dtable.x1)?; + fast_loop_4x1(&mut args, dst, src, &dtable.entries)?; for (stream, segment_end) in segment_ends(dst.len()).into_iter().enumerate() { let mut bit_d = args.remaining_dstream(src, stream, segment_end)?; let out = dst.sub_mut(args.op.at(stream)?..segment_end)?; - decode_stream_x1(out, &mut bit_d, &dtable.x1, HUF_DECODER_FAST_TABLELOG); + decode_stream_x1(out, &mut bit_d, &dtable.entries, HUF_DECODER_FAST_TABLELOG); } Ok(Some(dst.len())) } /// `HUF_decompress4X1_usingDTable_internal` -fn decompress_4x1_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { +fn decompress_4x1_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result { if let Some(size) = decompress_4x1_fast(dst, src, dtable)? { return Ok(size); } @@ -600,7 +607,7 @@ fn fill_dtable_x2( } /// `HUF_readDTableX2_wksp`, once the description of the table is read. -fn build_dtable_x2(dtable: &mut ExactTable, stats: &Code) -> Result<()> { +fn build_dtable_x2(dtable: &mut ExactTable, stats: &Code) -> Result<()> { let mut rank_val = [[0u32; NB_RANKS]; HUF_TABLELOG_MAX as usize]; let rank_stats = stats.rank_stats; // `rank_start0[w + 1]` is where the symbols of weight `w` start, until sorting moves it to where they end. @@ -661,7 +668,7 @@ fn build_dtable_x2(dtable: &mut ExactTable, stats: &Code) -> Result<()> { } fill_dtable_x2( - &mut dtable.x2, + &mut dtable.entries, max_table_log, &sorted_symbol, &rank_start0, @@ -671,7 +678,6 @@ fn build_dtable_x2(dtable: &mut ExactTable, stats: &Code) -> Result<()> { )?; dtable.table_log = max_table_log; - dtable.table_type = TableType::X2; Ok(()) } @@ -760,9 +766,13 @@ fn decode_stream_x2( } /// `HUF_decompress1X2_usingDTable_internal_body` -fn decompress_1x2_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { +fn decompress_1x2_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result { let mut bit_d = BitDStream::new(src)?; - decode_stream_x2(dst, &mut bit_d, &dtable.x2, dtable.table_log)?; + decode_stream_x2(dst, &mut bit_d, &dtable.entries, dtable.table_log)?; if !bit_d.is_at_end() { return Err(Error::CorruptionDetected); } @@ -770,7 +780,7 @@ fn decompress_1x2_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) } /// `HUF_decompress4X2_usingDTable_internal_body` -fn decompress_4x2_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { +fn decompress_4x2_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { if src.len() < 10 { return Err(Error::CorruptionDetected); } @@ -778,7 +788,7 @@ fn decompress_4x2_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Resul return Err(Error::CorruptionDetected); } let oend = dst.len(); - let dt = &dtable.x2; + let dt = &dtable.entries; let dt_log = dtable.table_log; let streams = Streams::new(src)?; let [_, _, _, length4] = streams.length; @@ -896,22 +906,30 @@ fn fast_loop_4x2( } /// `HUF_decompress4X2_usingDTable_internal_fast`. `None` if the input has to be decoded by the other loop. -fn decompress_4x2_fast(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result> { +fn decompress_4x2_fast( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result> { let Some(mut args) = FastArgs::new(dst.len(), src, dtable.table_log)? else { return Ok(None); }; - fast_loop_4x2(&mut args, dst, src, &dtable.x2)?; + fast_loop_4x2(&mut args, dst, src, &dtable.entries)?; for (stream, segment_end) in segment_ends(dst.len()).into_iter().enumerate() { let mut bit_d = args.remaining_dstream(src, stream, segment_end)?; let out = dst.sub_mut(args.op.at(stream)?..segment_end)?; - decode_stream_x2(out, &mut bit_d, &dtable.x2, HUF_DECODER_FAST_TABLELOG)?; + decode_stream_x2(out, &mut bit_d, &dtable.entries, HUF_DECODER_FAST_TABLELOG)?; } Ok(Some(dst.len())) } /// `HUF_decompress4X2_usingDTable_internal` -fn decompress_4x2_using_dtable(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { +fn decompress_4x2_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result { if let Some(size) = decompress_4x2_fast(dst, src, dtable)? { return Ok(size); } @@ -957,17 +975,6 @@ pub(super) fn select_decoder_x2(dst_size: usize, src_size: usize) -> Result Result { - let mut table = Self::new(); - match table_type { - TableType::X1 => build_dtable_x1(&mut table, code)?, - TableType::X2 => build_dtable_x2(&mut table, code)?, - } - Ok(table) - } -} - /// `HUF_decompress1X_usingDTable` #[cold] #[inline(never)] @@ -977,10 +984,17 @@ pub(super) fn decompress_1x( code: &Code, table_type: TableType, ) -> Result { - let dtable = ExactTable::build(code, table_type)?; - match dtable.table_type { - TableType::X1 => decompress_1x1_using_dtable(dst, src, &dtable), - TableType::X2 => decompress_1x2_using_dtable(dst, src, &dtable), + match table_type { + TableType::X1 => { + let mut dtable = ExactTable::new(); + build_dtable_x1(&mut dtable, code)?; + decompress_1x1_using_dtable(dst, src, &dtable) + } + TableType::X2 => { + let mut dtable = ExactTable::new(); + build_dtable_x2(&mut dtable, code)?; + decompress_1x2_using_dtable(dst, src, &dtable) + } } } @@ -993,9 +1007,16 @@ pub(super) fn decompress_4x( code: &Code, table_type: TableType, ) -> Result { - let dtable = ExactTable::build(code, table_type)?; - match dtable.table_type { - TableType::X1 => decompress_4x1_using_dtable(dst, src, &dtable), - TableType::X2 => decompress_4x2_using_dtable(dst, src, &dtable), + match table_type { + TableType::X1 => { + let mut dtable = ExactTable::new(); + build_dtable_x1(&mut dtable, code)?; + decompress_4x1_using_dtable(dst, src, &dtable) + } + TableType::X2 => { + let mut dtable = ExactTable::new(); + build_dtable_x2(&mut dtable, code)?; + decompress_4x2_using_dtable(dst, src, &dtable) + } } } From 07e8c4e6aa936b59be3712c0b2f52ac98635e4c8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 20:04:35 +0000 Subject: [PATCH 178/199] zstd: the second tags of the rows are given up within a block where they rule out next to nothing Rows of 64 entries have a second tag per entry, which the C code has not, if the input is at least as large as the hash table: 128 KB at level 10, 4 MiB and more at level 11, 8 MiB and more at level 12. They rule out candidates without a look at the data, and cost time for every position that is inserted. Whether they are worth it was asked at the end of a block, so an input of one block paid for them all the way. With records, long matches, numbers or hex digits they rule out nothing. The parser now stops every 8 KB while there are second tags, and goes on from where it was, with them or without. Up to a block's worth of positions they are given up if they have ruled out less than one candidate for two positions; from then on the rule is what it was, 2.5 a position over the whole frame. Positions that are not inserted no longer count. 128 KB at level 10, time of the port / time of C zstd 1.5.7, before -> after: records of 200 bytes 1.44 -> 1.08, of 100 bytes 1.33 -> 1.03, with a random field 1.29 -> 0.98, of 20 bytes 1.23 -> 1.01, HTML 1.06 -> 0.92, CSV 1.04 -> 0.91, matches of 2,000 bytes 1.14 -> 0.98, hex digits 1.08 -> 0.95. Where the second tags pay (text, source code, DNA, floats) it takes up to 2% longer than before, at 0.45 to 0.84 of C. Levels 5 to 12 at 16 KB, 128 KB and 1 MB are otherwise as they were. --- src/zstd/compress/cctx.rs | 53 +++++++++ src/zstd/compress/internal.rs | 3 + src/zstd/compress/lazy.rs | 216 ++++++++++++++++++++++++++++------ test/js/bun/util/zstd.test.ts | 68 ++++++++++- 4 files changed, 300 insertions(+), 40 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 157048c23a4f..5c4fb5bf16df 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -1952,4 +1952,57 @@ pub(crate) mod tests { } Ok(()) } + + #[test] + #[cfg_attr(miri, ignore = "takes minutes")] + fn second_tags_are_given_up_within_a_block() -> Result<()> { + const SIZE: usize = 1 << 17; + // Many positions start like another one and go on otherwise. + let mut state = 1u32; + let few_symbols = Vec::from_iter((0..SIZE).map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + b'A' + (state >> 30) as u8 + })); + // What starts like another position goes on like it. + let records = Vec::from_iter((0..SIZE).map(|n| match n % 100 { + 0 => (n / 100) as u8, + 1 => (n / 25_600) as u8, + n => n as u8, + })); + let tags_after_a_block_of = |data: &[u8]| -> Result { + let input = Input { + bufs: Buffers { + in_buff: &[], + user: data, + dict: &[], + }, + cdict: None, + }; + let src = Span { + buf: BufId::User, + start: 0, + end: data.len(), + }; + let mut params = small_params(Strategy::Lazy2, 17); + params.c_params.window_log = 17; + params.c_params.search_log = 6; + params.use_row_match_finder = ParamSwitch::Enable; + let mut cctx = FrameCompressor::new(); + cctx.begin(&input, None, ¶ms, data.len() as u64, false)?; + assert!(cctx.match_state.row_tags != RowTags::Flat); + cctx.compress_continue(&mut vec![0; 2 * SIZE], &input, src)?; + Ok(cctx.match_state.row_tags) + }; + + assert!(tags_after_a_block_of(&few_symbols)? != RowTags::Flat); + assert!(tags_after_a_block_of(&records)? == RowTags::Flat); + // Over the whole block they rule out a lot, either way. There is no way back. + let (records, _) = records.split_at(16 << 10); + let (few_symbols, _) = few_symbols.split_at(112 << 10); + assert!(tags_after_a_block_of(&[few_symbols, records].concat())? != RowTags::Flat); + assert!(tags_after_a_block_of(&[records, few_symbols].concat())? == RowTags::Flat); + Ok(()) + } } diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index c122a3c44d30..b273c62f5e02 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -2152,6 +2152,9 @@ impl core::ops::DerefMut for Table { /// What the second tags of the row-based match finder have been good for, in a frame. #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) struct SecondTagStats { + /// How many positions were inserted since it was last asked whether to keep them. + pub(crate) nb_new_positions: u32, + /// How many positions were inserted. pub(crate) nb_positions: u64, /// How many candidates they ruled out. pub(crate) nb_ruled_out: u64, diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 077e969f348b..f9da37c22947 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -12,7 +12,7 @@ use super::internal::{ bt_walk, count_2segments, hash_ptr, index_overlap_check, offbase_is_offset, offbase_to_offset, offset_to_offbase, table_get, table_put, }; -use crate::common::ZSTD_REP_NUM; +use crate::common::{ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM}; use crate::error::Result; use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64}; @@ -766,6 +766,8 @@ struct Skipped { trait Finder<'a>: Sized { /// How far from the end of the block the last search is. const MARGIN: u32; + /// After how many bytes the parser stops, for another finder to go on if that is better. + const TRIAL: Option = None; /// `block` is at least `MARGIN` bytes. fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result; @@ -794,7 +796,7 @@ trait Finder<'a>: Sized { /// Every position is indexed again from now on. fn stop_skipping(&mut self) -> Result<()>; - /// The end of the block. + /// The end of the block, or of the trial. fn finish(self) -> Result<()>; } @@ -1198,6 +1200,8 @@ const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; /// How long a match is at least, if it has the same second tag for a reason. const SECOND_TAG_LENGTH: u32 = 8; +/// After how many bytes the parser stops for [`SecondTagStats::record_trial`]. +const SECOND_TAG_TRIAL: u32 = 1 << 13; /// How the tag table is laid out at the start of a frame of `src_size` bytes, if that is known. pub(crate) const fn row_tags_at_start( @@ -1208,15 +1212,40 @@ pub(crate) const fn row_tags_at_start( // them, for a short input. match src_size { Some(src_size) if c_params.search_log >= 6 && src_size >> c_params.hash_log != 0 => { - RowTags::Second(SecondTagStats { - nb_positions: 0, - nb_ruled_out: 0, - }) + RowTags::Second(SecondTagStats::AT_START) } _ => RowTags::Flat, } } +impl SecondTagStats { + const AT_START: Self = Self { + nb_new_positions: 0, + nb_positions: 0, + nb_ruled_out: 0, + }; + + /// The second tags have ruled out `nb_ruled_out` more candidates while `nb_positions` more were inserted. Returns + /// whether to keep them. + fn record_trial(&mut self, nb_positions: u32, nb_ruled_out: u32) -> bool { + self.nb_new_positions += nb_positions; + self.nb_positions += u64::from(nb_positions); + self.nb_ruled_out += u64::from(nb_ruled_out); + // Where few positions are inserted, they cost little, and it takes some to tell. + if self.nb_new_positions < SECOND_TAG_TRIAL / 2 { + return true; + } + self.nb_new_positions = 0; + // Below that, they cost more than they save. There is no way back, and less to rule out while the rows fill up: + // at first they are only given up where there is next to nothing. + if self.nb_positions < ZSTD_BLOCKSIZE_MAX as u64 { + 2 * self.nb_ruled_out >= self.nb_positions + } else { + 2 * self.nb_ruled_out >= 5 * self.nb_positions + } + } +} + /// The size of the tag table that goes with a hash table of `h_size` entries. pub(crate) const fn row_tag_table_size(tags: RowTags, h_size: usize) -> usize { match tags { @@ -1518,6 +1547,8 @@ struct RowIndex<'a> { base_index: u32, /// `ms.next_to_update`, as a position. next_to_update: u32, + /// How many positions were inserted, if there are second tags. + nb_inserted: u32, } impl RowIndex<'_> { @@ -1584,6 +1615,9 @@ impl RowIndex<'_> { .row::(hash.hash)? .insert(hash, index); } + if SECOND_TAGS { + self.nb_inserted += end.saturating_sub(self.next_to_update); + } self.next_to_update = end; Some(()) } @@ -1793,6 +1827,7 @@ impl<'a> RowState<'a> { near: block.near, base_index: block.base_index, next_to_update: ms.next_to_update.wrapping_sub(block.base_index), + nb_inserted: 0, }, search: RowSearch { block, @@ -1810,7 +1845,7 @@ impl<'a> RowState<'a> { } } - /// The end of a block. + /// The end of a block, or of a trial of the second tags. fn finish(self) -> Result<()> { let Self { index, @@ -1820,13 +1855,10 @@ impl<'a> RowState<'a> { } = self; *saved.next_to_update = index.base_index.wrapping_add(index.next_to_update); *saved.hash_salt_entropy = hash_salt_entropy; - if let RowTags::Second(stats) = saved.tags { - stats.nb_positions += u64::from(search.block.near.len() - search.block.istart); - stats.nb_ruled_out += u64::from(search.nb_ruled_out); - // Below that, the second tags cost more than they save. - if 2 * stats.nb_ruled_out < 5 * stats.nb_positions { - row_flatten_tags(saved.tags, index.rows.tags)?; - } + if let RowTags::Second(stats) = saved.tags + && !stats.record_trial(index.nb_inserted, search.nb_ruled_out) + { + row_flatten_tags(saved.tags, index.rows.tags)?; } Ok(()) } @@ -1841,6 +1873,11 @@ impl<'a, const ROW_ENTRIES: usize, const SECOND_TAGS: bool, const DICT_MODE: u8> for RowFinder<'a, ROW_ENTRIES, SECOND_TAGS, DICT_MODE> { const MARGIN: u32 = 8 + ZSTD_ROW_HASH_CACHE_SIZE as u32; + const TRIAL: Option = if SECOND_TAGS { + Some(SECOND_TAG_TRIAL) + } else { + None + }; #[inline(always)] fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result { @@ -1884,6 +1921,9 @@ impl<'a, const ROW_ENTRIES: usize, const SECOND_TAGS: bool, const DICT_MODE: u8> }; *hash_salt_entropy = hash_salt_entropy.wrapping_add(hash.hash); index.next_to_update = ip + 1; + if SECOND_TAGS { + index.nb_inserted += 1; + } let mut row = index .rows @@ -1970,6 +2010,7 @@ fn rep_length(block: &LazyBlock<'_>, ip: u32, offset: u32) } /// The repeat offsets at the start of a block. +#[derive(Clone, Copy)] struct Offsets { offset_1: u32, offset_2: u32, @@ -1978,12 +2019,23 @@ struct Offsets { saved_2: u32, } -/// Where the parser starts in a block, and with what. +/// Where the parser starts in a block, or goes on, and with what. +#[derive(Clone, Copy)] struct Start { ip: u32, + anchor: u32, + skipping: bool, offsets: Offsets, } +/// How far the parser got. +enum Parsed { + /// To the end of the block, the last literals of which are this many. + Block(usize), + /// To the end of [`Finder::TRIAL`]. + Trial(Start), +} + impl Start { fn new( ms: &MatchState, @@ -2015,17 +2067,23 @@ impl Start { offsets.offset_1 = 0; } } - Self { ip, offsets } + Self { + ip, + anchor: block.istart, + skipping: false, + offsets, + } } } /// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `depth` is how many positions -/// further a better match is looked for, once there is one, and `LAZY` whether that is any. +/// further a better match is looked for, once there is one, and `LAZY` whether that is any. From `args.start` up to the end +/// of the block, or of [`Finder::TRIAL`]. #[inline(never)] -fn compress_block_lazy_generic<'a, F: Finder<'a>, const LAZY: bool, const DICT_MODE: u8>( +fn compress_block_lazy_part<'a, F: Finder<'a>, const LAZY: bool, const DICT_MODE: u8>( args: LazyArgs<'a>, depth: u32, -) -> Result { +) -> Result { let LazyArgs { ms, seq_store, @@ -2038,21 +2096,25 @@ fn compress_block_lazy_generic<'a, F: Finder<'a>, const LAZY: bool, const DICT_M let near = block.near; let Some(ilimit) = near.len().checked_sub(F::MARGIN) else { // There is no position to search, nor to hash. - return Ok((near.len() - block.istart) as usize); + return Ok(Parsed::Block((near.len() - start.anchor) as usize)); }; let mut finder = F::new(ms, src, *block)?; let mut seq_store = SeqSink::new(seq_store); let mut ip = start.ip; - let mut anchor = block.istart; + let mut anchor = start.anchor; + let stop = match F::TRIAL { + Some(trial) => ilimit.min(ip + trial), + None => ilimit, + }; let Offsets { mut offset_1, mut offset_2, saved_1, mut saved_2, } = start.offsets; - let mut skipping = false; + let mut skipping = start.skipping; - 'parse: while ip < ilimit { + 'parse: while ip < stop { let mut no_repeat = false; if !skipping { let skipped = finder.skip_literals(LiteralRun { @@ -2186,6 +2248,19 @@ fn compress_block_lazy_generic<'a, F: Finder<'a>, const LAZY: bool, const DICT_M } } finder.finish()?; + if F::TRIAL.is_some() && ip < ilimit { + return Ok(Parsed::Trial(Start { + ip, + anchor, + skipping, + offsets: Offsets { + offset_1, + offset_2, + saved_1, + saved_2, + }, + })); + } // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. if saved_1 != 0 && offset_1 != 0 { @@ -2197,11 +2272,23 @@ fn compress_block_lazy_generic<'a, F: Finder<'a>, const LAZY: bool, const DICT_M *rep_0 = if offset_1 != 0 { offset_1 } else { saved_1 }; *rep_1 = if offset_2 != 0 { offset_2 } else { saved_2 }; - // The size of the last literals. - Ok((near.len() - anchor) as usize) + Ok(Parsed::Block((near.len() - anchor) as usize)) +} + +/// [`compress_block_lazy_part`] with a finder that is not on trial. Returns the size of the last literals. +#[inline(always)] +fn compress_block_lazy_generic<'a, F: Finder<'a>, const LAZY: bool, const DICT_MODE: u8>( + args: LazyArgs<'a>, + depth: u32, +) -> Result { + const { assert!(F::TRIAL.is_none()) }; + match compress_block_lazy_part::(args, depth)? { + Parsed::Block(last_literals) => Ok(last_literals), + Parsed::Trial(_) => Err(BUG), + } } -/// What [`compress_block_lazy_generic`] works on. +/// What [`compress_block_lazy_part`] works on. struct LazyArgs<'a> { ms: &'a mut MatchState, seq_store: &'a mut SeqStore, @@ -2211,10 +2298,23 @@ struct LazyArgs<'a> { start: Start, } +impl LazyArgs<'_> { + fn reborrow(&mut self) -> LazyArgs<'_> { + LazyArgs { + ms: self.ms, + seq_store: self.seq_store, + rep: self.rep, + src: self.src, + block: self.block, + start: self.start, + } + } +} + /// [`compress_block_lazy_generic`], with the hash chains or the rows. #[inline(always)] fn compress_block_lazy_hashed( - args: LazyArgs<'_>, + mut args: LazyArgs<'_>, depth: u32, use_row: bool, ) -> Result { @@ -2232,21 +2332,27 @@ fn compress_block_lazy_hashed( if DICT_MODE != NO_DICT { row_flatten_tags(&mut args.ms.row_tags, &mut args.ms.tag_table)?; } - match (args.ms.c_params.search_log.clamp(4, 6), args.ms.row_tags) { - (4, _) => compress_block_lazy_generic::, LAZY, DICT_MODE>( + match args.ms.c_params.search_log.clamp(4, 6) { + 4 => compress_block_lazy_generic::, LAZY, DICT_MODE>( args, depth, ), - (5, _) => compress_block_lazy_generic::, LAZY, DICT_MODE>( + 5 => compress_block_lazy_generic::, LAZY, DICT_MODE>( args, depth, ), - (_, RowTags::Flat) => { + _ => { + while DICT_MODE == NO_DICT && args.ms.row_tags != RowTags::Flat { + let part = args.reborrow(); + match compress_block_lazy_part::, LAZY, NO_DICT>( + part, depth, + )? { + Parsed::Block(last_literals) => return Ok(last_literals), + Parsed::Trial(start) => args.start = start, + } + } compress_block_lazy_generic::, LAZY, DICT_MODE>( args, depth, ) } - (_, RowTags::Second(_)) => { - compress_block_lazy_generic::, LAZY, NO_DICT>(args, depth) - } } } @@ -2304,6 +2410,48 @@ pub(crate) fn compress_block_lazy( mod tests { use super::*; + #[test] + fn second_tags_are_kept_for_as_long_as_they_rule_out_enough() { + const TRIAL: u32 = SECOND_TAG_TRIAL; + const TRIALS_IN_A_BLOCK: usize = ZSTD_BLOCKSIZE_MAX / TRIAL as usize; + // How many of `trials`, of so many positions and candidates that were ruled out, they get through. + let nb_kept = |trials: &[(u32, u32)]| { + let mut stats = SecondTagStats::AT_START; + trials + .iter() + .take_while(|&&(nb_positions, nb_ruled_out)| { + stats.record_trial(nb_positions, nb_ruled_out) + }) + .count() + }; + let (none, little, some, a_lot) = ( + (TRIAL, 0), + (TRIAL, TRIAL / 2), + (TRIAL, TRIAL), + (TRIAL, 4 * TRIAL), + ); + + assert_eq!(nb_kept(&[none]), 0); + assert_eq!(nb_kept(&[(TRIAL, TRIAL / 2 - 1)]), 0); + assert_eq!(nb_kept(&[little, none]), 1); + // What came before makes up for a stretch with little. + assert_eq!(nb_kept(&[some, none, a_lot, none, none, none, none]), 7); + assert_eq!(nb_kept(&[some, none, none]), 2); + // More is asked for once the rows had the time to fill up. + assert_eq!( + nb_kept(&[some; 2 * TRIALS_IN_A_BLOCK]), + TRIALS_IN_A_BLOCK - 1 + ); + assert_eq!( + nb_kept(&[a_lot; 2 * TRIALS_IN_A_BLOCK]), + 2 * TRIALS_IN_A_BLOCK + ); + // It takes some positions to tell. + let few = (TRIAL / 4, 0); + assert_eq!(nb_kept(&[few, few]), 1); + assert_eq!(nb_kept(&[few, (TRIAL / 4, TRIAL), few]), 3); + } + #[test] fn a_long_candidate_that_falls_short_is_what_the_next_ones_have_to_beat() -> Result<()> { const RECORD: u32 = 200; diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 883bec619959..7250ff14e4a5 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1501,13 +1501,16 @@ describe("the output is that of zstd 1.5.7", () => { ]); }); + const level = (level: number) => ({ [c.ZSTD_c_compressionLevel]: level }); + const piece = (n: number, size: number) => corpus.subarray(500_000 + n * size, 500_000 + (n + 1) * size); + /** Records of a counter and the same `size` bytes. */ + const counted = (size: number, count: number) => + Buffer.concat( + Array.from({ length: count }, (_, i) => [Buffer.from(`${i}`.padStart(8, "0")), piece(0, size)]).flat(), + ); + it("where records differ in a few fields", () => { // What is before a position ends at a field, and may be the same again after it. - const piece = (n: number, size: number) => corpus.subarray(500_000 + n * size, 500_000 + (n + 1) * size); - const counted = (size: number, count: number) => - Buffer.concat( - Array.from({ length: count }, (_, i) => [Buffer.from(`${i}`.padStart(8, "0")), piece(0, size)]).flat(), - ); const flags: Uint8Array[] = []; for (let i = 0, x = 7; i < 2_400; i++) { x ^= x << 13; @@ -1516,7 +1519,6 @@ describe("the output is that of zstd 1.5.7", () => { flags.push(Buffer.from(`${x & 1}`), piece(i % 4, 80)); } const flagged = Buffer.concat(flags); - const level = (level: number) => ({ [c.ZSTD_c_compressionLevel]: level }); const dictionary = flagged.subarray(0, 30_000); const cases: [Buffer, zlib.ZstdOptions][] = [ // Hash chains. @@ -1562,6 +1564,60 @@ describe("the output is that of zstd 1.5.7", () => { `); }); + it("where the kind of data changes within a block, for rows of 64 entries", () => { + let x = 11; + const next = () => { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + return x >>> 0; + }; + // Many positions start like another one and go on otherwise. + const fewSymbols = (size: number) => Uint8Array.from({ length: size }, () => 65 + (next() >>> 30)); + const noise = (size: number) => Uint8Array.from({ length: size }, () => next() >>> 24); + // What starts like another position goes on like it. + const records = (size: number, record: number) => counted(record, Math.ceil(size / record)).subarray(0, size); + const text = (size: number) => corpus.subarray(0, size); + const rowsOf64 = (strategy: number, hashLog: number) => ({ + [c.ZSTD_c_strategy]: strategy, + [c.ZSTD_c_searchLog]: 6, + [c.ZSTD_c_hashLog]: hashLog, + [c.ZSTD_c_windowLog]: 17, + }); + const cases: [Uint8Array[], Record][] = [ + [[fewSymbols(1 << 17)], level(10)], + [[text(1 << 17)], level(10)], + [[records(1 << 17, 100)], level(10)], + [[records(1 << 17, 600)], level(10)], + [[fewSymbols(80 << 10), records(48 << 10, 100)], level(10)], + [[fewSymbols(30_000), records(30_000, 100)], rowsOf64(c.ZSTD_lazy2, 12)], + [[records(30_000, 100), fewSymbols(30_000)], rowsOf64(c.ZSTD_lazy2, 12)], + [[fewSymbols(12_000), noise(30_000), fewSymbols(12_000)], rowsOf64(c.ZSTD_lazy2, 12)], + [[noise(20_000), text(30_000)], rowsOf64(c.ZSTD_lazy, 12)], + [[text(20_000), Buffer.alloc(20_000), text(20_000)], rowsOf64(c.ZSTD_lazy, 13)], + [[fewSymbols(20_000), records(20_000, 600), text(20_000)], rowsOf64(c.ZSTD_greedy, 12)], + // More than a block. + [[fewSymbols(125_000), records(40_000, 100), fewSymbols(20_000)], rowsOf64(c.ZSTD_lazy2, 14)], + ]; + expect(cases.map(([data, params]) => digest(zlib.zstdCompressSync(Buffer.concat(data), { params })))) + .toMatchInlineSnapshot(` + [ + "38882 e2cf1ebea8540fae", + "17398 f542e301793c6b24", + "1513 1e09820fa83c397", + "574 9c214dab588dfad4", + "26008 3a7c167a6f55827b", + "10030 3433cfd18a1e6b89", + "10066 f5047098fb382ae5", + "38583 6ec8f3e7937fbdfb", + "25171 39b72ef5650bbcd5", + "2868 4efe031f2810b3ce", + "9636 ec7bd23f67b5ae31", + "45730 6bb17dea79d5fff0", + ] + `); + }); + it("where matches are far between, for the optimal parsers", () => { const data = Buffer.alloc(100_000); let x = 9; From 9778f063c3c58aa187e8ef4845721608d82f9939 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 23:11:03 +0000 Subject: [PATCH 179/199] zstd: btlazy2 gives no hints in the first block and adds skipped positions in a tighter loop 128 KB at levels 11 and 12 took 1.09 of the time of C over 25 kinds of data, up to 1.5 on short records. The hints at what the trees are about to read started at 64 KB of input. For btlazy2 they take more time than they save within the first block, so they now start after it. The levels of the optimal parser keep 64 KB: there it helps records and HTML but costs numbers and text. On data that compresses well, half of the time goes to adding the positions that a match skipped to the trees. That loop now takes the nodes as a slice, up to the end of the table, instead of masking and checking every index, and reads 4 bytes where only 4 count. --- src/zstd/compress/internal.rs | 37 +++++++++++++++ src/zstd/compress/lazy.rs | 85 +++++++++++++++++++++++++++++------ 2 files changed, 109 insertions(+), 13 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index b273c62f5e02..0dd78dfec71d 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1657,6 +1657,16 @@ impl Hasher { } } + /// The same from the first 4 of the 8 bytes, if no more than these count. + pub(crate) const fn of_4_bytes(self) -> Option { + let h_bits = 64 - self.shift; + if self.factor == (PRIME_4_BYTES as u64) << 32 && h_bits <= 32 { + Some(Hash4 { h_bits }) + } else { + None + } + } + /// [`hash_ptr`] of the position that the 8 bytes `bytes` were read at. #[inline(always)] pub(crate) const fn hash(self, bytes: u64) -> usize { @@ -1683,6 +1693,19 @@ fn hash_ptr_salted(buf: &[u8], pos: usize, h_bits: u32, mls: u32, hash_salt: u64 }) } +/// [`hash_ptr`] of 4 bytes. To read no more than these crosses fewer lines of the cache than a [`Hasher`] does. +#[derive(Clone, Copy)] +pub(crate) struct Hash4 { + h_bits: u32, +} + +impl Hash4 { + #[inline(always)] + pub(crate) const fn hash(self, bytes: u32) -> usize { + hash4(bytes, self.h_bits, 0) as usize + } +} + /// [`hash_ptr_salted`] of 4 to 6 bytes, out of the 8 bytes at the position: the multiplication also does the shift that /// leaves the other bytes out. #[derive(Clone, Copy)] @@ -2435,6 +2458,20 @@ bun_pod::impl_zeroable!(Match, RawSeq, LdmEntry); mod tests { use super::*; + #[test] + fn the_hash_of_4_bytes_is_the_same_from_4_bytes() -> Result<()> { + for h_bits in [6, 17, 32] { + let hasher = Hasher::new(h_bits, 4); + let of_4_bytes = hasher.of_4_bytes().ok_or(BUG)?; + for bytes in [0, 1, 0x0123_4567_89AB_CDEF, u64::MAX] { + assert_eq!(of_4_bytes.hash(bytes as u32), hasher.hash(bytes)); + } + } + assert!(Hasher::new(33, 4).of_4_bytes().is_none()); + assert!(Hasher::new(17, 5).of_4_bytes().is_none()); + Ok(()) + } + #[test] fn matches_are_counted_up_to_where_they_end() -> Result<()> { const DISTANCE: u32 = 7; diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index f9da37c22947..cbcad0871295 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -9,8 +9,8 @@ use super::internal::{ K_SEARCH_STRENGTH, MatchState, Near, Nodes, Prefix, REPCODE1_TO_OFFBASE, RowTags, SecondTagStats, SeqSink, SeqStore, Strategy, TreeInput, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, bt_children, bt_look_ahead, bt_low, bt_nodes, bt_set_children, - bt_walk, count_2segments, hash_ptr, index_overlap_check, offbase_is_offset, offbase_to_offset, - offset_to_offbase, table_get, table_put, + bt_walk, count_2segments, hash_ptr, hashed_swap, index_overlap_check, offbase_is_offset, + offbase_to_offset, offset_to_offbase, table_get, table_put, }; use crate::common::{ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM}; use crate::error::Result; @@ -314,7 +314,8 @@ const LONG_UPDATE: u32 = 8; /// The same where `update` gives hints at what is about to be read, which `update_many` does not. const LONG_UPDATE_FAR: u32 = 32; -/// `ZSTD_updateDUBT` of the indices `indices`, which are in `near`, whose first index is `dict_limit`. +/// `ZSTD_updateDUBT` of the indices `indices`, which are in `near`, whose first index is `dict_limit`. The sizes of the +/// tables are powers of 2, and that of `hash_table` is what `hasher` is for. #[cold] #[inline(never)] fn update_many( @@ -322,17 +323,46 @@ fn update_many( bt: &mut Nodes, near: Near<'_>, hasher: Hasher, - indices: core::ops::Range, + indices: Range, dict_limit: u32, ) -> Result<()> { - let first = indices.start.wrapping_sub(dict_limit) as usize; - let last = indices.end.wrapping_sub(dict_limit) as usize; - let bytes = near.bytes().sub(first..last + 7)?; - for (idx, bytes) in indices.zip(bytes.array_windows()) { - let h = hasher.hash(u64::from_le_bytes(*bytes)); - let match_index = table_get(hash_table, h); - table_put(hash_table, h, idx); - bt_set_children(bt, idx, [match_index, ZSTD_DUBT_UNSORTED_MARK]); + let bytes = near + .tail(indices.start.wrapping_sub(dict_limit)) + .ok_or(BUG)?; + match hasher.of_4_bytes() { + Some(hasher) => update_many_with(hash_table, bt, bytes, indices, |bytes| { + hasher.hash(u32::from_le_bytes(bytes)) + }), + None => update_many_with(hash_table, bt, bytes, indices, |bytes| { + hasher.hash(u64::from_le_bytes(bytes)) + }), + } +} + +/// [`update_many`] of `indices`, the first of which is at the start of `bytes`. `hash` gives the hash of a position from +/// its first `N` bytes. +#[inline(always)] +fn update_many_with( + hash_table: &mut [u32], + bt: &mut Nodes, + bytes: Near<'_>, + mut indices: Range, + hash: impl Fn([u8; N]) -> usize, +) -> Result<()> { + if bt.is_empty() { + return Err(BUG); + } + let mut bytes = bytes.bytes().sub(..indices.len() + 7)?; + while !indices.is_empty() { + // The nodes of indices that follow each other do so too, up to the end of the table. + let nodes = bt.sub_mut(indices.start as usize & (bt.len() - 1)..)?; + let nb_nodes = nodes.len().min(indices.len()); + for ((idx, node), bytes) in indices.clone().zip(nodes).zip(bytes.array_windows()) { + let match_index = hashed_swap(hash_table, hash(*bytes), idx); + *node = [match_index, ZSTD_DUBT_UNSORTED_MARK]; + } + indices.start += nb_nodes as u32; + bytes = bytes.sub(nb_nodes..)?; } Ok(()) } @@ -2367,7 +2397,8 @@ fn compress_block_lazy_any_strategy( Strategy::Greedy => compress_block_lazy_hashed::(args, 0, use_row), Strategy::Lazy => compress_block_lazy_hashed::(args, 1, use_row), Strategy::Lazy2 => compress_block_lazy_hashed::(args, 2, use_row), - Strategy::BtLazy2 if !use_row && TreeInput::is_far(args.block.near) => { + // In the first block, the hints take more time than they save. + Strategy::BtLazy2 if !use_row && args.block.near.len() > ZSTD_BLOCKSIZE_MAX as u32 => { compress_block_lazy_generic::, true, DICT_MODE>(args, 2) } Strategy::BtLazy2 if !use_row => { @@ -2452,6 +2483,34 @@ mod tests { assert_eq!(nb_kept(&[few, (TRIAL / 4, TRIAL), few]), 3); } + #[test] + fn many_positions_are_added_to_the_trees_as_one_by_one() -> Result<()> { + const DICT_LIMIT: u32 = 2; + const HASH_LOG: u32 = 6; + let bytes: Vec = (0..300u32).map(|n| (n * n % 7) as u8).collect(); + let near = Near::new(&bytes)?; + for min_match in [4, 5] { + let hasher = Hasher::new(HASH_LOG, min_match); + // They go around the table more than once, or stop at its end, or short of it. + for indices in [5..280, 40..64, 70..90] { + let (mut hash_table, mut chain_table) = ([1; 1 << HASH_LOG], [9; 64]); + let (mut expected_hash_table, mut expected_chain_table) = (hash_table, chain_table); + for idx in indices.clone() { + let h = hasher.hash(near.read64(idx - DICT_LIMIT).ok_or(BUG)?); + let children = [table_get(&expected_hash_table, h), ZSTD_DUBT_UNSORTED_MARK]; + bt_set_children(bt_nodes(&mut expected_chain_table), idx, children); + table_put(&mut expected_hash_table, h, idx); + } + + let bt = bt_nodes(&mut chain_table); + update_many(&mut hash_table, bt, near, hasher, indices, DICT_LIMIT)?; + assert_eq!(hash_table, expected_hash_table); + assert_eq!(chain_table, expected_chain_table); + } + } + Ok(()) + } + #[test] fn a_long_candidate_that_falls_short_is_what_the_next_ones_have_to_beat() -> Result<()> { const RECORD: u32 = 200; From b06f15db1c976da6d6f4203f277891ac4bd71011 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 23:11:30 +0000 Subject: [PATCH 180/199] zstd: the buffer of the literals is cleared as they come, not all at once A new context cleared room for as many literals as the block has bytes before its first block: 131 KB for 128 KB of input, at every level, however few the literals. On input that compresses well that took longer than the rest (a third of the time for a period of 1 KB at level 1). The buffer now gets longer 4 KB ahead of the literals. The loops write to it as before; what finds no room goes to a cold function that the buffer is moved to and back from, as lending it keeps it out of the registers. Not for dfast, whose loops are 3 to 6% slower with a buffer whose length changes: which of the two a finder gets is part of the type of its sink, and one that does not grow is cleared when the sink is made. `SeqStore::literals` fails if the length is off instead of returning nothing. --- src/zstd/compress/cctx.rs | 10 +- src/zstd/compress/double_fast.rs | 29 ++--- src/zstd/compress/fast.rs | 38 +++--- src/zstd/compress/internal.rs | 194 ++++++++++++++++++++++++++----- src/zstd/compress/lazy.rs | 6 +- src/zstd/compress/superblock.rs | 2 +- src/zstd/util.rs | 10 ++ 7 files changed, 220 insertions(+), 69 deletions(-) diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs index 5c4fb5bf16df..f2bc9bd24a74 100644 --- a/src/zstd/compress/cctx.rs +++ b/src/zstd/compress/cctx.rs @@ -38,8 +38,8 @@ use crate::common::{ }; use crate::error::{Error, Result}; use crate::util::{ - BUG, SliceExt, append, highbit32, read_le32, reset_table, write_le16, write_le24, write_le32, - write_le64, + SliceExt, append, append_at, highbit32, read_le32, reset_table, write_le16, write_le24, + write_le32, write_le64, }; /// `ZSTD_compressionStage_e` @@ -146,14 +146,10 @@ impl Output<'_> { if pos + size > *end { return Err(Error::DstSizeTooSmall); } - if pos > vec.len() { - return Err(BUG); - } // Appended, which takes no zeros first. - vec.truncate(pos); let header = block_header(BlockType::Raw, src.len(), last_block); let [header @ .., _] = header.to_le_bytes(); - append(vec, &header)?; + append_at(vec, pos, &header)?; append(vec, src)?; Ok(size) } diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs index 219a00153617..bdfdd0658257 100644 --- a/src/zstd/compress/double_fast.rs +++ b/src/zstd/compress/double_fast.rs @@ -72,6 +72,9 @@ fn look_ahead( } } +/// Its loops have no register to spare for a buffer that moves. +type Sink<'a, 'b> = MatchSink<'a, 'b, 16, false>; + /// What `dfast` looks for matches with, when there is no dictionary and the window is one segment. struct Finder { /// The index of the first position. @@ -96,7 +99,7 @@ impl Finder { &mut self, hash_long: &mut [u32], hash_small: &mut [u32], - sink: &mut MatchSink<'_, '_>, + sink: &mut Sink<'_, '_>, ) { const K_STEP_INCR: u32 = 1 << K_SEARCH_STRENGTH; let Self { @@ -145,7 +148,7 @@ impl Finder { if LONG == 0 || LONG >= 17 { look_ahead(near, end, hash_long, long, hash_small, small); } - sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); + sink.store(ip, end, REPCODE1_TO_OFFBASE); break end; } @@ -192,7 +195,7 @@ impl Finder { hashed_put(hash_long, long.hash(bytes1), base_index.wrapping_add(ip1)); } let back = near.count_backwards(ip, sink.anchor, match0, 0); - sink.store::<16>(ip - back, end, offset_to_offbase(offset_1)); + sink.store(ip - back, end, offset_to_offbase(offset_1)); break end; }; ip = end; @@ -256,7 +259,7 @@ pub(crate) fn compress_block_double_fast( offset_1: block.rep.offset_1, offset_2: block.rep.offset_2, }; - let mut sink = MatchSink::new(seq_store, &block); + let mut sink = Sink::new(seq_store, &block); let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; // What the levels come with, for an input of any size. @@ -292,7 +295,7 @@ impl Finder { &mut self, hash_long: &mut [u32], hash_small: &mut [u32], - sink: &mut MatchSink<'_, '_>, + sink: &mut Sink<'_, '_>, window: Segments<'_>, dict_tables: &DictTables<'_>, ) { @@ -320,7 +323,7 @@ impl Finder { let match_index_s = hashed_swap(hash_small, small.hash(bytes0), curr); if let Some(end) = window.repeat_end(ip + 1, offset_1, (bytes0 >> 8) as u32) { - sink.store::<16>(ip + 1, end, REPCODE1_TO_OFFBASE); + sink.store(ip + 1, end, REPCODE1_TO_OFFBASE); ip = end; } else { // `ip` is moved to where the match is for. @@ -369,7 +372,7 @@ impl Finder { }; offset_2 = offset_1; offset_1 = found.offset; - sink.store::<16>(ip - found.back, found.end, offset_to_offbase(offset_1)); + sink.store(ip - found.back, found.end, offset_to_offbase(offset_1)); ip = found.end; } @@ -400,7 +403,7 @@ impl Finder { let index = base_index.wrapping_add(ip); hashed_put(hash_small, small.hash(bytes), index); hashed_put(hash_long, long.hash(bytes), index); - sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); + sink.store(ip, end, REPCODE1_TO_OFFBASE); ip = end; } } @@ -450,7 +453,7 @@ pub(crate) fn compress_block_double_fast_dict_match_state( offset_1, offset_2, }; - let sink = &mut MatchSink::with_near(seq_store, block.window.near, block.istart); + let sink = &mut Sink::with_near(seq_store, block.window.near, block.istart); let long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let small = ms.chain_table.sub_mut(..1 << h_bits_s)?; finder.find_matches_dict(long, small, sink, block.window, tables); @@ -468,7 +471,7 @@ impl Finder { &mut self, hash_long: &mut [u32], hash_small: &mut [u32], - sink: &mut MatchSink<'_, '_>, + sink: &mut Sink<'_, '_>, window: Segments<'_>, ) { let Self { @@ -508,7 +511,7 @@ impl Finder { let rep_place = window.place_of(ip) - offset_1; let end = ip + 4 + window.count(ip + 4, rep_place + 4); look_ahead(near, end, hash_long, long, hash_small, small); - sink.store::<16>(ip, end, REPCODE1_TO_OFFBASE); + sink.store(ip, end, REPCODE1_TO_OFFBASE); ip = end; } else { // The match, and how much of it is known. `ip` is moved to where it is for. @@ -535,7 +538,7 @@ impl Finder { let back = window.count_backwards_if_any(ip, sink.anchor, place); offset_2 = offset_1; offset_1 = window.place_of(ip) - place; - sink.store::<16>(ip - back, end, offset_to_offbase(offset_1)); + sink.store(ip - back, end, offset_to_offbase(offset_1)); ip = end; } @@ -603,7 +606,7 @@ pub(crate) fn compress_block_double_fast_ext_dict( offset_1, offset_2, }; - let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); + let mut sink = Sink::with_near(seq_store, block.window.near, block.istart); let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; let (long, small, sink) = (hash_long, hash_small, &mut sink); diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs index 728fb6831d98..3cde100c5c53 100644 --- a/src/zstd/compress/fast.rs +++ b/src/zstd/compress/fast.rs @@ -289,8 +289,10 @@ impl<'a> ExtDictBlock<'a> { /// Where `fast` and `dfast` put the matches they find in a block, when there is no dictionary and the window is one /// segment. All of it is read and written once per match, so that the search for the next one has the registers. -pub(super) struct MatchSink<'a, 'b> { - seq_store: SeqSink<'b>, +/// +/// `SHORT` and `GROWS` are those of [`SeqSink`]. +pub(super) struct MatchSink<'a, 'b, const SHORT: usize, const GROWS: bool> { + seq_store: SeqSink<'b, SHORT, GROWS>, /// The input from the lowest index a match can be at. Positions are in this. pub(super) near: Near<'a>, /// The same with what is before it, as far as a repeat offset can reach. @@ -302,7 +304,7 @@ pub(super) struct MatchSink<'a, 'b> { failed: bool, } -impl<'a, 'b> MatchSink<'a, 'b> { +impl<'a, 'b, const SHORT: usize, const GROWS: bool> MatchSink<'a, 'b, SHORT, GROWS> { pub(super) fn new(seq_store: &'b mut SeqStore, block: &NoDictBlock<'a>) -> Self { Self { seq_store: SeqSink::new(seq_store), @@ -361,11 +363,11 @@ impl<'a, 'b> MatchSink<'a, 'b> { /// Stores the match from `ip` to `end`, with the literals before it. #[inline(always)] - pub(super) fn store(&mut self, ip: u32, end: u32, off_base: u32) { + pub(super) fn store(&mut self, ip: u32, end: u32, off_base: u32) { let anchor = self.anchor; self.anchor = end; if (self.seq_store) - .store_seq::(ip - anchor, self.near, anchor, off_base, end - ip) + .store_seq(ip - anchor, self.near, anchor, off_base, end - ip) .is_err() { core::hint::cold_path(); @@ -559,6 +561,8 @@ pub(super) fn after(condition: bool) -> bool { condition } +type Sink<'a, 'b> = MatchSink<'a, 'b, 32, true>; + /// What `fast` looks for matches with, when there is no dictionary and the window is one segment. struct Finder { /// The index of the first position. @@ -580,11 +584,7 @@ impl Finder { /// /// `HLOG` is how many bits `hasher` gives, or 0 for any: see [`table_of_size`]. #[inline(never)] - fn find_matches( - &mut self, - hash_table: &mut [u32], - sink: &mut MatchSink<'_, '_>, - ) { + fn find_matches(&mut self, hash_table: &mut [u32], sink: &mut Sink<'_, '_>) { const K_STEP_INCR: u32 = 1 << (K_SEARCH_STRENGTH - 1); let Self { base_index, @@ -663,7 +663,7 @@ impl Finder { }; ip0 = end; - sink.store::<32>(start, end, off_base); + sink.store(start, end, off_base); // `ip0` is not past the last position that can be looked at. let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { @@ -707,7 +707,7 @@ impl Finder { fn find_matches_ext_dict( &mut self, hash_table: &mut [u32], - sink: &mut MatchSink<'_, '_>, + sink: &mut Sink<'_, '_>, window: Segments<'_>, ) { const K_STEP_INCR: u32 = 1 << (K_SEARCH_STRENGTH - 1); @@ -785,7 +785,7 @@ impl Finder { }; ip0 = ip + 4 + window.count(ip + 4, place + 4); - sink.store::<32>(ip - back, ip0, off_base); + sink.store(ip - back, ip0, off_base); if next < ip0 && let Some(bytes) = near.read64(next) @@ -852,7 +852,7 @@ pub(crate) fn compress_block_fast( rep_offset1: block.rep.offset_1, rep_offset2: block.rep.offset_2, }; - let mut sink = MatchSink::new(seq_store, &block); + let mut sink = Sink::new(seq_store, &block); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; // What the levels come with, for an input of any size. match hlog { @@ -952,7 +952,7 @@ impl Finder { fn find_matches_dict( &mut self, hash_table: &mut [u32], - sink: &mut MatchSink<'_, '_>, + sink: &mut Sink<'_, '_>, window: Segments<'_>, dict_table: TaggedTable<'_>, ) { @@ -988,7 +988,7 @@ impl Finder { let match_index = hashed_swap(hash_table, hasher.hash(bytes0), curr); if let Some(end) = window.repeat_end(ip0 + 1, rep_offset1, (bytes0 >> 8) as u32) { - sink.store::<32>(ip0 + 1, end, REPCODE1_TO_OFFBASE); + sink.store(ip0 + 1, end, REPCODE1_TO_OFFBASE); ip0 = end; break None; } @@ -1018,7 +1018,7 @@ impl Finder { if let Some(found) = found { rep_offset2 = rep_offset1; rep_offset1 = found.offset; - sink.store::<32>(ip0 - found.back, found.end, offset_to_offbase(rep_offset1)); + sink.store(ip0 - found.back, found.end, offset_to_offbase(rep_offset1)); ip0 = found.end; } @@ -1082,7 +1082,7 @@ pub(crate) fn compress_block_fast_dict_match_state( rep_offset1, rep_offset2, }; - let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); + let mut sink = Sink::with_near(seq_store, block.window.near, block.istart); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; finder.find_matches_dict(hash_table, &mut sink, block.window, dict_table); let last_literals = sink.last_literals()?; @@ -1112,7 +1112,7 @@ pub(crate) fn compress_block_fast_ext_dict( rep_offset1: block.rep.offset_1, rep_offset2: block.rep.offset_2, }; - let mut sink = MatchSink::with_near(seq_store, block.window.near, block.istart); + let mut sink = Sink::with_near(seq_store, block.window.near, block.istart); let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; // What the levels come with, for an input that is large enough to get here. match hlog { diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 0dd78dfec71d..eaa609f898eb 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -13,7 +13,8 @@ use crate::common::{ }; use crate::error::{Error, Result}; use crate::util::{ - BUG, SliceExt, highbit32, make_room, read_le32, read_le64, reset_table, try_zeroed, write_le24, + BUG, SliceExt, append_at, copy, highbit32, make_room, read_le32, read_le64, reset_table, + try_zeroed, write_le24, }; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; @@ -265,7 +266,8 @@ pub(crate) enum LongLengthType { pub(crate) struct SeqStore { /// Never grows beyond the capacity it is given by [`Self::start_block`]. pub(crate) sequences: Vec, - /// As long as the longest block so far, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. + /// It has room for as many literals as a block has bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. Its length + /// follows the literals: to clear all of it for a few of them takes longer than many a block. pub(crate) lit_buffer: Vec, /// How much of `lit_buffer` is literals. pub(crate) lit_len: usize, @@ -366,16 +368,11 @@ impl SeqStore { let block_size = block_size.min(self.max_nb_lit); let nb_seq = (block_size / MINMATCH as usize).min(self.max_nb_seq); make_room(&mut self.sequences, nb_seq, self.max_nb_seq)?; - let len = block_size + WILDCOPY_OVERLENGTH; make_room( &mut self.lit_buffer, - len, + block_size + WILDCOPY_OVERLENGTH, self.max_nb_lit + WILDCOPY_OVERLENGTH, - )?; - if self.lit_buffer.len() < len { - self.lit_buffer.resize(len, 0); - } - Ok(()) + ) } /// Makes `codes` at least as long as `sequences`. @@ -397,8 +394,8 @@ impl SeqStore { } #[inline] - pub(crate) fn literals(&self) -> &[u8] { - self.lit_buffer.get(..self.lit_len).unwrap_or_default() + pub(crate) fn literals(&self) -> Result<&[u8]> { + self.lit_buffer.sub(..self.lit_len) } /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. @@ -436,9 +433,11 @@ impl SeqStore { self.lit_buffer.array_at_mut::<16>(self.lit_len), ) { (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, - _ => self - .lit_buffer - .write_at(self.lit_len, buf.span(literals, lit_length)?)?, + _ => write_literals( + &mut self.lit_buffer, + self.lit_len, + buf.span(literals, lit_length)?, + )?, } self.lit_len += lit_length; self.store_seq_only(lit_length, off_base, match_length); @@ -449,36 +448,74 @@ impl SeqStore { #[inline] pub(crate) fn store_last_literals(&mut self, block: &[u8], count: usize) -> Result<()> { let literals = block.sub(block.len().checked_sub(count).ok_or(BUG)?..)?; - self.lit_buffer.write_at(self.lit_len, literals)?; + write_literals(&mut self.lit_buffer, self.lit_len, literals)?; self.lit_len += literals.len(); Ok(()) } } +/// Writes `literals` at `lit_len` in `lit_buffer`. +#[inline(always)] +fn write_literals(lit_buffer: &mut Vec, lit_len: usize, literals: &[u8]) -> Result<()> { + if let Some(room) = lit_buffer.get_mut(lit_len..lit_len + literals.len()) { + return copy(room, literals); + } + // Moved there and back: a buffer that is lent could not stay in the registers of a hot loop. + let result; + (*lit_buffer, result) = append_literals(core::mem::take(lit_buffer), lit_len, literals); + result +} + +/// How much of the buffer of the literals is cleared ahead of them. +const LITERALS_AHEAD: usize = 1 << 12; + +/// [`write_literals`] beyond the length of `lit_buffer`, which gets longer than that takes. +#[cold] +#[inline(never)] +fn append_literals( + mut lit_buffer: Vec, + lit_len: usize, + literals: &[u8], +) -> (Vec, Result<()>) { + let result = append_at(&mut lit_buffer, lit_len, literals); + if result.is_ok() { + let len = (lit_buffer.len() + LITERALS_AHEAD).min(lit_buffer.capacity()); + lit_buffer.resize(len, 0); + } + (lit_buffer, result) +} + /// A [`SeqStore`] while a hot loop adds to it: what changes with every sequence is here, in a local variable, which can /// stay in registers. It goes back to the store on drop. -pub(crate) struct SeqSink<'a> { +/// +/// Up to `SHORT` literals, which is at most [`WILDCOPY_OVERLENGTH`], that many bytes are copied whatever their number. +/// `GROWS` is whether `lit_buffer` gets longer as literals are added. Else all of it is cleared first, for a loop that has +/// no register to spare for a buffer that moves. +pub(crate) struct SeqSink<'a, const SHORT: usize, const GROWS: bool> { store: &'a mut SeqStore, sequences: Vec, lit_buffer: Vec, lit_len: u32, } -impl<'a> SeqSink<'a> { +impl<'a, const SHORT: usize, const GROWS: bool> SeqSink<'a, SHORT, GROWS> { #[inline(always)] pub(crate) fn new(store: &'a mut SeqStore) -> Self { + let mut lit_buffer = core::mem::take(&mut store.lit_buffer); + if !GROWS { + lit_buffer.resize(lit_buffer.capacity(), 0); + } Self { sequences: core::mem::take(&mut store.sequences), - lit_buffer: core::mem::take(&mut store.lit_buffer), + lit_buffer, lit_len: store.lit_len as u32, store, } } - /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `src` from `literals` on. Up to `SHORT` of them, which - /// is at most [`WILDCOPY_OVERLENGTH`], that many bytes are copied whatever their number. + /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `src` from `literals` on. #[inline(always)] - pub(crate) fn store_seq( + pub(crate) fn store_seq( &mut self, lit_length: u32, src: Near<'_>, @@ -493,10 +530,14 @@ impl<'a> SeqSink<'a> { room.and_then(<[u8]>::first_chunk_mut::), ) { (Some(from), Some(to)) if lit_length as usize <= SHORT => *to = *from, - _ => self.lit_buffer.write_at( - lit_len, - src.bytes().span(literals as usize, lit_length as usize)?, - )?, + _ => { + let literals = src.bytes().span(literals as usize, lit_length as usize)?; + if GROWS { + write_literals(&mut self.lit_buffer, lit_len, literals)?; + } else { + self.lit_buffer.write_at(lit_len, literals)?; + } + } } self.lit_len += lit_length; self.store_seq_only(lit_length, off_base, match_length) @@ -530,7 +571,7 @@ impl<'a> SeqSink<'a> { } } -impl Drop for SeqSink<'_> { +impl Drop for SeqSink<'_, SHORT, GROWS> { #[inline(always)] fn drop(&mut self) { self.store.sequences = core::mem::take(&mut self.sequences); @@ -2458,6 +2499,107 @@ bun_pod::impl_zeroable!(Match, RawSeq, LdmEntry); mod tests { use super::*; + const BLOCK_SIZE: usize = 16_000; + const FULL_LIT_BUFFER: usize = BLOCK_SIZE + WILDCOPY_OVERLENGTH; + + fn block() -> Vec { + let mut block = Vec::from_iter(0..251).repeat(BLOCK_SIZE.div_ceil(251)); + block.truncate(BLOCK_SIZE); + block + } + + fn store_at_the_start_of_a_block() -> Result { + let mut store = SeqStore::new(); + store.set_limits(BLOCK_SIZE / 3, BLOCK_SIZE); + store.start_block(BLOCK_SIZE)?; + Ok(store) + } + + fn the_buffer_of_the_literals_follows_them() -> Result<()> { + const LAST: usize = 4000; + let block = block(); + let mut store = store_at_the_start_of_a_block()?; + // Of the second block, which finds the buffer as the first one left it, more is literals. + for lit_lengths in [ + [0, 3, 16, 33, 4200, 1, 0, 100], + [1, 9000, 32, 0, 2, 2, 2, 2], + ] { + store.start_block(BLOCK_SIZE)?; + let len_before = store.lit_buffer.len(); + let mut expected = Vec::new(); + let mut sink = SeqSink::::new(&mut store); + let mut pos = 0; + for lit_length in lit_lengths { + sink.store_seq(lit_length, Near::new(&block)?, pos, 1, MINMATCH)?; + expected.extend_from_slice(block.span(pos as usize, lit_length as usize)?); + pos += lit_length + MINMATCH; + } + drop(sink); + assert_eq!(store.literals()?, expected); + let len = store.lit_buffer.len(); + assert!(len >= expected.len() + SHORT); + assert!(len <= len_before.max(expected.len() + LITERALS_AHEAD)); + + store.store_last_literals(&block, LAST)?; + expected.extend_from_slice(block.sub(BLOCK_SIZE - LAST..)?); + assert_eq!(store.literals()?, expected); + } + Ok(()) + } + + #[test] + fn the_buffer_of_the_literals_follows_them_in_a_sink() -> Result<()> { + the_buffer_of_the_literals_follows_them::<16>()?; + the_buffer_of_the_literals_follows_them::<32>() + } + + #[test] + fn the_buffer_of_the_literals_follows_them_up_to_a_whole_block() -> Result<()> { + let block = block(); + let mut store = store_at_the_start_of_a_block()?; + let mut expected = Vec::new(); + let mut pos = 0; + for lit_length in [5, 17, 6000] { + store.store_seq(lit_length, &block, pos, 1, MINMATCH as usize)?; + expected.extend_from_slice(block.span(pos, lit_length)?); + pos += lit_length + MINMATCH as usize; + } + assert_eq!(store.literals()?, expected); + assert!(store.lit_buffer.len() < FULL_LIT_BUFFER); + store.store_last_literals(&block, BLOCK_SIZE - expected.len())?; + expected.extend_from_slice(block.sub(expected.len()..)?); + assert_eq!(store.literals()?, expected); + + // What does not fit leaves the buffer where it is. + let too_many = WILDCOPY_OVERLENGTH + 1; + assert!(store.store_last_literals(&block, too_many).is_err()); + assert_eq!(store.lit_buffer.capacity(), FULL_LIT_BUFFER); + + let mut store = store_at_the_start_of_a_block()?; + store.lit_len = 1; + assert!(store.store_last_literals(&block, 1).is_err()); + assert!(store.literals().is_err()); + Ok(()) + } + + #[test] + fn a_buffer_of_literals_that_does_not_grow_is_as_long_as_it_gets() -> Result<()> { + let block = block(); + // Whatever it was left like. + for left in [0, 100] { + let mut store = store_at_the_start_of_a_block()?; + store.store_last_literals(&block, left)?; + let mut sink = SeqSink::<16, false>::new(&mut store); + sink.store_seq(9000, Near::new(&block)?, 0, 1, MINMATCH)?; + drop(sink); + assert_eq!(store.lit_buffer.len(), FULL_LIT_BUFFER); + let (before, literals) = store.literals()?.split_at_checked(left).ok_or(BUG)?; + assert_eq!(before, block.sub(BLOCK_SIZE - left..)?); + assert_eq!(literals, block.sub(..9000)?); + } + Ok(()) + } + #[test] fn the_hash_of_4_bytes_is_the_same_from_4_bytes() -> Result<()> { for h_bits in [6, 17, 32] { diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index cbcad0871295..585a78163903 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -2129,7 +2129,7 @@ fn compress_block_lazy_part<'a, F: Finder<'a>, const LAZY: bool, const DICT_MODE return Ok(Parsed::Block((near.len() - start.anchor) as usize)); }; let mut finder = F::new(ms, src, *block)?; - let mut seq_store = SeqSink::new(seq_store); + let mut seq_store = SeqSink::<16, true>::new(seq_store); let mut ip = start.ip; let mut anchor = start.anchor; let stop = match F::TRIAL { @@ -2255,7 +2255,7 @@ fn compress_block_lazy_part<'a, F: Finder<'a>, const LAZY: bool, const DICT_MODE } // Store the sequence. - seq_store.store_seq::<16>(start - anchor, near, anchor, best.off_base.get(), best.len)?; + seq_store.store_seq(start - anchor, near, anchor, best.off_base.get(), best.len)?; ip = start + best.len; anchor = ip; @@ -2272,7 +2272,7 @@ fn compress_block_lazy_part<'a, F: Finder<'a>, const LAZY: bool, const DICT_MODE break; } (offset_1, offset_2) = (offset_2, offset_1); - seq_store.store_seq::<16>(0, near, anchor, REPCODE1_TO_OFFBASE, rep_len)?; + seq_store.store_seq(0, near, anchor, REPCODE1_TO_OFFBASE, rep_len)?; ip += rep_len; anchor = ip; } diff --git a/src/zstd/compress/superblock.rs b/src/zstd/compress/superblock.rs index 7aed3dbac5cc..b78065c5b7dc 100644 --- a/src/zstd/compress/superblock.rs +++ b/src/zstd/compress/superblock.rs @@ -414,7 +414,7 @@ fn compress_sub_block_multi( let sequences = seq_store.sequences.as_slice(); let nb_seqs = sequences.len(); let mut sp = 0usize; - let literals = seq_store.literals(); + let literals = seq_store.literals()?; let nb_literals = literals.len(); let mut lp = 0usize; let mut ip = 0usize; diff --git a/src/zstd/util.rs b/src/zstd/util.rs index 9995e78537e7..b03f1bb26085 100644 --- a/src/zstd/util.rs +++ b/src/zstd/util.rs @@ -191,6 +191,16 @@ pub(crate) fn append(to: &mut Vec, from: &[u8]) -> Result<()> { Ok(()) } +/// [`append`], in the place of what `to` has from `pos` on. +#[inline] +pub(crate) fn append_at(to: &mut Vec, pos: usize, from: &[u8]) -> Result<()> { + if pos > to.len() { + return Err(BUG); + } + to.truncate(pos); + append(to, from) +} + /// `to.extend_from_within(start..start + len)`, without a new allocation. #[inline] pub(crate) fn append_within(to: &mut Vec, start: usize, len: usize) -> Result<()> { From dd2ac327ca9cea7eb6977b4134c6516d7ca263eb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 07:17:39 +0000 Subject: [PATCH 181/199] zstd: which of two positions is the smaller one is told from 16 bytes at once On the way down a binary tree, a step compared 16 bytes, looked for the first ones that differ and read these two again to tell which way to go on, and the branch that depends on it waited for all of that. The 16 bytes as two big-endian numbers tell the same without waiting for where they differ (`Common::within`), and that needs no vectoriser. `compare_head` does so. `compare_chunks` comes in two shapes: one loop that does so at its exit, for btlazy2, where `compare_head` has had the first bytes, and for the trees of dictionaries; and, for the optimal parser, the first 16 bytes apart, where most positions differ, and a loop that reads the two bytes as before, since after 16 bytes that are the same the new way takes longer than it saves. Time relative to before, geometric mean of 25 kinds of data, each compressed many times over in one process: 128 KB at levels 11 and 12 0.964 and 0.962 (DNA and half noise 0.90, text 0.93, JSON 0.945), which is 1.000 of C at both (1.037 and 1.040 before); 64 KB at levels 11, 13, 16, 19 0.965, 0.979, 0.981, 0.983; 16 KB 0.979 to 0.983; 1 MB at levels 13 to 15, 14 kinds, 0.985, 0.980, 0.981. --- src/zstd/compress/internal.rs | 98 ++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 13 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index eaa609f898eb..c151ef13b965 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -901,7 +901,7 @@ impl<'a> Near<'a> { /// the order of what follows them. Only the sum of `match_pos` and `from` has to be a position. #[inline(always)] pub(crate) fn compare(self, pos: u32, match_pos: u32, from: u32) -> Common { - let mut len = match self.compare_chunks(pos, match_pos, from) { + let mut len = match self.compare_chunks::(pos, match_pos, from) { Ok(common) => return common, Err(len) => len, }; @@ -917,20 +917,20 @@ impl<'a> Near<'a> { } /// [`Self::compare`], 16 bytes at a time as long as there are as many. Fails with how many bytes are known to be in - /// common when there are not. + /// common when there are not. With `FIRST_APART`, the first 16 bytes, where most differ, have code of their own. #[inline(always)] - pub(crate) fn compare_chunks( + pub(crate) fn compare_chunks( self, pos: u32, match_pos: u32, from: u32, ) -> core::result::Result { - self.compare_chunks_with(pos, self, match_pos, from) + self.compare_chunks_with::(pos, self, match_pos, from) } /// [`Self::compare_chunks`], for a match that is at `match_pos` in `other`. #[inline(always)] - pub(crate) fn compare_chunks_with( + pub(crate) fn compare_chunks_with( self, pos: u32, other: Self, @@ -938,6 +938,19 @@ impl<'a> Near<'a> { from: u32, ) -> core::result::Result { let mut len = from; + if FIRST_APART + && let (Some(bytes), Some(match_bytes)) = ( + self.chunk16(pos.wrapping_add(len)), + other.chunk16(match_pos.wrapping_add(len)), + ) + { + let differ = differences(bytes, match_bytes); + if differ != 0 { + let len = len + differ.trailing_zeros(); + return Ok(Common::within(len, match_bytes, bytes)); + } + len += 16; + } while let (Some(bytes), Some(match_bytes)) = ( self.chunk16(pos.wrapping_add(len)), other.chunk16(match_pos.wrapping_add(len)), @@ -945,6 +958,10 @@ impl<'a> Near<'a> { let differ = differences(bytes, match_bytes); if differ != 0 { let first = differ.trailing_zeros(); + if !FIRST_APART { + return Ok(Common::within(len + first, match_bytes, bytes)); + } + // After 16 bytes that are the same, that takes longer than it saves. let byte = bytes.get(first as usize & 15).copied().unwrap_or(0); let match_byte = match_bytes.get(first as usize & 15).copied().unwrap_or(0); return Ok(Common::followed_by(len + first, match_byte, byte)); @@ -963,9 +980,7 @@ impl<'a> Near<'a> { if first >= 16 || first < from { return None; } - let byte = head.get(first as usize & 15).copied().unwrap_or(0); - let match_byte = match_bytes.get(first as usize & 15).copied().unwrap_or(0); - Some(Common::followed_by(first, match_byte, byte)) + Some(Common::within(first, match_bytes, head)) } /// A hint that `pos`, which may be anything, is about to be read. @@ -1087,6 +1102,16 @@ impl Common { } } + /// [`Self::followed_by`] for bytes that differ somewhere. Which ones are smaller does not wait for where they differ. + #[inline(always)] + pub(crate) fn within(len: u32, match_bytes: &[u8; 16], bytes: &[u8; 16]) -> Self { + debug_assert!(match_bytes != bytes); + Self { + len, + match_is_smaller: Some(u128::from_be_bytes(*match_bytes) < u128::from_be_bytes(*bytes)), + } + } + #[inline(always)] pub(crate) const fn up_to_the_end(len: u32) -> Self { Self { @@ -1379,10 +1404,17 @@ pub(crate) fn bt_walk( let match_pos = match_index.wrapping_sub(dict_limit); let common = match head.and_then(|head| near.compare_head(head, match_pos, from)) { Some(common) => common, - None => match near.compare_chunks(ip, match_pos, from) { - Ok(common) => common, - Err(from) => input.compare_rest(ip, match_index, from)?, - }, + None => { + let by_chunks = if BY_HEAD { + near.compare_chunks::(ip, match_pos, from) + } else { + near.compare_chunks::(ip, match_pos, from) + }; + match by_chunks { + Ok(common) => common, + Err(from) => input.compare_rest(ip, match_index, from)?, + } + } }; if !visit(match_index, common)? { break; @@ -1508,7 +1540,7 @@ impl<'a> DictTree<'a> { let match_pos = dict_match_index.wrapping_sub(self.low_limit); let chunks = input .near - .compare_chunks_with(ip, self.content, match_pos, from); + .compare_chunks_with::(ip, self.content, match_pos, from); let common = match chunks { Ok(common) => common, Err(from) => self.compare_rest(input, ip, dict_match_index, from)?, @@ -2614,6 +2646,46 @@ mod tests { Ok(()) } + #[test] + fn the_head_of_a_position_is_compared_as_all_of_it() -> Result<()> { + const POS: u32 = 40; + for len in 0..=17 { + for (byte, match_byte) in [(1, 2), (2, 1), (0x7F, 0x80), (0x80, 0x7F)] { + // What follows the difference is in the other order. + let mut bytes = [vec![byte; POS as usize], vec![match_byte; POS as usize]].concat(); + bytes.sub_mut(..len as usize)?.fill(9); + bytes.sub_mut(POS as usize..(POS + len) as usize)?.fill(9); + *bytes.at_mut(len as usize)? = match_byte; + *bytes.at_mut((POS + len) as usize)? = byte; + let near = Near::new(&bytes)?; + let head = near.chunk16(POS).ok_or(BUG)?; + for from in [0, len / 2, len] { + for common in [ + near.compare_chunks::(POS, 0, from), + near.compare_chunks::(POS, 0, from), + ] { + assert_eq!( + common.map(|common| (common.len, common.match_is_smaller)), + Ok((len, Some(match_byte < byte))), + "{len} {from}" + ); + } + } + for from in [0, len, len + 1] { + let common = near.compare_head(head, 0, from); + let common = common.map(|common| (common.len, common.match_is_smaller)); + let told = len < 16 && from <= len; + assert_eq!( + common, + told.then_some((len, Some(match_byte < byte))), + "{len} {from}" + ); + } + } + } + Ok(()) + } + #[test] fn matches_are_counted_up_to_where_they_end() -> Result<()> { const DISTANCE: u32 = 7; From 8f8c6f4812166e017dffaa6d59fcd0d8b7201fb3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 07:18:09 +0000 Subject: [PATCH 182/199] zstd: the optimal parser gives fewer hints while it adds the positions that a match skipped On inputs of more than 64 KB, every step down a tree hints at the two nodes and the two runs of bytes that the next step may read: 14 of its 36 instructions. The positions that a match skipped go down the trees past the positions of the match, which follow each other too, so most of that is at hand already. On HTML and records, nine tenths of the time at level 13 goes to adding them. A run of 32 positions or more is now added without these hints where a way down is at most 32 positions, and with the hints at the bytes only where it is longer: without those, records of 1 KB take 1.3 to 5% longer there. The hints are a parameter of `bt_walk` (`Hints`). They do not change the output. Time relative to before, this change alone: 128 KB, 25 kinds, levels 13 and 14 0.985 (HTML 0.93, records of 100 bytes 0.94, of 1 KB 0.965), levels 15 to 22 0.995 to 0.998; 1 MB, 14 kinds, levels 16 to 19 0.979, 0.981, 0.992, 0.997 (HTML 0.945 to 0.96). With the commit before: 128 KB at levels 13 to 22 0.966 to 0.984, 1 MB at levels 16 to 19 0.969, 0.968, 0.979, 0.986. Of 548 rows (16 KB to 1 MB, levels 11 to 22), 20 take more than 0.7% longer and none more than 2% once measured again; records of 100 bytes at levels 16 to 22, the most constant of them, lose it in functions that did not change. Also measured, with an earlier shape of the comparison: 4 MB (0.93 to 1.006), 48 MB (0.96 to 0.99), 64 MB with long distance matches from beyond the trees (0.97 to 1.008), and a dictionary of 64 KB (0.935 to 1.005). --- src/zstd/compress/internal.rs | 20 +++++++++++++++++--- src/zstd/compress/lazy.rs | 6 +++--- src/zstd/compress/opt.rs | 35 +++++++++++++++++++++++------------ src/zstd/lib.rs | 7 ++++++- 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index c151ef13b965..d5e8570c7263 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1,6 +1,7 @@ //! `zstd_compress_internal.h`: what the parts of the compressor share. use core::hint::{Locality, prefetch_read}; +use core::marker::ConstParamTy; use core::sync::atomic::{AtomicBool, Ordering, compiler_fence}; use std::sync::Arc; @@ -1369,12 +1370,21 @@ pub(crate) fn bt_look_ahead( } } +/// What each step of [`bt_walk`] tells, where `input.far`, of what the next one may read. Not [`bt_look_ahead`]. +#[derive(Clone, Copy, PartialEq, Eq, ConstParamTy)] +pub(crate) enum Hints { + /// The nodes and the bytes. + All, + Bytes, + None, +} + /// Goes down the tree whose root is `root` for the position `ip` of `input.near`, which `descent` is for and which /// becomes the root. `visit` is called with each position that is met on the way, which are all at least `low` and at /// most `nb_compares`, and what it has in common with `ip`, and returns whether to go on. Returns how many more positions /// could have been compared. `BY_HEAD` is whether to try [`Near::compare_head`], which is for ways down that are short. #[inline(always)] -pub(crate) fn bt_walk( +pub(crate) fn bt_walk( bt: &mut Nodes, input: &TreeInput<'_>, ip: u32, @@ -1397,8 +1407,12 @@ pub(crate) fn bt_walk( // Whichever way it goes, the next step does not wait as long for memory. if input.far { for child in children { - bt_prefetch(bt, child); - near.prefetch(child.wrapping_sub(dict_limit).wrapping_add(from)); + if matches!(HINTS, Hints::All) { + bt_prefetch(bt, child); + } + if matches!(HINTS, Hints::All | Hints::Bytes) { + near.prefetch(child.wrapping_sub(dict_limit).wrapping_add(from)); + } } } let match_pos = match_index.wrapping_sub(dict_limit); diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 585a78163903..35cfb8ea040b 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -6,7 +6,7 @@ use core::ops::Range; use super::internal::{ BlockSrc, CompressionParameters, Descent, DictMatchState, DictMode, DictTree, Hash8, Hasher, - K_SEARCH_STRENGTH, MatchState, Near, Nodes, Prefix, REPCODE1_TO_OFFBASE, RowTags, + Hints, K_SEARCH_STRENGTH, MatchState, Near, Nodes, Prefix, REPCODE1_TO_OFFBASE, RowTags, SecondTagStats, SeqSink, SeqStore, Strategy, TreeInput, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, bt_children, bt_look_ahead, bt_low, bt_nodes, bt_set_children, bt_walk, count_2segments, hash_ptr, hashed_swap, index_overlap_check, offbase_is_offset, @@ -219,7 +219,7 @@ impl<'a> Dubt<'a> { let mut search_dict = true; match_index = table_get(self.hash_table, h); table_put(self.hash_table, h, curr); - let nb_compares = bt_walk::( + let nb_compares = bt_walk::( self.bt, &self.input, ip, @@ -438,7 +438,7 @@ fn sort_dubt1( ) -> Result<()> { // The next sorted candidate is in the first slot. The other one has the previous unsorted candidate. let [next_sorted, _] = bt_children(bt, curr); - bt_walk::( + bt_walk::( bt, input, curr.wrapping_sub(input.dict_limit), diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index 7fd458c19481..635a606b739b 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -6,8 +6,8 @@ use core::mem; use super::hist; use super::huf::HufRepeat; use super::internal::{ - BlockSrc, Common, Descent, DictMatchState, DictMode, DictTree, EntropyCTables, Hasher, Match, - MatchState, Near, Nodes, OptState, ParamSwitch, Prefix, PriceTable, RawSeq, RawSeqStore, + BlockSrc, Common, Descent, DictMatchState, DictMode, DictTree, EntropyCTables, Hasher, Hints, + Match, MatchState, Near, Nodes, OptState, ParamSwitch, Prefix, PriceTable, RawSeq, RawSeqStore, SeqStore, Strategy, TreeInput, ZSTD_OPT_NUM, bt_look_ahead, bt_nodes, bt_set_children, bt_walk, hash3_of, index_overlap_check, ll_code, ml_code, new_rep, offset_to_offbase, table_get, table_put, @@ -20,6 +20,10 @@ use crate::util::{BUG, SliceExt, highbit32, read_le32}; /// How much a literal adds to its frequency, so that frequencies adapt fast to new statistics. const ZSTD_LITFREQ_ADD: u32 = 2; +/// From that many positions on, [`Trees::update_tree`] gives fewer hints on the way down. +const LONG_RUN: u32 = 32; +/// And none if the way down is at most that many positions: further down, it gets to bytes that are not at hand. +const SHORT_WAY_DOWN: u32 = 32; const ZSTD_MAX_PRICE: i32 = 1 << 30; /// A first block of at most this size is priced by fixed rules, not by statistics. const ZSTD_PREDEF_THRESHOLD: usize = 8; @@ -567,7 +571,7 @@ impl<'a> Trees<'a> { /// that is met on the way, which are all at least `low`, and what it has in common with `ip`, and returns whether /// to go on. Returns how many more positions could have been compared. #[inline(always)] - fn walk( + fn walk( &mut self, ip: u32, bytes: u64, @@ -589,7 +593,7 @@ impl<'a> Trees<'a> { } let descent = Descent::new(self.bt, curr); - bt_walk::( + bt_walk::( self.bt, &self.input, ip, @@ -604,7 +608,7 @@ impl<'a> Trees<'a> { /// `ZSTD_insertBt1`: adds `curr`, which is at least 8 bytes before the end, to its tree. `target` is up to where /// the caller is filling them. Returns how many positions to move forward. #[inline(always)] - fn insert_bt1(&mut self, curr: u32, target: u32) -> Result { + fn insert_bt1(&mut self, curr: u32, target: u32) -> Result { let ip = curr.wrapping_sub(self.input.dict_limit); let bytes = self.input.near.read64(ip).ok_or(BUG)?; // Only the positions that are still in the window once the trees are filled are of use. @@ -612,7 +616,7 @@ impl<'a> Trees<'a> { let mut match_end_idx = curr.wrapping_add(8 + 1); let mut best_length = 8u32; - self.walk(ip, bytes, window_low, |match_index, common| { + self.walk::(ip, bytes, window_low, |match_index, common| { if common.len > best_length { cold_path(); best_length = common.len; @@ -635,17 +639,24 @@ impl<'a> Trees<'a> { /// `ZSTD_updateTree_internal`: adds the positions from `next_to_update` up to `target` to the trees. #[inline(always)] fn update_tree(&mut self, target: u32) -> Result<()> { - self.update_tree_in_part(target, target)?; + if target.saturating_sub(self.next_to_update) < LONG_RUN || !self.input.far { + self.update_tree_in_part::<{ Hints::All }>(target, target)?; + // A match skipped them: they go down past its positions, which follow each other too and so are mostly at hand. + } else if self.nb_compares > SHORT_WAY_DOWN { + self.update_tree_in_part::<{ Hints::Bytes }>(target, target)?; + } else { + self.update_tree_in_part::<{ Hints::None }>(target, target)?; + } self.next_to_update = target; Ok(()) } /// The part of [`Self::update_tree`] that gets to `up_to`, or somewhat beyond. #[inline(never)] - fn update_tree_in_part(&mut self, up_to: u32, target: u32) -> Result<()> { + fn update_tree_in_part(&mut self, up_to: u32, target: u32) -> Result<()> { let mut idx = self.next_to_update; while idx < up_to { - idx = idx.wrapping_add(self.insert_bt1(idx, target)?); + idx = idx.wrapping_add(self.insert_bt1::(idx, target)?); } self.next_to_update = idx; Ok(()) @@ -839,7 +850,7 @@ impl<'a> Trees<'a> { ) -> Result { let curr = ip.wrapping_add(self.input.dict_limit); let mut is_over = false; - let nb_compares = self.walk(ip, bytes, match_low, |match_index, common| { + let left = self.walk::<{ Hints::All }>(ip, bytes, match_low, |match_index, common| { if common.len > found.best_length { if common.len > found.match_end_idx.wrapping_sub(match_index) { found.match_end_idx = match_index.wrapping_add(common.len); @@ -861,7 +872,7 @@ impl<'a> Trees<'a> { } Ok(true) })?; - Ok(if is_over { 0 } else { nb_compares }) + Ok(if is_over { 0 } else { left }) } /// [`Self::get_all_matches`] for `ip`, which comes after literals, and then for the positions after it, before @@ -1010,7 +1021,7 @@ pub(crate) fn update_tree( ms.check_cancelled()?; let up_to = ms.next_to_update.saturating_add(1 << 12).min(target); let mut trees = Trees::new(ms, prefix, no_dict, None, iend, mls)?; - trees.update_tree_in_part(up_to, target)?; + trees.update_tree_in_part::<{ Hints::All }>(up_to, target)?; ms.next_to_update = trees.next_to_update; } ms.next_to_update = target; diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 7f102e7823f0..ec16e118e192 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -1,6 +1,11 @@ //! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the //! same bytes in, in both directions, and the same errors. Where that is not so, a comment tells what the C library does. -#![feature(hint_prefetch, mpmc_channel, vec_push_within_capacity)] +#![feature( + adt_const_params, + hint_prefetch, + mpmc_channel, + vec_push_within_capacity +)] #![forbid(unsafe_code)] // Frames come from anywhere and a panic aborts the process. #![deny( From d0d121dc9269ba4fce63a88855f5732df4bf7be5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 13:00:47 +0000 Subject: [PATCH 183/199] zstd: a match in the older segment of the window that ends within 16 bytes is told at once With a dictionary whose tables are copied (inputs of more than 32 KB for `btlazy2` and `btopt`, 8 KB for `btultra`), and in a stream that is longer than its window, the binary trees lead to positions in the older segment. Each of them is compared out of line, by `compare_rest`, which counted what is common in two segments and then read the two bytes that differ: with a dictionary of 64 KB, 58% of the time for records of 1 KB at level 16, three and a half times what C spends on the same. `compare_rest` now starts with the next 16 bytes, if the match has them in the older segment, as the loops do for the current one. What it did before is for the matches that go on, in a function of its own (`compare_to_the_end`), so that the one that is called needs no register saved. To go on 16 bytes at a time takes longer than to count, for records of 1 KB. Nothing else changes: all other functions are the same instructions as before, and it is 208 bytes more code. The output does not change either. Time relative to before, 11 kinds of data at levels 11 to 19, means of 44 or 55 rows: a dictionary of 64 KB and inputs of 16, 128, 256 and 512 KB 0.963, 0.959, 0.971, 0.970; one of 30 KB and inputs of 40, 150 and 512 KB 0.969, 0.985, 0.992. Relative to C, 64 KB and 128 KB: 1.038 before, 0.996 now. There, at levels 13, 16 and 19: records of 100 bytes 0.83, 0.75, 0.78, of 1 KB 0.87, 0.775, 0.78, JSON 0.98, 0.95, 0.965, HTML 0.99, 0.96, 0.98. Records are still behind C: those of 100 bytes at 1.36, 1.10 and 0.89, those of 1 KB at 1.24, 1.38 and 1.36. Of the 352 rows, 5 were more than 0.7% slower than both of two runs of the code before, 1.5% at most; measured again they are at 0.99 to 1.002. Streams of 1 MB with windows of 128 and 256 KB, 28 rows: 0.98 to 1.01, where the code before, measured twice, is at 0.99 to 1.01. Without a dictionary this function is where comparisons go for the last 16 bytes of the input, now by one more call. 25 kinds, inputs of 1, 4, 16 and 128 KB at levels 11 to 19, 14 groups: the means of the best times are 1.000 to 1.003, the highest at 1 KB. The 7 rows furthest off are at 0.994 to 1.005 when measured again, where the code before, measured twice, is at 0.98 to 1.004. All of this is from builds with every function at a multiple of 64 bytes. They are placed at multiples of 16, and where one starts within a cache line has been worth up to 1.2% at these levels for the same instructions. Two unit tests, which see 14 of 15 faults planted in the new code (`>` for `>=` in `is_current` changes nothing). The test in `zstd.test.ts` compares with what Bun 1.4.4 writes, as all of them there, so it passes before this change too; all of its digests change with a fault in `compare_older`. To compare in the loops themselves, in second copies of the tree functions and of the optimal parser for where there is an older segment, gains half as much again on records (0.63 and 0.67 at level 16) for 24 KB of code. But these loops are 1 to 6.5% slower wherever the trees seldom lead to the older segment, which depends on the data: with dictionaries of 30 and 64 KB and inputs of 150 to 512 KB, 12 to 22 rows of 44 to 55 took longer than before. Not done. --- src/zstd/compress/internal.rs | 169 +++++++++++++++++++++++++++++++++- test/js/bun/util/zstd.test.ts | 42 +++++++++ 2 files changed, 209 insertions(+), 2 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index d5e8570c7263..891e6138087e 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1314,13 +1314,55 @@ impl<'a> TreeInput<'a> { ) } - /// [`Near::compare`] wherever the match is, for what [`Near::compare_chunks`] leaves. + /// Whether a match at `match_index` is in the current segment once it is `len` long. + #[inline(always)] + fn is_current(&self, match_index: u32, len: u32) -> bool { + u64::from(match_index) + u64::from(len) >= u64::from(self.dict_limit) + } + + /// [`Near::compare_chunks`] for the next 16 bytes alone, if the match has them in the older segment. + #[inline(always)] + fn compare_older( + &self, + ip: u32, + match_index: u32, + from: u32, + ) -> core::result::Result { + if self.is_current(match_index, from) { + return Err(from); + } + let older = self.dict.buf.get(..self.dict.pos_of(self.dict_limit)); + let match_pos = self.dict.pos_of(match_index + from); + let (Some(bytes), Some(match_bytes)) = ( + self.near.chunk16(ip.wrapping_add(from)), + older.and_then(|older| older.get(match_pos..)?.first_chunk()), + ) else { + return Err(from); + }; + let differ = differences(bytes, match_bytes); + if differ == 0 { + return Err(from + 16); + } + let len = from + differ.trailing_zeros(); + Ok(Common::within(len, match_bytes, bytes)) + } + + /// [`Near::compare`] wherever the match is, for what [`Near::compare_chunks`] leaves. Small, for a cheap call. #[cold] #[inline(never)] fn compare_rest(self, ip: u32, match_index: u32, from: u32) -> Result { + match self.compare_older(ip, match_index, from) { + Ok(common) => Ok(common), + Err(from) => self.compare_to_the_end(ip, match_index, from), + } + } + + /// [`Self::compare_rest`], however long the match is. + #[inline(never)] + fn compare_to_the_end(&self, ip: u32, match_index: u32, from: u32) -> Result { // A position of the older segment, as one of the current segment, only makes sense with a length that brings // it there. - if u64::from(match_index) + u64::from(from) >= u64::from(self.dict_limit) { + if self.is_current(match_index, from) { let match_pos = match_index.wrapping_sub(self.dict_limit); return Ok(self.near.compare(ip, match_pos, from)); } @@ -2700,6 +2742,129 @@ mod tests { Ok(()) } + /// `older`, whose first `dict_limit - low` bytes are the older segment, and then the first `end` bytes of `current`. + fn two_segments<'a>( + older: &'a [u8], + low: u32, + current: &'a [u8], + dict_limit: u32, + end: u32, + ) -> Result> { + let base_of = |index: u32| 0usize.wrapping_sub(index as usize); + let (buf, base) = (current, base_of(dict_limit)); + let prefix = Prefix { buf, base }; + let (buf, base) = (older, base_of(low)); + TreeInput::new(prefix, Prefix { buf, base }, dict_limit, end as usize) + } + + #[test] + fn the_next_16_bytes_of_a_match_in_the_older_segment_are_compared_at_once() -> Result<()> { + const LOW: u32 = 100; + const OLDER: u32 = 56; + const DICT_LIMIT: u32 = LOW + OLDER; + const IP: u32 = 3; + let input = two_segments(&[0; OLDER as usize], LOW, &[0; 64], DICT_LIMIT, 64)?; + // Not in the older segment, or only up to `from`, also where the sums wrap around. + for (index, from) in [ + (DICT_LIMIT, 0), + (DICT_LIMIT - 2, 2), + (LOW + 5, u32::MAX - 1), + ] { + let common = input.compare_older(IP, index, from); + assert!(common.is_err_and(|known| known == from)); + } + for len in [0, 1, 8, 15, 16, 17] { + for (byte, match_byte) in [(1, 2), (0x80, 0x7F)] { + for start in [0, OLDER - len - 16, OLDER - len - 15, OLDER - len - 1] { + // What follows the difference is in the other order, and the buffer goes on after the segment. + let mut older = vec![byte; OLDER as usize + 32]; + older + .sub_mut(start as usize..(start + len) as usize)? + .fill(9); + *older.at_mut((start + len) as usize)? = match_byte; + let mut current = vec![match_byte; 64]; + current.sub_mut(IP as usize..(IP + len) as usize)?.fill(9); + *current.at_mut((IP + len) as usize)? = byte; + let whole = (len, Some(match_byte < byte)); + // The input ends after the difference, or later. + for end in [IP + len + 1, 64] { + let input = two_segments(&older, LOW, ¤t, DICT_LIMIT, end)?; + for from in [0, len / 2, len] { + let expected = if start + from + 16 > OLDER || IP + from + 16 > end { + Err(from) + } else if len < from + 16 { + Ok(whole) + } else { + Err(from + 16) + }; + let common = input.compare_older(IP, LOW + start, from); + assert_eq!( + common.map(|common| (common.len, common.match_is_smaller)), + expected, + "{len} {start} {end} {from}" + ); + for common in [ + input.compare_rest(IP, LOW + start, from)?, + input.compare_to_the_end(IP, LOW + start, from)?, + ] { + assert_eq!((common.len, common.match_is_smaller), whole); + } + } + } + } + } + } + Ok(()) + } + + #[test] + fn a_match_that_leaves_the_older_segment_goes_on_in_the_current_one() -> Result<()> { + const LOW: u32 = 100; + const OLDER: u32 = 40; + const DICT_LIMIT: u32 = LOW + OLDER; + const IP: u32 = 60; + const END: u32 = IP + 56; + for in_older in [1, 16, 17, 33] { + let lens = [0, 1, 15, 16, 17].map(|more| in_older + more); + for len in [&[0, in_older - 1, END - IP - 1, END - IP], &lens[..]].concat() { + for match_is_smaller in [false, true] { + // The buffer goes on after the segment, with what no match goes on with. + let mut older: Vec = (1..=OLDER as u8).collect(); + older.extend_from_slice(&[0xEE; 32]); + let mut current: Vec = (0..END as u8).map(|pos| 100 + pos).collect(); + let window = [older.sub(..OLDER as usize)?, ¤t].concat(); + let matched = window.sub((OLDER - in_older) as usize..)?; + current + .sub_mut(IP as usize..(IP + len) as usize)? + .copy_from_slice(matched.sub(..len as usize)?); + if let Some(byte) = current.get_mut((IP + len) as usize) { + let match_byte = matched.at(len as usize)?; + *byte = if match_is_smaller { + match_byte + 1 + } else { + match_byte - 1 + }; + } + let input = two_segments(&older, LOW, ¤t, DICT_LIMIT, END)?; + let match_index = DICT_LIMIT - in_older; + for from in [0, len / 2, len] { + let match_pos = match_index.wrapping_sub(DICT_LIMIT); + let common = match input.near.compare_chunks::(IP, match_pos, from) { + Ok(common) => common, + Err(from) => input.compare_rest(IP, match_index, from)?, + }; + assert_eq!( + (common.len, common.match_is_smaller), + (len, (IP + len < END).then_some(match_is_smaller)), + "{in_older} {len} {from}" + ); + } + } + } + } + Ok(()) + } + #[test] fn matches_are_counted_up_to_where_they_end() -> Result<()> { const DISTANCE: u32 = 7; diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 7250ff14e4a5..749c369d6237 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1564,6 +1564,48 @@ describe("the output is that of zstd 1.5.7", () => { `); }); + it("where the binary trees lead to what is before the current segment of the window", () => { + // Fields on both sides of 0x80, so that which record comes first depends on how bytes are compared. + const fields = (size: number, count: number) => { + const records = counted(size, count); + for (let i = 0, x = 5; i < records.length; i += 37) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + if (x & 3) records[i] = 0x78 + (x >>> 28); + } + return records; + }; + const short = fields(100, 800); + const long = fields(1_000, 80); + const window = (level_: number, windowLog: number) => ({ ...level(level_), [c.ZSTD_c_windowLog]: windowLog }); + const cases: [Buffer, zlib.ZstdOptions][] = []; + // After a dictionary. + for (const data of [short, long]) { + const dictionary = data.subarray(0, 30_000); + for (const n of [11, 13, 16]) cases.push([data.subarray(30_000, 80_000), { params: level(n), dictionary }]); + } + // In a buffer that is written to from its start again. + cases.push( + [short, { params: window(11, 14) }], + [short, { params: window(13, 14) }], + [long, { params: window(16, 15) }], + ); + expect(cases.map(([data, options]) => digest(zlib.zstdCompressSync(data, options)))).toMatchInlineSnapshot(` + [ + "3714 c51791f02b5ce6e2", + "3821 722f4103ea02c85", + "2886 7b33aadb1b4c6928", + "3000 f11a3c6cbaee134b", + "3238 8a4a2b286c782c0d", + "2254 d22a01e8a758fe5b", + "6664 47c6eea66da02b71", + "6845 9bca37c99d215a5d", + "3911 a4190024e0f41380", + ] + `); + }); + it("where the kind of data changes within a block, for rows of 64 entries", () => { let x = 11; const next = () => { From 615100a88f9a426ef87d641dcfdef9e650b85a90 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 13:16:42 +0000 Subject: [PATCH 184/199] zstd: a unit test for where the current segment of the window starts The message of `d0d121dc9` says that `>` for `>=` in `TreeInput::is_current` changes nothing. That is wrong. It holds where there is an older segment, which is all that the unit tests had: the match is then looked for at the end of that segment, nothing is found there, and it goes on in the current one. Without an older segment there is no buffer to look in, and a match at the first position of the input that gets to `compare_rest` fails the frame. The fuzzing against the C library saw it in a frame of 17 bytes. The test now has that case, and sees all 15 faults. No other change. --- src/zstd/compress/internal.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 891e6138087e..546c677115db 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -2773,6 +2773,10 @@ mod tests { let common = input.compare_older(IP, index, from); assert!(common.is_err_and(|known| known == from)); } + // The first position of the current segment is not looked for in an older one, which there may not be. + let alone = two_segments(&[], 0, &[0; 64], DICT_LIMIT, 64)?; + let common = alone.compare_rest(IP, DICT_LIMIT, 0)?; + assert_eq!((common.len, common.match_is_smaller), (64 - IP, None)); for len in [0, 1, 8, 15, 16, 17] { for (byte, match_byte) in [(1, 2), (0x80, 0x7F)] { for start in [0, OLDER - len - 16, OLDER - len - 15, OLDER - len - 1] { From 8aec2cdde2bd8e4699e275747fe0c6d7acf22d17 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 16:03:28 +0000 Subject: [PATCH 185/199] zstd: rows with a dictionary that is searched in place: a tighter loop, and no search without a candidate A dictionary is searched in place for inputs of up to 32 KB, or of unknown size. With a dictionary of 16 to 128 KB, levels 5 to 10 do that with the row-based match finder, which had two costs there that it does not have elsewhere. The loop that inserts positions in the rows read the addresses and lengths of the tables, the hasher and the base index from the stack on every pass: 38 to 42 instructions, 9 of them with a stack operand, where the other instances of the same loop have 32 to 36 and 2 or 3. The state of the finder goes by reference to functions that are not inlined, and in the functions for `lazy` and `lazy2` with such a dictionary, which are the largest, LLVM does not move these loads out of the loop. With what follows, those for `greedy` get there too, and HTML takes 20% longer at level 5. For all six, `insert_up_to` now has a loop on copies of the fields: 31 to 36 instructions, 2 with a stack operand. A position whose row has no candidate was searched all the same, because the row of the dictionary may have one. Now that row is looked at right away. If it has none either, that is the end, as it is without a dictionary. If it has, the dictionary alone is searched (`longest_match_in_dict_alone`). Time relative to before: 11 kinds of data at levels 5 to 10, a dictionary of 64 KB and inputs of 1, 4, 16 and 32 KB, one of 30 KB and inputs of 4 and 16 KB, 396 rows. Means for levels 5 to 10: 0.94, 0.86, 0.86, 0.86, 0.83, 0.92. No row takes longer than the better of two runs of the code before. At levels 6 and 9: noise 0.82 and 0.71, base64 0.83 and 0.69, HTML 0.78 and 0.84, records of 100 bytes 0.81 and 0.79, hex digits 0.92 and 0.83, JSON 0.89 and 0.92, text 0.93 and 0.91. Level 9 gains more because in its functions, for rows of 32 entries, the mask of a row took some 55 instructions more than it should, and no longer does. That is the compiler's doing, not this change's. Relative to C, means for levels 5 to 10: 0.95, 1.05, 1.00, 0.93, 0.87, 0.70 before, 0.90, 0.90, 0.86, 0.80, 0.72, 0.65 now. The worst row of each level: 1.05, 1.25, 1.30, 1.25, 1.47, 1.11 before, 0.99, 1.04, 1.05, 1.05, 1.04, 1.02 now. 185 rows were behind C, 58 are: hex digits and skewed bytes, mostly. Nothing else changes. Of the functions of the crate, the six for rows with such a dictionary are different. In the six for `greedy` with rows, without a dictionary and with one that is copied, 7 instructions name another register, and in one for rows with second tags, 3 are in another order. All others are the same instructions as before. Level 5 without a dictionary, 25 kinds and 32 KB to 4 MB, and with a copied one, 11 kinds and 64 to 512 KB: means of 1.000 to 1.005 of the better of two runs, and 1.00. It is 5.7 KB more code. The output does not change. All of this is from builds with every function at a multiple of 64 bytes. Two loops that do the same are not pretty. What else was tried: - Copies in all instances: the loops of the others stay as they are, but their registers are allocated anew, and 4 MB of base64 take 3.5% longer at level 11. - One loop that takes references, to the fields or to copies of them: JSON takes 1.5% longer than before at level 10, where the two loops make it 5% faster. By what Cachegrind attributes to no line, 14.5 million instructions there move values to and from the stack, as before, and 3.5 million with the two loops. - One body for both loops, with the values as arguments: all instances change. - A copy of the whole state, hash cache included: noise of 4 KB at 0.96 instead of 0.81 at level 6. - The loop in a function of its own: 3% slower than inlined at level 5, noise 4 to 9%. - The search as it was for a row without a candidate, after a look at the dictionary's: its mask is computed twice, hex digits 3% slower than before at level 10. With the mask passed on as an `Option`: 2%. - One exit for all cases in `find`: noise 2 to 4% slower at levels 6 to 8. A unit test holds the two loops to the same tables, caches and counts, and sees the 4 faults put into the new one. The test in `zstd.test.ts` compares with what Bun 1.4.4 writes, as all of them there, so it passes before this change too; it fails with each of 8 faults put into the new code. Of 15 faults, runs against the C library with random parameters see 13; the other two cannot change the output (a lower bound for what to look at, and another slot of the hash cache for all who use it). --- src/zstd/compress/lazy.rs | 268 ++++++++++++++++++++++++++-------- test/js/bun/util/zstd.test.ts | 44 ++++++ 2 files changed, 255 insertions(+), 57 deletions(-) diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 35cfb8ea040b..35dfce6f110a 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -1564,6 +1564,19 @@ impl<'a> RowDict<'a> { lowest_index: dms.ms.window.dict_limit, } } + + /// The entries of the row for `hash` that have its tag. + #[inline(always)] + fn candidates( + &self, + hash: u32, + ) -> Option> { + let start = rel_row::(hash); + let tags = self.tags.get(start..start + ROW_ENTRIES)?; + let tags = tags.first_chunk::()?; + let indices = self.indices.get(start..start + ROW_ENTRIES)?; + RowCandidates::new(indices.first_chunk()?, tags, hash as u8) + } } /// The tables of the row-based match finder, and what it takes to add to them. @@ -1581,9 +1594,33 @@ struct RowIndex<'a> { nb_inserted: u32, } +/// The hash of the position at `index`, from the caches of [`RowIndex`], where `new_hash` takes its place: the hash of the +/// position 8 bytes further. +#[inline(always)] +fn replace_cached_hash( + hash_cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], + second_tag_cache: &mut [u8; ZSTD_ROW_HASH_CACHE_SIZE], + index: u32, + new_hash: RowHash, +) -> RowHash { + let cached = (index & ZSTD_ROW_HASH_CACHE_MASK) as usize; + let (Some(hash), Some(second_tag)) = + (hash_cache.get_mut(cached), second_tag_cache.get_mut(cached)) + else { + return new_hash; + }; + RowHash { + hash: core::mem::replace(hash, new_hash.hash), + second_tag: if SECOND_TAGS { + core::mem::replace(second_tag, new_hash.second_tag) + } else { + 0 + }, + } +} + impl RowIndex<'_> { - /// `ZSTD_row_nextCachedHash`: the hash of `pos`, from the cache, where `new_hash` takes its place: the hash of the - /// position 8 bytes further. + /// `ZSTD_row_nextCachedHash`: [`replace_cached_hash`] for `pos`, and a prefetch of the row for `new_hash`. #[inline(always)] fn next_cached_hash( &mut self, @@ -1592,21 +1629,12 @@ impl RowIndex<'_> { ) -> RowHash { self.rows .prefetch::(new_hash.hash); - let cached = (self.base_index.wrapping_add(pos) & ZSTD_ROW_HASH_CACHE_MASK) as usize; - let (Some(hash), Some(second_tag)) = ( - self.hash_cache.get_mut(cached), - self.second_tag_cache.get_mut(cached), - ) else { - return new_hash; - }; - RowHash { - hash: core::mem::replace(hash, new_hash.hash), - second_tag: if SECOND_TAGS { - core::mem::replace(second_tag, new_hash.second_tag) - } else { - 0 - }, - } + replace_cached_hash::( + &mut self.hash_cache, + &mut self.second_tag_cache, + self.base_index.wrapping_add(pos), + new_hash, + ) } /// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `next_to_update` on, or fewer so as not to go past @@ -1625,9 +1653,11 @@ impl RowIndex<'_> { Some(()) } - /// `ZSTD_row_update_internalImpl`: inserts the positions from `next_to_update` up to `end`. + /// `ZSTD_row_update_internalImpl`: inserts the positions from `next_to_update` up to `end`. `COPIES` is for the + /// largest functions, where the loop would read the fields again after every write to the tables. In the others it + /// does not, and with copies, 4 MB of base64 take 3% longer at level 11. #[inline(always)] - fn insert_up_to( + fn insert_up_to( &mut self, end: u32, ) -> Option<()> { @@ -1635,15 +1665,38 @@ impl RowIndex<'_> { if end as usize + 2 * ZSTD_ROW_HASH_CACHE_SIZE > bytes.len() { return None; } - for pos in self.next_to_update..end { - let ahead = pos as usize + ZSTD_ROW_HASH_CACHE_SIZE; - let ahead = bytes.get(ahead..ahead + 8)?.first_chunk()?; - let new_hash = self.hasher.of::(u64::from_le_bytes(*ahead)); - let hash = self.next_cached_hash::(pos, new_hash); - let index = self.base_index.wrapping_add(pos); - self.rows - .row::(hash.hash)? - .insert(hash, index); + if COPIES { + let (hasher, base_index) = (self.hasher, self.base_index); + let mut rows = Rows { + indices: &mut *self.rows.indices, + tags: &mut *self.rows.tags, + }; + for pos in self.next_to_update..end { + let ahead = pos as usize + ZSTD_ROW_HASH_CACHE_SIZE; + let ahead = bytes.get(ahead..ahead + 8)?.first_chunk()?; + let new_hash = hasher.of::(u64::from_le_bytes(*ahead)); + rows.prefetch::(new_hash.hash); + let index = base_index.wrapping_add(pos); + let hash = replace_cached_hash::( + &mut self.hash_cache, + &mut self.second_tag_cache, + index, + new_hash, + ); + rows.row::(hash.hash)? + .insert(hash, index); + } + } else { + for pos in self.next_to_update..end { + let ahead = pos as usize + ZSTD_ROW_HASH_CACHE_SIZE; + let ahead = bytes.get(ahead..ahead + 8)?.first_chunk()?; + let new_hash = self.hasher.of::(u64::from_le_bytes(*ahead)); + let hash = self.next_cached_hash::(pos, new_hash); + let index = self.base_index.wrapping_add(pos); + self.rows + .row::(hash.hash)? + .insert(hash, index); + } } if SECOND_TAGS { self.nb_inserted += end.saturating_sub(self.next_to_update); @@ -1653,15 +1706,18 @@ impl RowIndex<'_> { } /// `ZSTD_row_update_internal`: inserts the positions from `next_to_update` up to `ip`, but not all of them if they - /// are many. + /// are many. See [`Self::insert_up_to`] for `COPIES`. #[inline(always)] - fn update(&mut self, ip: u32) -> Option<()> { + fn update( + &mut self, + ip: u32, + ) -> Option<()> { const K_SKIP_THRESHOLD: u32 = 384; if ip.wrapping_sub(self.next_to_update) > K_SKIP_THRESHOLD { self.skip::(ip)?; } - self.insert_up_to::(ip) + self.insert_up_to::(ip) } /// The part of `ZSTD_row_update_internal` for a long way up to `ip`. @@ -1672,7 +1728,7 @@ impl RowIndex<'_> { const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; let end = self.next_to_update + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; - self.insert_up_to::(end)?; + self.insert_up_to::(end)?; self.next_to_update = ip - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; self.fill_hash_cache::(ip + 1) } @@ -1687,7 +1743,19 @@ struct RowCandidates<'a, const ROW_ENTRIES: usize> { matches: u64, } -impl RowCandidates<'_, ROW_ENTRIES> { +impl<'a, const ROW_ENTRIES: usize> RowCandidates<'a, ROW_ENTRIES> { + /// The entries of a row that have the tag `tag`. + #[inline(always)] + fn new(indices: &'a [u32; ROW_ENTRIES], tags: &[u8; ROW_ENTRIES], tag: u8) -> Option { + let head = u32::from(*tags.first()?) & (ROW_ENTRIES as u32 - 1); + let matches = row_get_match_mask(tags, tag, head); + Some(Self { + indices, + head, + matches: matches & row_entries_mask::(head), + }) + } + /// The most recent one. There is one. #[inline(always)] fn first(&self) -> u32 { @@ -1776,7 +1844,8 @@ impl RowSearch<'_> { } if DICT_MODE == DICT_MATCH_STATE { - self.longest_dict_match::(dict_hash, ip, nb_attempts, &mut longest)?; + let in_dict = self.dms?.candidates::(dict_hash)?; + self.longest_dict_match(in_dict, ip, nb_attempts, &mut longest)?; } let best = longest.best(); if all || best.len >= SECOND_TAG_LENGTH || floor.len + 1 >= SECOND_TAG_LENGTH { @@ -1787,29 +1856,38 @@ impl RowSearch<'_> { } } + /// [`Self::longest_match`] where the row has no candidate, and that of the dictionary has `in_dict`. + #[inline(always)] + fn longest_match_in_dict_alone( + &self, + in_dict: RowCandidates<'_, ROW_ENTRIES>, + ip: u32, + wanted: Wanted, + ) -> Option { + let block = &self.block; + let curr = block.base_index.wrapping_add(ip); + let low_limit = block.reach.low_limit(curr); + let segment = block.near.tail(low_limit.wrapping_sub(block.base_index))?; + let floor = Best { + len: wanted.floor::(curr, low_limit), + off_base: 0, + }; + let mut longest = Longest::new(segment, low_limit, curr, floor)?; + self.longest_dict_match(in_dict, ip, self.nb_attempts, &mut longest)?; + longest.best().found() + } + /// The part of `ZSTD_RowFindBestMatch` that searches the dictionary. #[inline(always)] fn longest_dict_match( &self, - hash: u32, + mut candidates: RowCandidates<'_, ROW_ENTRIES>, ip: u32, mut nb_attempts: u32, longest: &mut Longest<'_>, ) -> Option<()> { let block = &self.block; let dms = self.dms?; - let start = rel_row::(hash); - let tags = dms.tags.get(start..start + ROW_ENTRIES)?; - let tags = tags.first_chunk::()?; - let indices = dms.indices.get(start..start + ROW_ENTRIES)?; - let head = u32::from(*tags.first()?) & (ROW_ENTRIES as u32 - 1); - let matches = row_get_match_mask(tags, hash as u8, head); - let mut candidates = RowCandidates { - indices: indices.first_chunk::()?, - head, - matches: matches & row_entries_mask::(head), - }; - while candidates.matches != 0 && nb_attempts > 0 { let match_index = candidates.first(); candidates.remove_first(); @@ -1943,7 +2021,15 @@ impl<'a, const ROW_ENTRIES: usize, const SECOND_TAGS: bool, const DICT_MODE: u8> index.hasher.of::(bytes) } else { if index.next_to_update != ip { - index.update::(ip).ok_or(BUG)?; + if DICT_MODE == DICT_MATCH_STATE { + index + .update::(ip) + .ok_or(BUG)?; + } else { + index + .update::(ip) + .ok_or(BUG)?; + } } let ahead = index.near.read64(ip + ZSTD_ROW_HASH_CACHE_SIZE as u32); let new_hash = index.hasher.of::(ahead.ok_or(BUG)?); @@ -1959,19 +2045,23 @@ impl<'a, const ROW_ENTRIES: usize, const SECOND_TAGS: bool, const DICT_MODE: u8> .rows .row::(hash.hash) .ok_or(BUG)?; - let head = u32::from(*row.tags.first().ok_or(BUG)?) & (ROW_ENTRIES as u32 - 1); - let matches = row_get_match_mask(row.tags, hash.hash as u8, head) - & row_entries_mask::(head); + let candidates = RowCandidates::new(row.indices, row.tags, hash.hash as u8).ok_or(BUG)?; + let RowCandidates { head, matches, .. } = candidates; let curr = index.base_index.wrapping_add(ip); - if DICT_MODE != DICT_MATCH_STATE && matches == 0 { + if matches == 0 { + if DICT_MODE == DICT_MATCH_STATE { + let dms = search.dms.ok_or(BUG)?; + let in_dict = dms.candidates::(dict_hash).ok_or(BUG)?; + if in_dict.matches != 0 { + let found = search + .longest_match_in_dict_alone::(in_dict, ip, wanted); + row.insert(hash, curr); + return Ok(found); + } + } row.insert(hash, curr); return Ok(None); } - let candidates = RowCandidates { - indices: row.indices, - head, - matches, - }; let second_tags = row.second_tags.as_ref(); let long_matches = second_tags .map(|tags| matches & row_get_second_match_mask(tags, hash.second_tag, head)); @@ -2483,6 +2573,70 @@ mod tests { assert_eq!(nb_kept(&[few, (TRIAL / 4, TRIAL), few]), 3); } + #[test] + fn positions_are_inserted_in_the_rows_the_same_with_copies_of_the_fields() -> Result<()> { + type Inserted = (Vec, Vec, [u32; 8], [u8; 8], u32, u32); + fn inserted( + bytes: &[u8], + ) -> Result { + const ROW_HASH_LOG: u32 = 4; + let mut indices = vec![0; ROW_ENTRIES << ROW_HASH_LOG]; + let mut tags = vec![0; 2 * indices.len()]; + let hash_bits = ROW_HASH_LOG + ZSTD_ROW_HASH_TAG_BITS; + let mut index = RowIndex { + rows: Rows { + indices: &mut indices, + tags: &mut tags, + }, + hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + second_tag_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + hasher: RowHasher(Hash8::new(4, hash_bits, 7)), + near: Near::new(bytes)?, + base_index: 1_003, + next_to_update: 2, + nb_inserted: 0, + }; + index + .fill_hash_cache::(400) + .ok_or(BUG)?; + for end in [3, 60, 60, 400] { + index + .insert_up_to::(end) + .ok_or(BUG)?; + } + let (hashes, second_tags) = (index.hash_cache, index.second_tag_cache); + let (next, nb_inserted) = (index.next_to_update, index.nb_inserted); + Ok((indices, tags, hashes, second_tags, next, nb_inserted)) + } + + let mut x = 9_u32; + let bytes: Vec = core::iter::repeat_with(|| { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + (x >> 30) as u8 + }) + .take(500) + .collect(); + let with_copies = inserted::<16, false, true>(&bytes)?; + assert_eq!(with_copies.4, 400); + assert!(with_copies.0.contains(&(1_003 + 399))); + assert_eq!(with_copies, inserted::<16, false, false>(&bytes)?); + assert_eq!( + inserted::<32, false, true>(&bytes)?, + inserted::<32, false, false>(&bytes)? + ); + assert_eq!( + inserted::<64, false, true>(&bytes)?, + inserted::<64, false, false>(&bytes)? + ); + assert_eq!( + inserted::<64, true, true>(&bytes)?, + inserted::<64, true, false>(&bytes)? + ); + Ok(()) + } + #[test] fn many_positions_are_added_to_the_trees_as_one_by_one() -> Result<()> { const DICT_LIMIT: u32 = 2; diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 749c369d6237..efca75d00064 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1606,6 +1606,50 @@ describe("the output is that of zstd 1.5.7", () => { `); }); + it("where the rows of a dictionary have candidates and those of the input have none", () => { + // Noise of so many bits. What follows the dictionary has pieces of it, and of itself, among the noise. + const withPieces = (bits: number) => { + let x = 5; + const next = () => { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + return (x >>>= 0); + }; + const all = Buffer.alloc(42_000); + for (let at = 0; at < all.length;) { + for (let n = 4 + (next() % 24); n > 0; n--) all[at++] = next() >>> (32 - bits); + if (at < 30_000) continue; + const from = next() % 4 ? next() % 29_000 : 30_000 + (next() % (at - 30_000 + 1)); + const length = 4 + (next() % 12); + all.copyWithin(at, from, from + length); + at += length; + } + return all; + }; + // Rows of 16 entries up to level 8, then of 32 and of 64. + const digests = (all: Buffer) => + [5, 6, 7, 9, 10].map(n => + digest(zlib.zstdCompressSync(all.subarray(30_000), { params: level(n), dictionary: all.subarray(0, 30_000) })), + ); + expect([6, 3].map(bits => digests(withPieces(bits)))).toEqual([ + [ + "7108 e6ce3932970435bf", + "7106 33de66f30821afb9", + "7105 94b0e055b688b475", + "7105 94b0e055b688b475", + "7105 94b0e055b688b475", + ], + [ + "4799 cea9496d4270b5a1", + "4733 b9472094002e88fd", + "4685 d7691fb52ce63095", + "4623 2532b3477d8a5f2d", + "4624 856abd3827ed1af8", + ], + ]); + }); + it("where the kind of data changes within a block, for rows of 64 entries", () => { let x = 11; const next = () => { From 8d25bbb052faba11b307f1840af493e26b3a9045 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 22:03:31 +0000 Subject: [PATCH 186/199] The zstd side of the ICU hook is compiled for FreeBSD too Since #47316 bun_icu_decompress.cpp defines the hook on FreeBSD as well, whose prebuilt ICU now has its data compressed. Its other half, src/runtime/icu_decompress.rs, was compiled for Linux, Android and Windows, so bun did not link there: undefined symbol Bun__IcuDecompressor__create. --- src/runtime/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/runtime/lib.rs b/src/runtime/lib.rs index b83aff90a756..75dee05f0781 100644 --- a/src/runtime/lib.rs +++ b/src/runtime/lib.rs @@ -55,7 +55,12 @@ pub(crate) mod checker_queries_testing; pub(crate) mod code_resolver_testing; pub(crate) mod dispatch; pub(crate) mod hw_exports; -#[cfg(any(target_os = "linux", target_os = "android", windows))] +#[cfg(any( + target_os = "linux", + target_os = "android", + target_os = "freebsd", + windows +))] pub(crate) mod icu_decompress; #[cfg(debug_assertions)] pub(crate) mod imported_class_test; From 21acd17dfeed66d86c03eba9d91e52c5f0fd2fc4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 22:03:31 +0000 Subject: [PATCH 187/199] zstd: the contexts that are kept are not in place `Contexts` held a `FrameCompressor` and a `DCtx` in place, 1,544 bytes, and src/sema/program/retire.rs has one in a thread-local variable. Every thread pays for those: test/aot/static-tls-size.test.ts allows 23,552 bytes and found 24,544. They are boxed now, so it is two pointers, as it is on main. In a debug build the block shrinks by 3,064 bytes. `decompress_into` takes what `ZSTD_decompressDCtx` takes, which is more than one frame, and says so. --- src/zstd/lib.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index 0ea237dc0513..2d75cf46a136 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -148,8 +148,9 @@ fn compress_append_with( /// The contexts that [`compress_append`] and [`decompress_alloc`] make in every call, kept from one call to the next. /// Making one costs more than a few kilobytes of data do. pub struct Contexts { - compression: FrameCompressor, - decompression: DCtx, + // Not in place: thread-local variables hold these, and every thread pays for their size. + compression: Box, + decompression: Box, } impl Default for Contexts { @@ -161,8 +162,8 @@ impl Default for Contexts { impl Contexts { pub fn new() -> Self { Self { - compression: FrameCompressor::new(), - decompression: DCtx::new(), + compression: Box::new(FrameCompressor::new()), + decompression: Box::new(DCtx::new()), } } @@ -171,7 +172,7 @@ impl Contexts { compress_append_with(&mut self.compression, out, src, level) } - /// Decompresses one frame, which fills `dest`. + /// Decompresses `src`, whole frames that fill `dest`. pub fn decompress_into(&mut self, dest: &mut [u8], src: &[u8]) -> Result<(), ZstdError> { let size = (self.decompression.decompress(dest, src)) .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed))?; From 306afc849aba1b6944e82072baf88897fc2a374d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 22:41:34 +0000 Subject: [PATCH 188/199] zstd: the optimal parser passes what has no match faster with a dictionary that is searched in place too A dictionary is searched in place for inputs of up to 32 KB at levels 13 to 15 and of up to 8 KB from level 16 on, or of unknown size. On data with few matches these levels took 1.15 to 1.36 times the time of C there. Without a dictionary, and with one whose tables are copied, the parser has a loop for positions that come after literals and have no match (`skip_matchless`), which does without most of what the parser does for a position. With a dictionary to search in place it returned at once. Now there is `skip_matchless_in_place`, the same loop with the dictionary's tree searched after the input's. It hints at the root of that tree before it is known whether there is one (`root_at_once`): where there are few matches that is as good as random, and with `root` these rows take 1 to 3% longer. It also hints at the slot of the dictionary's table for the next position. Time relative to before, noise, base64 and skewed bytes, with a dictionary of 64 KB: inputs of 1 and 4 KB 0.64 to 0.69 at level 13 and 0.66 to 0.76 at levels 15 to 19; of 16 and 32 KB, which are searched in place up to level 15, 0.67 to 0.84, but skewed bytes of 32 KB at level 13 only 0.96. A dictionary of 30 KB: the same. Relative to C these rows were at 1.15 to 1.36 and are at 0.78 to 1.03. Still behind: base64 of 16 and 32 KB at level 13 (1.01 to 1.14) and skewed bytes of 32 KB there, 1.47, where most positions have a match. Of the 286 rows with such a dictionary (11 kinds, dictionaries of 30 and 64 KB, inputs of 1 to 32 KB, levels 13 to 19), 111 were more than 3% behind C, and 61 are. The mean relative to C goes from 0.979 to 0.912. The 8 other kinds, by level: 0.9985, 1.0000, 1.0000, 1.0001, 0.9995 of before. How this was measured is new. Two builds of one source in which the linker puts the functions in another order (`--shuffle-sections`), each function at a multiple of 64 bytes in both, differ by 0.5 to 2% in most rows, by 5 to 10% for records of 1 KB at level 13 and by up to 35% at level 1. One build is one draw. So every number here is the mean of 16 orders of the code before and of 16 of the code after (8 and 8 for what was not done), and a difference counts by how many standard errors it is. That tells apart what single builds could not, and it is why the change has the shape that it has: - One parser that chooses between the two loops is 5.5 KB of code and not 19.7. But of 150 rows without a dictionary at levels 13, 16 and 19, 65 take 0.4 to 1.4% longer, at 4 to 19 standard errors. So the parsers for such a dictionary are copies that differ in one call, and `compress_block_opt` chooses. - `skip_matchless::`: records of 100 bytes take 2.4% longer at levels 16 and 19 with such a dictionary, at 10 to 27 standard errors. An instance of a generic function is compiled in another unit, so `search_dms` is called from outside its own, and comes out different. With a `bool` for an inlined body and two plain functions around it, it is the same instructions as before. - The same for `get_all_matches` (the hints of `root_at_once` and at the next slot): skewed bytes of 32 KB 2.7% faster at level 13, most other rows 1 to 3% slower. Not done. - The hints of `root` and `root_at_once` in one function that both inline: `btlazy2`, which calls `root`, comes out different, and noise of 128 KB takes 1.4% longer at level 11. So they are written twice. `compress_block_opt0` was inlined into `compress_block`, so that whatever changes there changes the parser of levels 13 to 15. It no longer is. As a function of its own it is other instructions than before. 250 rows without a dictionary at levels 13 to 16 and 19 (8 orders each): means of 0.999 to 1.000 by level. DNA and half noise of 128 KB take 0.3% longer at level 15, at 7 standard errors, hex digits of 128 KB 0.6 to 0.8% less at level 13. With a copied dictionary, records of 100 bytes of 256 KB take 0.7% longer at level 13. Nothing else changes. `compress_block` is what it was without that parser (levels 1, 5 and 9 on 1 KB: 0.999 to 1.000), the two copies differ from their originals in the one call, and the other 377 functions of the crate are the same instructions as before, `skip_matchless`, `get_all_matches`, `search_dms` and `compress_block_opt2` among them. It is 19.7 KB more code. The output does not change. The test in `zstd.test.ts` compares with what Bun 1.4.4 writes, as all of them there, so it passes before this change too. Of 22 faults put into the new code, runs against the C library with random parameters see 20, and 19 of these change one of the 15 digests of the test (not: `next_to_update` set to the next position after a match). The two that nothing sees: a root at or below the lowest index of the dictionary, where the way down ends by itself, and one position more at the end of a block. Not done, and next: the loop does not start while one of the three repeat offsets leads into the dictionary, which is most of the time after a match there. --- src/zstd/compress/internal.rs | 15 +++++ src/zstd/compress/opt.rs | 104 +++++++++++++++++++++++++++++++--- test/js/bun/util/zstd.test.ts | 78 ++++++++++++++++++------- 3 files changed, 169 insertions(+), 28 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 546c677115db..4c5fe56c1f13 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1542,6 +1542,21 @@ impl<'a> DictTree<'a> { Ok(root) } + /// [`Self::root`] with the hint before it is known whether there is a root, which is as good as random where there + /// are few matches. + #[inline(always)] + pub(crate) fn root_at_once(dms: DictMatchState<'_>, hash: usize) -> Result { + let root = dms.ms.hash_table.at(hash)?; + let (bt, _) = dms.ms.chain_table.as_chunks(); + bt_prefetch(bt, root); + prefetch_read( + dms.buf.as_ptr().wrapping_add(dms.pos_of(root)), + Locality::L1, + ); + let is_root = root > dms.ms.window.low_limit; + Ok(if is_root { root } else { 0 }) + } + /// [`Near::compare`] for a match in the dictionary, for what [`Near::compare_chunks_with`] leaves. A match that /// reaches the end of the dictionary goes on at the start of the current segment. #[cold] diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index 635a606b739b..5bb8d66593b7 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -1,6 +1,6 @@ //! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. -use core::hint::cold_path; +use core::hint::{Locality, cold_path, prefetch_read}; use core::mem; use super::hist; @@ -881,28 +881,65 @@ impl<'a> Trees<'a> { /// it stops, and how many matches it has found there. #[inline(never)] fn skip_matchless( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_generic(matches, ip, limit, rep, false) + } + + /// [`Self::skip_matchless`] where a dictionary is searched in place. + #[inline(never)] + fn skip_matchless_in_place( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_generic(matches, ip, limit, rep, true) + } + + /// `in_place` is not a const parameter: an instance of a generic function is compiled apart from the rest of this + /// file, after which [`Self::search_dms`] takes 2% longer on short records. + #[inline(always)] + fn skip_matchless_generic( &mut self, matches: &mut [Match], mut ip: u32, limit: u32, rep: &[u32; ZSTD_REP_NUM], + in_place: bool, ) -> Result<(u32, u32)> { let near = self.input.near; let dict_limit = self.input.dict_limit; let min_match = self.min_match; // No repeat offset leads before the current segment, from `ip` on. if ip.wrapping_add(dict_limit) != self.next_to_update - || self.dms.is_some() + || (!in_place && self.dms.is_some()) || rep.iter().any(|rep_offset| *rep_offset > ip) { return Ok((ip, 0)); } + let dict = if in_place { self.dms } else { None }; + let dict = dict.map(|dms| (dms, self.hasher.with_log(dms.ms.c_params.hash_log))); while ip < limit && let Some(bytes) = near.read64(ip) { let curr = ip.wrapping_add(dict_limit); - let match_low = self.lowest_match_index(curr).max(1); + let mut dict_root = 0; + if let Some((dms, hasher)) = dict { + if let Some(next) = near.read64(ip + 1) { + let slot = dms.ms.hash_table.as_ptr().wrapping_add(hasher.hash(next)); + prefetch_read(slot, Locality::L1); + } + dict_root = DictTree::root_at_once(dms, hasher.hash(bytes))?; + } + let window_low = self.lowest_match_index(curr); + let match_low = window_low.max(1); let starts_alike = |pos: u32, min_match: u32| { let there = near.read32(pos); there.is_none_or(|there| minmatch_differs(there, bytes as u32, min_match) == 0) @@ -921,7 +958,20 @@ impl<'a> Trees<'a> { } let mut found = Found::new(0, min_match - 1, curr); - self.tree_matches(matches, ip, bytes, match_low, &mut found)?; + let nb_compares = self.tree_matches(matches, ip, bytes, match_low, &mut found)?; + if dict_root != 0 && nb_compares != 0 { + // A copy, so that `found` need not be in memory: 1 to 2% where there are no matches. + let mut with_dict = found; + self.search_dms( + matches, + ip, + dict_root, + window_low, + nb_compares, + &mut with_dict, + )?; + found = with_dict; + } if found.mnum != 0 { self.next_to_update = found.match_end_idx.wrapping_sub(8); return Ok((ip, found.mnum)); @@ -1255,7 +1305,7 @@ fn try_lengths( } } -/// `ZSTD_compressBlock_opt_generic` +/// `ZSTD_compressBlock_opt_generic`. `in_place` is whether there may be a dictionary to search in place. #[inline(always)] fn compress_block_opt_generic( ms: &mut MatchState, @@ -1264,6 +1314,7 @@ fn compress_block_opt_generic( src: &BlockSrc<'_>, opt_level: i32, dict_mode: DictMode, + in_place: bool, ) -> Result { // The match finders borrow the rest of `ms`. let mut opt_state = mem::replace(&mut ms.opt, OptState::new()); @@ -1279,6 +1330,7 @@ fn compress_block_opt_generic( src, opt_level, dict_mode, + in_place, ); opt_state.match_table = match_table; opt_state.price_table = price_table; @@ -1298,6 +1350,7 @@ fn parse_block( src: &BlockSrc<'_>, opt_level: i32, dict_mode: DictMode, + in_place: bool, ) -> Result { let buf = src.buf; let istart = src.start; @@ -1353,7 +1406,11 @@ fn parse_block( if ip != anchor && opt_ldm.is_over() { let first = (ip - prefix_start) as u32; let limit = (ilimit - prefix_start) as u32; - let (stop, found) = trees.skip_matchless(matches, first, limit, rep)?; + let (stop, found) = if in_place { + trees.skip_matchless_in_place(matches, first, limit, rep)? + } else { + trees.skip_matchless(matches, first, limit, rep)? + }; ip += (stop - first) as usize; if ip >= ilimit { break 'series; @@ -1652,6 +1709,7 @@ fn parse_block( } /// `ZSTD_compressBlock_opt0` +#[inline(never)] fn compress_block_opt0( ms: &mut MatchState, seq_store: &mut SeqStore, @@ -1659,7 +1717,20 @@ fn compress_block_opt0( src: &BlockSrc<'_>, dict_mode: DictMode, ) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode) + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, false) +} + +/// [`compress_block_opt0`] with [`Trees::skip_matchless_in_place`]. A copy: one parser that chooses between the two takes +/// 0.4 to 1.4% longer on most data without a dictionary. +#[inline(never)] +fn compress_block_opt0_in_place( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, true) } /// `ZSTD_compressBlock_opt2` @@ -1670,7 +1741,19 @@ fn compress_block_opt2( src: &BlockSrc<'_>, dict_mode: DictMode, ) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode) + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode, false) +} + +/// [`compress_block_opt2`], likewise. +#[inline(never)] +fn compress_block_opt2_in_place( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode, true) } /// `ZSTD_initStats_ultra`: a first pass over the block, only to have better statistics to start from. It has to be the @@ -1735,11 +1818,16 @@ pub(crate) fn compress_block_opt( strategy: Strategy, dict_mode: DictMode, ) -> Result { + let in_place = src.dms.is_some(); match (strategy, dict_mode) { (Strategy::BtUltra2, DictMode::NoDict) => compress_block_btultra2(ms, seq_store, rep, src), + (Strategy::BtUltra | Strategy::BtUltra2, _) if in_place => { + compress_block_opt2_in_place(ms, seq_store, rep, src, dict_mode) + } (Strategy::BtUltra | Strategy::BtUltra2, _) => { compress_block_opt2(ms, seq_store, rep, src, dict_mode) } + _ if in_place => compress_block_opt0_in_place(ms, seq_store, rep, src, dict_mode), _ => compress_block_opt0(ms, seq_store, rep, src, dict_mode), } } diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index efca75d00064..13758bd0e96b 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1606,27 +1606,32 @@ describe("the output is that of zstd 1.5.7", () => { `); }); - it("where the rows of a dictionary have candidates and those of the input have none", () => { - // Noise of so many bits. What follows the dictionary has pieces of it, and of itself, among the noise. - const withPieces = (bits: number) => { - let x = 5; - const next = () => { - x ^= x << 13; - x ^= x >>> 17; - x ^= x << 5; - return (x >>>= 0); - }; - const all = Buffer.alloc(42_000); - for (let at = 0; at < all.length;) { - for (let n = 4 + (next() % 24); n > 0; n--) all[at++] = next() >>> (32 - bits); - if (at < 30_000) continue; - const from = next() % 4 ? next() % 29_000 : 30_000 + (next() % (at - 30_000 + 1)); - const length = 4 + (next() % 12); - all.copyWithin(at, from, from + length); - at += length; - } - return all; + /** + * Noise of so many bits. What follows the dictionary, which is the first 30,000 bytes, has pieces of it and of itself + * among the noise, three of the one for one of the other. + */ + const withPieces = (bits: number, shortest = 4, longest = 15, mostlyOfTheDictionary = true) => { + let x = 5; + const next = () => { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + return (x >>>= 0); }; + const all = Buffer.alloc(42_000); + for (let at = 0; at < all.length;) { + for (let n = 4 + (next() % 24); n > 0; n--) all[at++] = next() >>> (32 - bits); + if (at < 30_000) continue; + const ofTheDictionary = next() % 4 !== 0 ? mostlyOfTheDictionary : !mostlyOfTheDictionary; + const from = ofTheDictionary ? next() % 29_000 : 30_000 + (next() % (at - 30_000 + 1)); + const length = shortest + (next() % (longest - shortest + 1)); + all.copyWithin(at, from, from + length); + at += length; + } + return all; + }; + + it("where the rows of a dictionary have candidates and those of the input have none", () => { // Rows of 16 entries up to level 8, then of 32 and of 64. const digests = (all: Buffer) => [5, 6, 7, 9, 10].map(n => @@ -1650,6 +1655,39 @@ describe("the output is that of zstd 1.5.7", () => { ]); }); + it("where there are literals before a match in the trees of a dictionary that is searched in place", () => { + // Few trees and few steps down them, so that it counts how many are left for the dictionary. + const crowded = (n: number) => ({ ...level(n), [c.ZSTD_c_hashLog]: 10, [c.ZSTD_c_searchLog]: 2 }); + // From level 16 on, the tables of the dictionary are copied for more than 8 KB. + const digests = (all: Buffer) => + [level(13), level(16), level(19), crowded(13), crowded(16)].map(params => + digest(zlib.zstdCompressSync(all.subarray(30_000, 38_000), { params, dictionary: all.subarray(0, 30_000) })), + ); + expect([withPieces(6, 3, 6, false), withPieces(5, 4, 15, false), withPieces(5, 3, 6, false)].map(digests)).toEqual([ + [ + "5719 406ca55e1f7a8e51", + "5723 638b7b1d34f5e7e4", + "5723 638b7b1d34f5e7e4", + "5775 e3985bd1b7f3cc06", + "5777 f4d41a4a3fe9753d", + ], + [ + "4036 957012d8fbbad463", + "4032 4a494ca0f71ad17d", + "4032 4a494ca0f71ad17d", + "4216 9e7fdaed4122c373", + "4205 a05cfe9758c76e6c", + ], + [ + "4895 2d4708a58b44a327", + "4897 1f4aad7c4526b8fd", + "4897 1f4aad7c4526b8fd", + "4919 5671cafb2cb68402", + "4917 92674bc1d6487eb", + ], + ]); + }); + it("where the kind of data changes within a block, for rows of 64 entries", () => { let x = 11; const next = () => { From 285d751112d4df054c02b8d19b8e8cc8e0f4f434 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 01:54:45 +0000 Subject: [PATCH 189/199] test: whether zstd's sources are fetched does not depend on the cache of the machine `zstd.enabled` is true for a prebuilt WebKit whose tarball has the objects of `jsc` and `testaot`, because `testaot` then compiles zstd's decoder. The case that expects false looked at the shared build cache, so it failed on a machine that has fetched such a tarball. It has a cache of its own now, which is empty, and the test that puts these objects in a cache expects true. --- test/internal/build-local-webkit.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/internal/build-local-webkit.test.ts b/test/internal/build-local-webkit.test.ts index a2df0542295b..dd7ad1112f04 100644 --- a/test/internal/build-local-webkit.test.ts +++ b/test/internal/build-local-webkit.test.ts @@ -129,6 +129,7 @@ describe("which builds compile WebKit", () => { name === "testaot" ? ["zstd"] : undefined, ]), ); + expect(zstd.enabled!(cfg)).toBe(true); expect(webkit.provides(cfg).files).toEqual(expect.arrayContaining([objects("jsc"), objects("testaot")])); expect([webkitJSCPath(cfg), webkitTestAotPath(cfg)]).toEqual([join(buildDir, "jsc"), join(buildDir, "testaot")]); }); @@ -768,7 +769,8 @@ describe.skipIf(Bun.which(python) === null)("the graph of a checkout", () => { try { process.env.BUN_ICU_ROOT = join("/somewhere", "icu"); expect(zstd.enabled!(cfg)).toBe(true); - expect(zstd.enabled!(linux({ buildDir }))).toBe(false); + // A prebuilt WebKit that is not fetched yet, whatever the cache of this machine has. + expect(zstd.enabled!(linux({ buildDir, cacheDir: join(buildDir, "cache") }))).toBe(false); delete process.env.BUN_ICU_ROOT; expect(zstd.enabled!(cfg)).toBe(false); } finally { From 666b766027cdd00cb9e1d8a80ae67042db5780ab Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 02:41:51 +0000 Subject: [PATCH 190/199] zstd: the optimal parser's loop for what has no match goes on while a repeat offset leads into the dictionary With a dictionary that is searched in place, `skip_matchless_in_place` returned at once when one of the three repeat offsets led before the input. After a match in the dictionary that is so until three other offsets have come, or until the position is past the offset: 86% of the calls, in runs with random parameters. Now there is `skip_matchless_far_reps`, the same loop with such an offset looked up in the dictionary. It runs up to the position from which all three offsets lead into the input, where the loop that was there takes over. What it looks at is `far_rep`, the part of `rep_len` that finds where the offset leads and compares the first 3 or 4 bytes. It is inlined in the loop: with a call of `rep_len`, which also counts, noise with pieces of the dictionary takes 19 to 28% longer at level 13. Before the loop, `skip_matchless_in_place` looks at the last offset alone. In a copy of the dictionary in which single bytes differ, the position after such a byte goes on at that offset, and all that the loop does there is done again by `get_all_matches`: without this look, such copies take 3 to 8% longer than before at level 13. Time relative to before, means of 16 orders of the functions in the binary for each, a dictionary of 64 KB. At level 13, inputs of 1, 4, 16 and 32 KB: - skewed bytes 0.96, 0.91, 0.84, 0.57; the last was at 1.45 of C and is at 0.83 - base64 0.97, 0.93, 0.77, 0.72; the last was at 1.12 of C and is at 0.81 - noise 0.98, 0.96, 0.92, 0.86 These three kinds, wherever the dictionary is searched in place (54 rows, levels 13 to 19, a dictionary of 30 KB too): 0.57 to 1.00 of before. They were at 0.78 to 1.45 of C and are at 0.75 to 0.88. The review of a first form of this change measured what the 11 kinds of data so far do not have, so there are 11 more: - Noise of 6 or 8 bits with 16 bytes of the dictionary every 40 or 200 bytes (48 rows): 0.53 to 0.79 of before. They were at 1.25 to 1.68 of C, the worst rows there were, and are at 0.88 to 1.00. - Pieces of the dictionary with one byte altered, at 2 or 4 distances in turn, among noise of 5 bits (24 rows): 0.91 to 0.99. 1.14 to 1.29 of C before, 1.07 to 1.21 now. - The dictionary itself with one byte in 5 altered (12 rows): 0.92 to 0.99. - The same with one byte in 8, 16, 32 or 100 (48 rows): 0.99 to 1.017, mean 1.004. 15 of these rows take 0.5 to 1.7% longer, at 3 to 7 standard errors. That is the look at the last offset, which is for nothing there. The 8 other kinds of the first 11, by level: 1.0001, 1.0012, 1.0009, 1.0007, 1.0000. Four of their 208 rows are more than 3 standard errors slower: hex digits of 16 and 32 KB at level 15 (0.7 and 0.4%), 16 KB of records of 1 KB at level 13 and 1 KB of JSON at level 15 (0.3%). `rep_len` is other instructions than before, for all who call it. A dictionary whose tables are copied (66 rows of 64 and 256 KB, and the 44 of the new kinds): means of 0.999 to 1.001, no row of the 66 beyond 3 standard errors, two of the 44 (one byte in 8 altered, 32 KB, levels 16 and 19: 1.2%, where a second run over 8 of the orders has 0.993). No dictionary (75 rows of 16 KB): 1.001 to 1.002, none beyond 3. Nothing else changes: `skip_matchless_in_place` and `rep_len` are other instructions, `skip_matchless_far_reps` is new, and the other 383 functions of the crate are the same as before, `get_all_matches`, `skip_matchless`, `search_dms` and the parsers among them. It is 4.3 KB more code. The output does not change. What else was tried, 8 orders each: - `far_rep` for the candidates of `get_all_matches`, which calls `rep_len` for every offset that leads before the input: the function comes out different, and hex digits take 14 to 19% longer, text 5%. - The offsets before the root of the dictionary's tree in the new loop: copies with altered bytes 0 to 2% faster, noise 2 to 4% slower. - No loop after a single literal, and no look at the last offset: free where one byte is altered, but two in a row are common with one in 8 (2.4% at level 15), and noise with pieces every 40 bytes takes 3% longer. Both rules: the loop that was there comes out different, and noise of 1 KB gains nothing. - The look at the last offset in `skip_matchless_far_reps` (1 to 3% on the copies) or in a function of its own (the old loop 2% slower on noise). - `rep_len` as it was and a second function for the loop: no faster. - `far_rep` with a closure for what to do with the place: the closure is not inlined, and most rows with a dictionary take 1 to 5% longer. - One condition, `in_place != self.dms.is_some()`, for two at the start of the loop: noise and base64 of 1 KB 3 to 4.5% slower. - Whether the trees are up to date before the look at the last offset, which is the cheaper question: the old loop comes out different, noise, base64 and skewed bytes 1 to 3% slower, nothing faster. - The new loop for an older segment of the window as well, which is where a dictionary is whose tables are copied: noise with pieces of it, 16 KB at level 16, goes from 1.66 of C to 1.00. That changes `skip_matchless`, which all data without a dictionary goes through, so it is for a change of its own. The test in `zstd.test.ts` compares with what Bun 1.4.4 writes, as all of them there, so it passes before this change too. Of 24 faults put into the new code, runs against the C library with random parameters see 16, and each of these changes one of the 8 digests of the test. Five of the others only cost time (the new loop not entered, left one position early, or entered without the look at the last offset, and the like). Three are bounds that no offset gets to, which were in `rep_len` before. The generator of the test is one that 4 other places of that `describe` had a copy of. They share it now. --- src/zstd/compress/opt.rs | 116 ++++++++++++++++++++++++++-------- test/js/bun/util/zstd.test.ts | 72 +++++++++++++-------- 2 files changed, 132 insertions(+), 56 deletions(-) diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index 5bb8d66593b7..33ce395e4f5a 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -674,37 +674,57 @@ impl<'a> Trees<'a> { .count(ip + min_match, ip - rep_offset + min_match); return Ok(more + min_match); } + let Some((buf, pos, end)) = self.far_rep(self.dms, ip, rep_offset, window_low)? else { + return Ok(0); + }; + let after = pos + min_match as usize; + Ok(input.count_2segments(ip + min_match, buf, after, end)? + min_match) + } + + /// Where the repeat offset `rep_offset` of `ip` leads, which is before the current segment: a buffer, the position + /// in it, and that of the end of the segment. `None` unless what is there starts like `ip`. `dms` is `self.dms`, + /// of which a loop has a copy at hand. + #[inline(always)] + fn far_rep( + &self, + dms: Option>, + ip: u32, + rep_offset: u32, + window_low: u32, + ) -> Result> { + let input = self.input; let curr = ip.wrapping_add(input.dict_limit); let rep_index = curr.wrapping_sub(rep_offset); - let same_start = - |bytes: u32| Some(minmatch_differs(bytes, input.near.read32(ip)?, min_match) == 0); if !index_overlap_check(input.dict_limit, rep_index) { - return Ok(0); + return Ok(None); } - if let Some(dms) = self.dms { + let (buf, pos, end) = if let Some(dms) = dms { let dms_high_limit = dms.end.wrapping_sub(dms.base) as u32; let dms_index_delta = window_low.wrapping_sub(dms_high_limit); let dms_low = dms.ms.window.low_limit.wrapping_add(dms_index_delta); // `curr > rep_index >= dms_low` - if rep_offset.wrapping_sub(1) < curr.wrapping_sub(dms_low) { - let rep_match = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); - if same_start(read_le32(dms.buf, rep_match)?).ok_or(BUG)? { - let after = rep_match + min_match as usize; - let more = input.count_2segments(ip + min_match, dms.buf, after, dms.end)?; - return Ok(more + min_match); - } + if rep_offset.wrapping_sub(1) >= curr.wrapping_sub(dms_low) { + return Ok(None); } - } else if self.low_limit < input.dict_limit { + let pos = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); + (dms.buf, pos, dms.end) + } else { // `curr > rep_index >= window_low` - if rep_offset.wrapping_sub(1) < curr.wrapping_sub(window_low) - && same_start(read_le32(input.dict.buf, input.dict.pos_of(rep_index))?) - .ok_or(BUG)? + if self.low_limit >= input.dict_limit + || rep_offset.wrapping_sub(1) >= curr.wrapping_sub(window_low) { - let after = rep_index.wrapping_add(min_match); - return Ok(input.count_ext_dict(ip + min_match, after)? + min_match); + return Ok(None); } - } - Ok(0) + let dict = input.dict; + ( + dict.buf, + dict.pos_of(rep_index), + dict.pos_of(input.dict_limit), + ) + }; + let here = input.near.read32(ip).ok_or(BUG)?; + let is_alike = minmatch_differs(read_le32(buf, pos)?, here, self.min_match) == 0; + Ok(is_alike.then_some((buf, pos, end))) } /// `ZSTD_btGetAllMatches_internal`: adds the positions up to `ip`, a position of `near`, to the trees, and stores in @@ -887,23 +907,50 @@ impl<'a> Trees<'a> { limit: u32, rep: &[u32; ZSTD_REP_NUM], ) -> Result<(u32, u32)> { - self.skip_matchless_generic(matches, ip, limit, rep, false) + self.skip_matchless_generic(matches, ip, limit, rep, false, false) } /// [`Self::skip_matchless`] where a dictionary is searched in place. #[inline(never)] fn skip_matchless_in_place( + &mut self, + matches: &mut [Match], + mut ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + let [rep0, rep1, rep2] = *rep; + let farthest = rep0.max(rep1).max(rep2); + if ip < farthest { + // What was at the last offset goes on after a few bytes that differ: nothing to pass. + let window_low = self.lowest_match_index(ip.wrapping_add(self.input.dict_limit)); + if ip < rep0 && self.far_rep(self.dms, ip, rep0, window_low)?.is_some() { + return Ok((ip, 0)); + } + let found; + (ip, found) = self.skip_matchless_far_reps(matches, ip, limit.min(farthest), rep)?; + if ip < farthest { + return Ok((ip, found)); + } + } + self.skip_matchless_generic(matches, ip, limit, rep, true, false) + } + + /// [`Self::skip_matchless_in_place`] where a repeat offset leads into the dictionary, which after a match there is + /// most of the time. + #[inline(never)] + fn skip_matchless_far_reps( &mut self, matches: &mut [Match], ip: u32, limit: u32, rep: &[u32; ZSTD_REP_NUM], ) -> Result<(u32, u32)> { - self.skip_matchless_generic(matches, ip, limit, rep, true) + self.skip_matchless_generic(matches, ip, limit, rep, true, true) } - /// `in_place` is not a const parameter: an instance of a generic function is compiled apart from the rest of this - /// file, after which [`Self::search_dms`] takes 2% longer on short records. + /// `in_place` and `far_reps` are not const parameters: an instance of a generic function is compiled apart from the + /// rest of this file, after which [`Self::search_dms`] takes 2% longer on short records. #[inline(always)] fn skip_matchless_generic( &mut self, @@ -912,19 +959,20 @@ impl<'a> Trees<'a> { limit: u32, rep: &[u32; ZSTD_REP_NUM], in_place: bool, + far_reps: bool, ) -> Result<(u32, u32)> { let near = self.input.near; let dict_limit = self.input.dict_limit; let min_match = self.min_match; - // No repeat offset leads before the current segment, from `ip` on. + // Unless `far_reps`, no repeat offset leads before the current segment, from `ip` on. if ip.wrapping_add(dict_limit) != self.next_to_update || (!in_place && self.dms.is_some()) - || rep.iter().any(|rep_offset| *rep_offset > ip) + || (!far_reps && rep.iter().any(|rep_offset| *rep_offset > ip)) { return Ok((ip, 0)); } - let dict = if in_place { self.dms } else { None }; - let dict = dict.map(|dms| (dms, self.hasher.with_log(dms.ms.c_params.hash_log))); + let dms = if in_place { self.dms } else { None }; + let dict = dms.map(|dms| (dms, self.hasher.with_log(dms.ms.c_params.hash_log))); while ip < limit && let Some(bytes) = near.read64(ip) @@ -944,7 +992,19 @@ impl<'a> Trees<'a> { let there = near.read32(pos); there.is_none_or(|there| minmatch_differs(there, bytes as u32, min_match) == 0) }; - if (rep.iter()).any(|rep_offset| starts_alike(ip - rep_offset, min_match)) { + if far_reps { + let mut is_repeat = false; + for &rep_offset in rep { + is_repeat |= if rep_offset <= ip { + starts_alike(ip - rep_offset, min_match) + } else { + self.far_rep(dms, ip, rep_offset, window_low)?.is_some() + }; + } + if is_repeat { + break; + } + } else if (rep.iter()).any(|rep_offset| starts_alike(ip - rep_offset, min_match)) { break; } if min_match == 3 { diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 13758bd0e96b..7bbb32078b2a 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1362,15 +1362,16 @@ describe.concurrent("Zstandard HTTP compression", () => { // The compressor is a port of zstd 1.5.7, and gives the same bytes. The expected values are from the C library. describe("the output is that of zstd 1.5.7", () => { + const xorshift = (x: number) => () => { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + return (x >>>= 0); + }; + // Words, copies of earlier parts at all distances, noise and runs: something for every part of the compressor. function sample(size: number, seed: number) { - let state = seed >>> 0 || 1; - const next = () => { - state ^= state << 13; - state ^= state >>> 17; - state ^= state << 5; - return (state >>>= 0); - }; + const next = xorshift(seed >>> 0 || 1); const words = [ "function ", "return ", @@ -1611,13 +1612,7 @@ describe("the output is that of zstd 1.5.7", () => { * among the noise, three of the one for one of the other. */ const withPieces = (bits: number, shortest = 4, longest = 15, mostlyOfTheDictionary = true) => { - let x = 5; - const next = () => { - x ^= x << 13; - x ^= x >>> 17; - x ^= x << 5; - return (x >>>= 0); - }; + const next = xorshift(5); const all = Buffer.alloc(42_000); for (let at = 0; at < all.length;) { for (let n = 4 + (next() % 24); n > 0; n--) all[at++] = next() >>> (32 - bits); @@ -1688,14 +1683,41 @@ describe("the output is that of zstd 1.5.7", () => { ]); }); - it("where the kind of data changes within a block, for rows of 64 entries", () => { - let x = 11; - const next = () => { - x ^= x << 13; - x ^= x >>> 17; - x ^= x << 5; - return x >>> 0; + it("where there are literals while a repeat offset leads into a dictionary", () => { + // Noise of 5 bits. What follows the dictionary has pieces from so far back, in turn, with one byte of each altered. + const withAlteredPieces = (distances: number[]) => { + const next = xorshift(5); + const all = Buffer.alloc(39_000); + for (let at = 0; at < 30_000; at++) all[at] = next() >>> 27; + for (let at = 30_000, n = 0; at < all.length; n++) { + for (let k = 1 + (next() % 12); k > 0; k--) all[at++] = next() >>> 27; + const from = at - distances[n % distances.length]; + all.copyWithin(at, from, from + 16); + all[at + 4 + (next() % 8)] ^= 1 + (next() % 3); + at += 16; + } + return all; }; + const digests = (all: Buffer, size: number) => + [13, 16].flatMap(n => { + const compress = (from: number, to: number) => + zlib.zstdCompressSync(all.subarray(from, to), { params: level(n), dictionary: all.subarray(0, 30_000) }); + // The offsets that a frame starts with, 1, 4 and 8, lead into the dictionary for its first 8 bytes. + const starts = Array.from({ length: 8 }, (_, start) => compress(30_000 + start, 30_300 + start)); + return [digest(compress(30_000, 30_000 + size)), digest(Buffer.concat(starts))]; + }); + expect([ + digests(withAlteredPieces([25_000, 12_345, 50]), 8_000), + // A piece goes on over the end of the dictionary, whose tables level 16 copies for more than 8 KB. + digests(withAlteredPieces([20]), 9_000), + ]).toEqual([ + ["2519 24ce676ed4360f6b", "1093 e9f462ba5074daef", "2482 b0633ed5d69d3d65", "1061 d826405e9ce9c5d2"], + ["2689 ac0ddc2b63fa3cf0", "1044 9e83e945e3585cf6", "2593 665843657aba87c4", "1018 d57ef3f0c34b3189"], + ]); + }); + + it("where the kind of data changes within a block, for rows of 64 entries", () => { + const next = xorshift(11); // Many positions start like another one and go on otherwise. const fewSymbols = (size: number) => Uint8Array.from({ length: size }, () => 65 + (next() >>> 30)); const noise = (size: number) => Uint8Array.from({ length: size }, () => next() >>> 24); @@ -1744,13 +1766,7 @@ describe("the output is that of zstd 1.5.7", () => { it("where matches are far between, for the optimal parsers", () => { const data = Buffer.alloc(100_000); - let x = 9; - const next = () => { - x ^= x << 13; - x ^= x >>> 17; - x ^= x << 5; - return x >>> 0; - }; + const next = xorshift(9); const copy = (at: number, distance: number, length: number) => { const end = Math.min(at + length, data.length); while (at < end) data[at] = data[at++ - distance]; From 3e025692600d8073ee2410474fb3c1d166c7b145 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 02:52:17 +0000 Subject: [PATCH 191/199] zstd: an enum for where the repeat offsets lead, in place of a second bool `skip_matchless_generic(.., true, false)` does not say which is which, and mordant's `bare_bool_args` refuses it. `far_reps` is `Reps::Near` or `Reps::Far` now. All 386 functions of the crate are the same instructions as before. --- src/zstd/compress/opt.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index 33ce395e4f5a..f64a9e8672a0 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -483,6 +483,15 @@ impl<'a> Prices<'a> { // ── binary tree search ──────────────────────────────────────────────────── +/// Where the repeat offsets lead, for [`Trees::skip_matchless_generic`]. +#[derive(Clone, Copy)] +enum Reps { + /// Into the current segment, all three. + Near, + /// Before it, at least one of them. + Far, +} + /// What the match finders work on for the time of a block: the tables of a [`MatchState`], and what does not change /// of it. struct Trees<'a> { @@ -907,7 +916,7 @@ impl<'a> Trees<'a> { limit: u32, rep: &[u32; ZSTD_REP_NUM], ) -> Result<(u32, u32)> { - self.skip_matchless_generic(matches, ip, limit, rep, false, false) + self.skip_matchless_generic(matches, ip, limit, rep, false, Reps::Near) } /// [`Self::skip_matchless`] where a dictionary is searched in place. @@ -933,7 +942,7 @@ impl<'a> Trees<'a> { return Ok((ip, found)); } } - self.skip_matchless_generic(matches, ip, limit, rep, true, false) + self.skip_matchless_generic(matches, ip, limit, rep, true, Reps::Near) } /// [`Self::skip_matchless_in_place`] where a repeat offset leads into the dictionary, which after a match there is @@ -946,11 +955,11 @@ impl<'a> Trees<'a> { limit: u32, rep: &[u32; ZSTD_REP_NUM], ) -> Result<(u32, u32)> { - self.skip_matchless_generic(matches, ip, limit, rep, true, true) + self.skip_matchless_generic(matches, ip, limit, rep, true, Reps::Far) } - /// `in_place` and `far_reps` are not const parameters: an instance of a generic function is compiled apart from the - /// rest of this file, after which [`Self::search_dms`] takes 2% longer on short records. + /// `in_place` and `reps` are not const parameters: an instance of a generic function is compiled apart from the rest + /// of this file, after which [`Self::search_dms`] takes 2% longer on short records. #[inline(always)] fn skip_matchless_generic( &mut self, @@ -959,8 +968,9 @@ impl<'a> Trees<'a> { limit: u32, rep: &[u32; ZSTD_REP_NUM], in_place: bool, - far_reps: bool, + reps: Reps, ) -> Result<(u32, u32)> { + let far_reps = matches!(reps, Reps::Far); let near = self.input.near; let dict_limit = self.input.dict_limit; let min_match = self.min_match; From 3efec5739e74069ae433cfe642bed962e294d0a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 02:53:50 +0000 Subject: [PATCH 192/199] verify-baseline: the timeout of main for the emulated run 8f7a87e11 gave the emulated run of simd-baseline.test.ts 90 s per test, because its first JPEG test took 2 to 5.7 s under Intel SDE against a default of 5 s. main has 15 s for the same reason since #47070, which is enough. The file is main's again. --- scripts/verify-baseline.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/verify-baseline.ts b/scripts/verify-baseline.ts index 3f708de4ac07..39a14ec2a531 100644 --- a/scripts/verify-baseline.ts +++ b/scripts/verify-baseline.ts @@ -237,10 +237,8 @@ if (values["skip-emulation"]) { } else { // Phase 1: SIMD code path verification const simdTestPath = join(repoRoot, "test", "js", "bun", "jsc-stress", "fixtures", "simd-baseline.test.ts"); - // The default per-test timeout (5 s) assumes native speed. Intel SDE runs this file hundreds of - // times slower, so the emulated run gets a per-test timeout of its own. - const emulatedTestTimeoutMs = 90_000; - await runTest("SIMD baseline tests", ["test", `--timeout=${emulatedTestTimeoutMs}`, simdTestPath], { live: true }); + // Under SDE code is slow the first time it runs: the first test to use a codec took 3 to 6 s. The default is 5 s. + await runTest("SIMD baseline tests", ["test", "--timeout=15000", simdTestPath], { live: true }); const aotDir = mkdtempSync(join(tmpdir(), "verify-baseline-aot-")); const aotProgram = join(aotDir, "baseline-cpu.ts"); From 55c964301f2e48d57e82d52cb5090d4cece3a443 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 06:04:42 +0000 Subject: [PATCH 193/199] zstd: the same loop while a repeat offset leads into an older segment of the window A dictionary whose tables are copied (inputs of more than 32 KB at levels 13 to 15, of more than 8 KB from level 16 on) is an older segment of the window, as is what a stream has before the place where its buffer starts again, and what a job of a worker has of the job before. After a match there, `skip_matchless` returned at once, as `skip_matchless_in_place` did until 666b76602. On noise with pieces of the dictionary these levels took 1.25 to 1.71 times the time of C. Now there is `skip_matchless_older`: the look at the last offset, the loop of 666b76602 with the offsets looked up in the older segment (`skip_matchless_far_reps_older`), and then the loop for offsets that are near. What the two functions for before the current segment share is `skip_matchless_far_then_near`. `skip_matchless_far_reps` is `skip_matchless_far_reps_in_place` now. Where a match can be 3 bytes, the loop also left at every candidate from the table for 3 bytes that is in the older segment, without a look at it. In a stream that is longer than its window, that is how nearly all calls ended: of 545,000 on 3 MB of noise at level 19, 537,000, and 4,200 of those candidates were a match. `older_starts_alike` compares them now. That is in the two loops for an older segment, of which the one for near offsets is `skip_matchless_near_reps_older`. With a longer minimum that loop would be `skip_matchless` again, so it is `skip_matchless` that is called. The parsers that call all this are copies again, `compress_block_opt0_older` and `compress_block_opt2_older`, which `compress_block_opt` takes for `DictMode::ExtDict`. What there is before the current segment is an enum now, `Before`, where the parsers and the loop had a bool. Time relative to before, means of 16 orders of the functions in the binary for each. A dictionary of 64 KB whose tables are copied: - skewed bytes: 64 KB at level 13 0.66 (1.33 of C, now 0.88), 256 KB 0.79; 16 to 64 KB at levels 16 to 19 0.81 to 0.89 (1.01 to 1.08 of C, now 0.88 to 0.90) - noise: 64 and 256 KB at level 13 0.78 and 0.82; levels 16 to 19 0.96 to 1.002 - base64: 64 KB at level 13 0.79 (1.09 of C, now 0.86); levels 16 to 19 0.94 to 1.008 - noise with 16 bytes of the dictionary every 40 or 200 bytes (12 rows): 0.60 to 0.86. They were at 1.25 to 1.71 of C and are at 1.00 to 1.08. - pieces of the dictionary with a byte altered among noise (8 rows): 0.93 to 1.006; the dictionary with one byte in 5 altered: 0.93 to 0.98; with one in 8 to 100 (16 rows): 0.99 to 1.002 - the 8 other kinds, by level (13, 16, 17, 19): 0.9986, 1.0001, 1.0004, 1.0004 Of these 139 rows, 69 were more than 3% behind C and 55 are. Their mean relative to C goes from 1.040 to 0.983. Five take longer by more than 3 standard errors: hex digits of 32 KB at level 17 and of 16 KB at level 19 (0.6 and 0.7%), 64 KB of the pieces with an altered byte at level 16 (0.6%), 64 KB of records of 100 bytes at level 13 (0.5%), 16 KB of text at level 16 (0.2%). Streams were not timed so far: 1 MB in pieces of 64 KB, no size given, windows of 16, 64 and 256 KB, levels 16 to 19, 17 kinds of data (204 rows). By window: 0.92, 0.95, 0.985 of before. By level: 0.94, 0.95, 0.96, 0.96. By kind, mean and best row: skewed bytes 0.84 and 0.66, base64 0.86 and 0.69, noise 0.90 and 0.74, noise of 5 bits 0.94 and 0.65, noise with pieces from up to 12,000 bytes back 0.88 to 0.94 and 0.76, the same with a byte altered 0.98 and 0.91, hex digits 0.99 and 0.91, the 7 others 0.999 to 1.001. The mean relative to C goes from 0.84 to 0.80. These rows are less certain than the others. At level 13, which without a size is not an optimal parser and runs the same instructions as before, the 51 rows are at 0.97 to 1.04, one of them 3.5 standard errors slower (in the run for another form of this change, one by 7% at 4.6). At levels 16 to 19 four rows are more than 3 slower, by 0.6 to 3.6%. Each was run three more times with more rounds: 0.992 to 1.004. Jobs of workers were not timed. Without a dictionary nothing of the optimal parser changes: `skip_matchless`, `get_all_matches`, `compress_block_opt0` and `compress_block_opt2` are the same instructions, as are the functions for a dictionary that is searched in place. `compress_block`, which all levels go through, has 21 instructions more, for the choice of a parser. 175 rows without a dictionary (25 kinds, 1 KB at levels 1, 5, 9, 13 and 16, 16 KB at levels 13 and 16): means of 0.9991 to 1.0000 by level, no row slower by more than 3 standard errors. It is 22 KB more code. The output does not change. What else was tried, 8 or 16 orders each: - No copies of the parsers, `skip_matchless` itself with the new part in front: its loop comes out different, and without a dictionary noise, base64 and skewed bytes take 2 to 6% longer, all rows 1%. - The candidates of 3 bytes as before, and `skip_matchless` for all near offsets: what the review of this change found. In streams at level 18, noise and the like take 1.0 to 1.8% longer than before, at 3 to 5 standard errors, since every one of those calls goes through one function more. - A loop of its own for near offsets whatever the minimum, inlined in `skip_matchless_older`: noise in a stream takes 3.5 to 8% longer at level 16. Out of line: 64 KB of records of 1 KB 1.4% at level 13. - That loop for a minimum of 3 only, but inlined: 256 KB of noise with a copied dictionary 1.5% longer, at 12 standard errors. - `older_starts_alike` not inlined: the same rows 0.9 to 1.2%, and streams gain less at levels 18 and 19. - `index_overlap_check` for the same condition in it: the loop comes out with 697 instructions for 663. Not timed. - `skip_matchless_older` inlined in the two parsers: nothing faster, HTML, JSON and records 0.3 to 0.6% slower. - The parsers for a dictionary that is searched in place for an older segment too, which is one line and no more code (the review, one order): most rows up to 6% slower than this, two 11 to 12%, and noise 2 to 7% slower than before. The test in `zstd.test.ts` has one more input, 33,000 bytes, for which both of its levels copy the tables of the dictionary, and there is a second test: 63,000 bytes without a dictionary in windows of 8 and 16 KB. They compare with what Bun 1.4.4 writes, as all of them there, so they pass before this change too. Of 19 faults put into the new code, runs against the C library with random parameters see 11, and each of these changes a digest of the new inputs. Three of them, in `older_starts_alike`, are seen only by runs that are aimed at this: optimal parsers, windows of 1 to 256 KB, pieces from far back. The tree mostly finds the same match of 3 bytes. The other eight cannot change the output: six choose another of the loops or parsers, leave the loop where it could go on, or hand on a dictionary where there is none, and two are at the last 3 bytes of the older segment, which never get into the table for 3 bytes. --- src/zstd/compress/opt.rs | 175 +++++++++++++++++++++++++++++----- test/js/bun/util/zstd.test.ts | 76 +++++++++------ 2 files changed, 198 insertions(+), 53 deletions(-) diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index f64a9e8672a0..c99ca47c80b8 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -483,6 +483,16 @@ impl<'a> Prices<'a> { // ── binary tree search ──────────────────────────────────────────────────── +/// What a parser has to search before the current segment. +#[derive(Clone, Copy)] +enum Before { + Nothing, + /// Of the window. + OlderSegment, + /// One that is searched in place. + Dictionary, +} + /// Where the repeat offsets lead, for [`Trees::skip_matchless_generic`]. #[derive(Clone, Copy)] enum Reps { @@ -692,7 +702,7 @@ impl<'a> Trees<'a> { /// Where the repeat offset `rep_offset` of `ip` leads, which is before the current segment: a buffer, the position /// in it, and that of the end of the segment. `None` unless what is there starts like `ip`. `dms` is `self.dms`, - /// of which a loop has a copy at hand. + /// of which a loop has a copy at hand, or `None` where the caller knows that there is none. #[inline(always)] fn far_rep( &self, @@ -736,6 +746,18 @@ impl<'a> Trees<'a> { Ok(is_alike.then_some((buf, pos, end))) } + /// Whether what is at `index` of the older segment may start with the first 3 bytes of `bytes`. + #[inline(always)] + fn older_starts_alike(&self, index: u32, bytes: u32) -> Result { + let input = self.input; + // Not met: the last 8 positions of a segment do not get into the table. + if input.dict_limit.wrapping_sub(index) < 4 { + return Ok(true); + } + let there = read_le32(input.dict.buf, input.dict.pos_of(index))?; + Ok(minmatch_differs(there, bytes, 3) == 0) + } + /// `ZSTD_btGetAllMatches_internal`: adds the positions up to `ip`, a position of `near`, to the trees, and stores in /// `matches` the matches for it that are found on the way, each longer than the one before and all at least /// `min_match` long. `ll0` is whether no literals come before `ip`. Returns how many there are. @@ -916,49 +938,111 @@ impl<'a> Trees<'a> { limit: u32, rep: &[u32; ZSTD_REP_NUM], ) -> Result<(u32, u32)> { - self.skip_matchless_generic(matches, ip, limit, rep, false, Reps::Near) + self.skip_matchless_generic(matches, ip, limit, rep, Before::Nothing, Reps::Near) + } + + /// [`Self::skip_matchless`] where the window has an older segment. + #[inline(never)] + fn skip_matchless_older( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_far_then_near(matches, ip, limit, rep, Before::OlderSegment) } /// [`Self::skip_matchless`] where a dictionary is searched in place. #[inline(never)] fn skip_matchless_in_place( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_far_then_near(matches, ip, limit, rep, Before::Dictionary) + } + + /// [`Self::skip_matchless`] with repeat offsets that lead before the current segment, which after a match there is + /// most of the time: one loop up to where none does any more, then the other. + #[inline(always)] + fn skip_matchless_far_then_near( &mut self, matches: &mut [Match], mut ip: u32, limit: u32, rep: &[u32; ZSTD_REP_NUM], + before: Before, ) -> Result<(u32, u32)> { + let in_place = matches!(before, Before::Dictionary); let [rep0, rep1, rep2] = *rep; let farthest = rep0.max(rep1).max(rep2); if ip < farthest { // What was at the last offset goes on after a few bytes that differ: nothing to pass. let window_low = self.lowest_match_index(ip.wrapping_add(self.input.dict_limit)); - if ip < rep0 && self.far_rep(self.dms, ip, rep0, window_low)?.is_some() { + let dms = if in_place { self.dms } else { None }; + if ip < rep0 && self.far_rep(dms, ip, rep0, window_low)?.is_some() { return Ok((ip, 0)); } let found; - (ip, found) = self.skip_matchless_far_reps(matches, ip, limit.min(farthest), rep)?; + (ip, found) = if in_place { + self.skip_matchless_far_reps_in_place(matches, ip, limit.min(farthest), rep)? + } else { + self.skip_matchless_far_reps_older(matches, ip, limit.min(farthest), rep)? + }; if ip < farthest { return Ok((ip, found)); } } - self.skip_matchless_generic(matches, ip, limit, rep, true, Reps::Near) + if in_place { + self.skip_matchless_generic(matches, ip, limit, rep, before, Reps::Near) + } else if self.min_match == 3 { + self.skip_matchless_near_reps_older(matches, ip, limit, rep) + } else { + // The same loop but for the table of 3 bytes. A copy comes out 3 to 8% slower on noise. + self.skip_matchless(matches, ip, limit, rep) + } } - /// [`Self::skip_matchless_in_place`] where a repeat offset leads into the dictionary, which after a match there is - /// most of the time. + /// The first part of [`Self::skip_matchless_older`]. #[inline(never)] - fn skip_matchless_far_reps( + fn skip_matchless_far_reps_older( &mut self, matches: &mut [Match], ip: u32, limit: u32, rep: &[u32; ZSTD_REP_NUM], ) -> Result<(u32, u32)> { - self.skip_matchless_generic(matches, ip, limit, rep, true, Reps::Far) + self.skip_matchless_generic(matches, ip, limit, rep, Before::OlderSegment, Reps::Far) } - /// `in_place` and `reps` are not const parameters: an instance of a generic function is compiled apart from the rest + /// The second part of [`Self::skip_matchless_older`], where a match can be 3 bytes. + #[inline(never)] + fn skip_matchless_near_reps_older( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_generic(matches, ip, limit, rep, Before::OlderSegment, Reps::Near) + } + + /// The first part of [`Self::skip_matchless_in_place`]. + #[inline(never)] + fn skip_matchless_far_reps_in_place( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_generic(matches, ip, limit, rep, Before::Dictionary, Reps::Far) + } + + /// `before` and `reps` are not const parameters: an instance of a generic function is compiled apart from the rest /// of this file, after which [`Self::search_dms`] takes 2% longer on short records. #[inline(always)] fn skip_matchless_generic( @@ -967,9 +1051,10 @@ impl<'a> Trees<'a> { mut ip: u32, limit: u32, rep: &[u32; ZSTD_REP_NUM], - in_place: bool, + before: Before, reps: Reps, ) -> Result<(u32, u32)> { + let in_place = matches!(before, Before::Dictionary); let far_reps = matches!(reps, Reps::Far); let near = self.input.near; let dict_limit = self.input.dict_limit; @@ -1021,7 +1106,13 @@ impl<'a> Trees<'a> { let match_index3 = self.insert_and_find_first_index_hash3(curr, bytes as u32)?; if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) - && (match_index3 < dict_limit || starts_alike(match_index3 - dict_limit, 3)) + && if match_index3 >= dict_limit { + starts_alike(match_index3 - dict_limit, 3) + } else if matches!(before, Before::OlderSegment) { + self.older_starts_alike(match_index3, bytes as u32)? + } else { + true + } { break; } @@ -1375,7 +1466,7 @@ fn try_lengths( } } -/// `ZSTD_compressBlock_opt_generic`. `in_place` is whether there may be a dictionary to search in place. +/// `ZSTD_compressBlock_opt_generic`. `before` is what `dict_mode` says, as a constant. #[inline(always)] fn compress_block_opt_generic( ms: &mut MatchState, @@ -1384,7 +1475,7 @@ fn compress_block_opt_generic( src: &BlockSrc<'_>, opt_level: i32, dict_mode: DictMode, - in_place: bool, + before: Before, ) -> Result { // The match finders borrow the rest of `ms`. let mut opt_state = mem::replace(&mut ms.opt, OptState::new()); @@ -1400,7 +1491,7 @@ fn compress_block_opt_generic( src, opt_level, dict_mode, - in_place, + before, ); opt_state.match_table = match_table; opt_state.price_table = price_table; @@ -1420,7 +1511,7 @@ fn parse_block( src: &BlockSrc<'_>, opt_level: i32, dict_mode: DictMode, - in_place: bool, + before: Before, ) -> Result { let buf = src.buf; let istart = src.start; @@ -1476,10 +1567,14 @@ fn parse_block( if ip != anchor && opt_ldm.is_over() { let first = (ip - prefix_start) as u32; let limit = (ilimit - prefix_start) as u32; - let (stop, found) = if in_place { - trees.skip_matchless_in_place(matches, first, limit, rep)? - } else { - trees.skip_matchless(matches, first, limit, rep)? + let (stop, found) = match before { + Before::Nothing => trees.skip_matchless(matches, first, limit, rep)?, + Before::OlderSegment => { + trees.skip_matchless_older(matches, first, limit, rep)? + } + Before::Dictionary => { + trees.skip_matchless_in_place(matches, first, limit, rep)? + } }; ip += (stop - first) as usize; if ip >= ilimit { @@ -1787,11 +1882,11 @@ fn compress_block_opt0( src: &BlockSrc<'_>, dict_mode: DictMode, ) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, false) + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, Before::Nothing) } -/// [`compress_block_opt0`] with [`Trees::skip_matchless_in_place`]. A copy: one parser that chooses between the two takes -/// 0.4 to 1.4% longer on most data without a dictionary. +/// [`compress_block_opt0`] with [`Trees::skip_matchless_in_place`]. A copy: one parser that chooses takes 0.4 to 1.4% +/// longer on most data without a dictionary. #[inline(never)] fn compress_block_opt0_in_place( ms: &mut MatchState, @@ -1800,7 +1895,19 @@ fn compress_block_opt0_in_place( src: &BlockSrc<'_>, dict_mode: DictMode, ) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, true) + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, Before::Dictionary) +} + +/// [`compress_block_opt0`] with [`Trees::skip_matchless_older`], likewise. +#[inline(never)] +fn compress_block_opt0_older( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, Before::OlderSegment) } /// `ZSTD_compressBlock_opt2` @@ -1811,7 +1918,7 @@ fn compress_block_opt2( src: &BlockSrc<'_>, dict_mode: DictMode, ) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode, false) + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode, Before::Nothing) } /// [`compress_block_opt2`], likewise. @@ -1823,7 +1930,19 @@ fn compress_block_opt2_in_place( src: &BlockSrc<'_>, dict_mode: DictMode, ) -> Result { - compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode, true) + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode, Before::Dictionary) +} + +/// [`compress_block_opt2`], likewise. +#[inline(never)] +fn compress_block_opt2_older( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode, Before::OlderSegment) } /// `ZSTD_initStats_ultra`: a first pass over the block, only to have better statistics to start from. It has to be the @@ -1894,10 +2013,14 @@ pub(crate) fn compress_block_opt( (Strategy::BtUltra | Strategy::BtUltra2, _) if in_place => { compress_block_opt2_in_place(ms, seq_store, rep, src, dict_mode) } + (Strategy::BtUltra | Strategy::BtUltra2, DictMode::ExtDict) => { + compress_block_opt2_older(ms, seq_store, rep, src, dict_mode) + } (Strategy::BtUltra | Strategy::BtUltra2, _) => { compress_block_opt2(ms, seq_store, rep, src, dict_mode) } _ if in_place => compress_block_opt0_in_place(ms, seq_store, rep, src, dict_mode), + (_, DictMode::ExtDict) => compress_block_opt0_older(ms, seq_store, rep, src, dict_mode), _ => compress_block_opt0(ms, seq_store, rep, src, dict_mode), } } diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 7bbb32078b2a..adf6c8a28331 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1683,36 +1683,58 @@ describe("the output is that of zstd 1.5.7", () => { ]); }); + // Noise of 5 bits. After 30,000 bytes it has pieces from so far back, in turn, with one byte of each altered. + const withAlteredPieces = (distances: number[]) => { + const next = xorshift(5); + const all = Buffer.alloc(63_000); + for (let at = 0; at < 30_000; at++) all[at] = next() >>> 27; + for (let at = 30_000, n = 0; at < all.length; n++) { + for (let k = 1 + (next() % 12); k > 0; k--) all[at++] = next() >>> 27; + const from = at - distances[n % distances.length]; + all.copyWithin(at, from, from + 16); + all[at + 4 + (next() % 8)] ^= 1 + (next() % 3); + at += 16; + } + return all; + }; + it("where there are literals while a repeat offset leads into a dictionary", () => { - // Noise of 5 bits. What follows the dictionary has pieces from so far back, in turn, with one byte of each altered. - const withAlteredPieces = (distances: number[]) => { - const next = xorshift(5); - const all = Buffer.alloc(39_000); - for (let at = 0; at < 30_000; at++) all[at] = next() >>> 27; - for (let at = 30_000, n = 0; at < all.length; n++) { - for (let k = 1 + (next() % 12); k > 0; k--) all[at++] = next() >>> 27; - const from = at - distances[n % distances.length]; - all.copyWithin(at, from, from + 16); - all[at + 4 + (next() % 8)] ^= 1 + (next() % 3); - at += 16; - } - return all; - }; - const digests = (all: Buffer, size: number) => - [13, 16].flatMap(n => { - const compress = (from: number, to: number) => - zlib.zstdCompressSync(all.subarray(from, to), { params: level(n), dictionary: all.subarray(0, 30_000) }); - // The offsets that a frame starts with, 1, 4 and 8, lead into the dictionary for its first 8 bytes. - const starts = Array.from({ length: 8 }, (_, start) => compress(30_000 + start, 30_300 + start)); - return [digest(compress(30_000, 30_000 + size)), digest(Buffer.concat(starts))]; - }); + const compress = (all: Buffer, n: number, from: number, size: number) => + zlib.zstdCompressSync(all.subarray(from, from + size), { params: level(n), dictionary: all.subarray(0, 30_000) }); + const digests = (all: Buffer, size: number) => [13, 16].map(n => digest(compress(all, n, 30_000, size))); + const fromThreePlaces = withAlteredPieces([25_000, 12_345, 50]); + const fromNearby = withAlteredPieces([20]); expect([ - digests(withAlteredPieces([25_000, 12_345, 50]), 8_000), - // A piece goes on over the end of the dictionary, whose tables level 16 copies for more than 8 KB. - digests(withAlteredPieces([20]), 9_000), + digests(fromThreePlaces, 8_000), + // The tables of the dictionary are copied for more than 8 KB from level 16 on, else for more than 32 KB. + digests(fromThreePlaces, 33_000), + // A piece goes on over the end of the dictionary. + digests(fromNearby, 9_000), ]).toEqual([ - ["2519 24ce676ed4360f6b", "1093 e9f462ba5074daef", "2482 b0633ed5d69d3d65", "1061 d826405e9ce9c5d2"], - ["2689 ac0ddc2b63fa3cf0", "1044 9e83e945e3585cf6", "2593 665843657aba87c4", "1018 d57ef3f0c34b3189"], + ["2519 24ce676ed4360f6b", "2482 b0633ed5d69d3d65"], + ["10359 10c42f22fc137208", "10027 1fd5fb810e81ee70"], + ["2689 ac0ddc2b63fa3cf0", "2593 665843657aba87c4"], + ]); + // The offsets that a frame starts with, 1, 4 and 8, lead into the dictionary for its first 8 bytes. + const starts = (all: Buffer) => + [13, 16].map(n => + digest(Buffer.concat(Array.from({ length: 8 }, (_, start) => compress(all, n, 30_000 + start, 300)))), + ); + expect([starts(fromThreePlaces), starts(fromNearby)]).toEqual([ + ["1093 e9f462ba5074daef", "1061 d826405e9ce9c5d2"], + ["1044 9e83e945e3585cf6", "1018 d57ef3f0c34b3189"], + ]); + }); + + it("where there are literals while a repeat offset leads before the place where the buffer of a stream starts again", () => { + // Of the distances, one is just out of a window of 16 KB and one just in it. + const all = withAlteredPieces([16_450, 16_300, 9_000]); + const window = (n: number, windowLog: number) => ({ params: { ...level(n), [c.ZSTD_c_windowLog]: windowLog } }); + expect( + [13, 14].map(windowLog => [13, 16, 19].map(n => digest(zlib.zstdCompressSync(all, window(n, windowLog))))), + ).toEqual([ + ["38845 7721ae005eb10446", "38929 da0285f338ad0055", "38929 da0285f338ad0055"], + ["32494 857cd1a443168536", "32293 4e37f66382f2b3a3", "32293 4e37f66382f2b3a3"], ]); }); From de7ed9c8950af177822d5413acd2abccd75ea90b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 08:31:56 +0000 Subject: [PATCH 194/199] zstd: positions that are added to the binary trees are compared 8 bytes at a time At levels 13 to 19, HTML and records were what took longest relative to C: with a dictionary whose tables are copied 1.05, 1.07 and 1.29 times its time in the mean (HTML, records of 100 bytes and of 1 KB), up to 1.35. After a long match, all its positions are added to the trees, and that is where the time goes: 96% of it for 64 KB of records of 100 bytes at level 13 with such a dictionary, in `update_tree_in_part` and what it calls. A step down a tree takes as many instructions there as in `ZSTD_insertBt1`, 86. The comparison of the two positions takes 59 where C takes 30, and a step 31 cycles where C takes 23. The comparison took 16 bytes at once, as a vector. Now it takes them as two words of 8, each with a branch (`Common::within_words`): - in `insert_bt1`, which is what `update_tree_in_part` calls for each position. Where matches are looked for, `tree_matches`, it stays as it was. - in `TreeInput::compare_older`, for all who get to a match in the older segment. It also has two conditions where it had five: `TreeInput::new` cuts the buffer of the older segment where the segment ends. The field is private now, so that there is no other way to make one. What `Near::compare_chunks` does with the first 16 bytes is an enum now, `First`, where it was a bool, and `bt_walk` takes it where it had a bool for `Near::compare_head`, which goes with `First::Alike`. Time relative to before, means of 16 orders of the functions in the binary for each, 1,206 rows: 283 were more than 3% behind C, 218 are. 444 take less time by more than 3 standard errors, 289 by more than 1.5%. By kind, all rows of levels 13 to 19 (mean, best row, how many are more than 3% behind C): - HTML (61 rows) 0.92, 0.79, 36 before and 11 now - records of 1 KB (61) 0.93, 0.83, 43 and 31 - records of 100 bytes (61) 0.95, 0.82, 35 and 25 - JSON (61) 0.98, 0.95, 5 and none By what there is before the input (rows, how many behind C, mean relative to C): - a dictionary of 64 KB whose tables are copied, 11 kinds (99): 30 and 21, 0.970 and 0.926. HTML 0.89 of before (1.05 of C, now 0.93), records of 1 KB 0.87 (1.29, now 1.12), of 100 bytes 0.91 (1.07, now 0.97). - the same, pieces of the dictionary and copies with bytes altered (40): 24 and 22, 1.015 and 1.006 - a dictionary of 64 or 30 KB that is searched in place, 11 kinds (286): 51 and 37, 0.89 and 0.87; the pieces and copies (176): 75 and 72, 0.976 and 0.972 - nothing, 1 and 16 KB (175): 24 and 22, 0.928 and 0.922; 128 and 512 KB (175, not timed so far): 35 and 26, 0.914 and 0.902 - a stream that is longer than its window (255): 44 and 18, 0.820 and 0.801 Six rows take longer by more than 3 standard errors. Each was run three more times with more rounds. 128 KB at level 13 of words drawn from 60 at random: 0.9%, then 0.9 to 1.2% (it is at 0.91 of C). 128 KB of text at level 13 0.4%, then 0.2 to 0.4%. 64 KB of pieces of a copied dictionary with a byte altered at level 13 0.5%, then 0.3 to 0.5%. 4 KB of text at level 16 with a dictionary of 30 KB 0.3%, then 0.1 to 0.3%. Two of streams, 2.1 and 1.6%, then 0.986 to 1.018. `get_all_matches`, the `skip_matchless` functions and the six parsers are the same instructions. Other ones: `compare_rest`, which has `compare_older`, the three `update_tree_in_part`, `Trees::new`, and the six `compress_block_lazy_part` of the binary tree finder of levels 13 to 15, which have `TreeInput::new`. 512 KB at levels 13 and 15, where that finder is used: 0.998 in the mean, 0.967 to 1.017, none beyond 3 standard errors; streams at level 13 0.989. It is 223 bytes more code. The output does not change. What else was tried, 8 or 16 orders each: - The two words where matches are looked for as well: HTML and records as here, but 35 of 314 rows take 0.3 to 1.8% longer, at up to 18 standard errors: text, logs, JSON, floats, and the copies of a dictionary with bytes altered. There the first difference is often in the second word. - The same and `compare_older` as it was: with a copied dictionary, 41 of 139 rows take 0.5 to 1.6% longer, all but two of them from level 16 on. - The two words only after 16 bytes in common, else the vector: half the gain, and noise, base64 and skewed bytes take 1 to 5% longer, in streams up to 13%. - One branch for whether the 16 bytes differ, and the word chosen with `select_unpredictable`: HTML takes 2 to 3% longer than before. - The same with 128 bits (`u128`): no gain (one order). - `le64_at` for the words: all four are read before the first is compared, and `compare_rest` has 104 lines of assembly for 84. Not timed. - A loop over 16 bytes at a time in `compare_rest`: it has registers to save then, and records take 2.5% longer (one order). Of 12 faults put into the new code, runs against the C library with random parameters see 9. The unit tests see these 9, and a tenth that only costs time. 8 of the 9 change a digest of the tests in `zstd.test.ts` for literals while a repeat offset leads before the current segment. The ninth is the buffer of the older segment not being cut. The last two only choose the other comparison. `the_head_of_a_position_is_compared_as_all_of_it` and `a_match_that_leaves_the_older_segment_goes_on_in_the_current_one` go over the three values of `First`. All times are from x86-64. --- src/zstd/compress/internal.rs | 156 +++++++++++++++++++++++----------- src/zstd/compress/lazy.rs | 12 +-- src/zstd/compress/opt.rs | 27 +++--- 3 files changed, 128 insertions(+), 67 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 4c5fe56c1f13..eb45ba767ec2 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -743,6 +743,18 @@ pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { #[derive(Clone, Copy)] pub(crate) struct Near<'a>(&'a [u8]); +/// What [`Near::compare_chunks`] does with the first 16 bytes, where most positions differ. +#[derive(Clone, Copy, PartialEq, Eq, ConstParamTy)] +pub(crate) enum First { + /// The same as with the others. + Alike, + /// They have code of their own. + Apart, + /// That code takes them 8 at a time. On x86-64, HTML and records take 5 to 13% less time where positions are only + /// added to the trees. Where matches are looked for, text takes 1% longer. + Words, +} + /// A bit for each of the 16 bytes that is not the same in both. A comparison of vectors. #[inline(always)] fn differences(bytes: &[u8; 16], match_bytes: &[u8; 16]) -> u32 { @@ -902,7 +914,7 @@ impl<'a> Near<'a> { /// the order of what follows them. Only the sum of `match_pos` and `from` has to be a position. #[inline(always)] pub(crate) fn compare(self, pos: u32, match_pos: u32, from: u32) -> Common { - let mut len = match self.compare_chunks::(pos, match_pos, from) { + let mut len = match self.compare_chunks::<{ First::Alike }>(pos, match_pos, from) { Ok(common) => return common, Err(len) => len, }; @@ -918,20 +930,20 @@ impl<'a> Near<'a> { } /// [`Self::compare`], 16 bytes at a time as long as there are as many. Fails with how many bytes are known to be in - /// common when there are not. With `FIRST_APART`, the first 16 bytes, where most differ, have code of their own. + /// common when there are not. #[inline(always)] - pub(crate) fn compare_chunks( + pub(crate) fn compare_chunks( self, pos: u32, match_pos: u32, from: u32, ) -> core::result::Result { - self.compare_chunks_with::(pos, self, match_pos, from) + self.compare_chunks_with::(pos, self, match_pos, from) } /// [`Self::compare_chunks`], for a match that is at `match_pos` in `other`. #[inline(always)] - pub(crate) fn compare_chunks_with( + pub(crate) fn compare_chunks_with( self, pos: u32, other: Self, @@ -939,16 +951,22 @@ impl<'a> Near<'a> { from: u32, ) -> core::result::Result { let mut len = from; - if FIRST_APART + let is_apart = match FIRST { + First::Alike => false, + First::Apart | First::Words => true, + }; + if is_apart && let (Some(bytes), Some(match_bytes)) = ( self.chunk16(pos.wrapping_add(len)), other.chunk16(match_pos.wrapping_add(len)), ) { - let differ = differences(bytes, match_bytes); - if differ != 0 { - let len = len + differ.trailing_zeros(); - return Ok(Common::within(len, match_bytes, bytes)); + let common = match FIRST { + First::Words => Common::within_words(len, match_bytes, bytes), + First::Alike | First::Apart => Common::within_chunk(len, match_bytes, bytes), + }; + if let Some(common) = common { + return Ok(common); } len += 16; } @@ -959,7 +977,7 @@ impl<'a> Near<'a> { let differ = differences(bytes, match_bytes); if differ != 0 { let first = differ.trailing_zeros(); - if !FIRST_APART { + if !is_apart { return Ok(Common::within(len + first, match_bytes, bytes)); } // After 16 bytes that are the same, that takes longer than it saves. @@ -1113,6 +1131,33 @@ impl Common { } } + /// [`Self::within`] after `from` bytes in common, unless the bytes are the same. + #[inline(always)] + fn within_chunk(from: u32, match_bytes: &[u8; 16], bytes: &[u8; 16]) -> Option { + let differ = differences(bytes, match_bytes); + (differ != 0).then(|| Self::within(from + differ.trailing_zeros(), match_bytes, bytes)) + } + + /// [`Self::within_chunk`], 8 bytes at a time. + #[inline(always)] + fn within_words(from: u32, match_bytes: &[u8; 16], bytes: &[u8; 16]) -> Option { + let (bytes, match_bytes) = ( + u128::from_le_bytes(*bytes), + u128::from_le_bytes(*match_bytes), + ); + for (half, from) in [(0, from), (64, from + 8)] { + let (word, match_word) = ((bytes >> half) as u64, (match_bytes >> half) as u64); + let differ = word ^ match_word; + if differ != 0 { + return Some(Self { + len: from + (differ.trailing_zeros() >> 3), + match_is_smaller: Some(match_word.swap_bytes() < word.swap_bytes()), + }); + } + } + None + } + #[inline(always)] pub(crate) const fn up_to_the_end(len: u32) -> Self { Self { @@ -1252,8 +1297,8 @@ pub(crate) struct TreeInput<'a> { /// The current segment, up to where matches end. A position in it is an index minus `dict_limit`. pub(crate) near: Near<'a>, pub(crate) dict_limit: u32, - /// The older segment. - pub(crate) dict: Prefix<'a>, + /// The older segment, with which its buffer ends. + dict: Prefix<'a>, /// Whether there is enough of it for hints at what is about to be read to be worth giving. pub(crate) far: bool, } @@ -1267,6 +1312,11 @@ impl<'a> TreeInput<'a> { iend: usize, ) -> Result { let near = Near::new(prefix.buf.sub(prefix.pos_of(dict_limit)..iend)?)?; + let older = dict.buf.get(..dict.pos_of(dict_limit)); + let dict = Prefix { + buf: older.unwrap_or_default(), + base: dict.base, + }; Ok(Self { near, dict_limit, @@ -1275,6 +1325,12 @@ impl<'a> TreeInput<'a> { }) } + /// The older segment, with which its buffer ends. + #[inline(always)] + pub(crate) const fn dict(&self) -> Prefix<'a> { + self.dict + } + /// What `far` is for a current segment. #[inline(always)] pub(crate) const fn is_far(near: Near<'_>) -> bool { @@ -1320,7 +1376,8 @@ impl<'a> TreeInput<'a> { u64::from(match_index) + u64::from(len) >= u64::from(self.dict_limit) } - /// [`Near::compare_chunks`] for the next 16 bytes alone, if the match has them in the older segment. + /// [`Near::compare_chunks`] for the next 16 bytes alone, if the match has them in the older segment. In words, + /// whatever the walk does in the current segment: there is one copy of this, out of line. #[inline(always)] fn compare_older( &self, @@ -1328,23 +1385,14 @@ impl<'a> TreeInput<'a> { match_index: u32, from: u32, ) -> core::result::Result { - if self.is_current(match_index, from) { - return Err(from); - } - let older = self.dict.buf.get(..self.dict.pos_of(self.dict_limit)); - let match_pos = self.dict.pos_of(match_index + from); + let pos = self.dict.pos_of(match_index).wrapping_add(from as usize); let (Some(bytes), Some(match_bytes)) = ( self.near.chunk16(ip.wrapping_add(from)), - older.and_then(|older| older.get(match_pos..)?.first_chunk()), + (self.dict.buf.get(pos..pos.wrapping_add(16))).and_then(<[u8]>::first_chunk), ) else { return Err(from); }; - let differ = differences(bytes, match_bytes); - if differ == 0 { - return Err(from + 16); - } - let len = from + differ.trailing_zeros(); - Ok(Common::within(len, match_bytes, bytes)) + Common::within_words(from, match_bytes, bytes).ok_or(from + 16) } /// [`Near::compare`] wherever the match is, for what [`Near::compare_chunks`] leaves. Small, for a cheap call. @@ -1424,9 +1472,10 @@ pub(crate) enum Hints { /// Goes down the tree whose root is `root` for the position `ip` of `input.near`, which `descent` is for and which /// becomes the root. `visit` is called with each position that is met on the way, which are all at least `low` and at /// most `nb_compares`, and what it has in common with `ip`, and returns whether to go on. Returns how many more positions -/// could have been compared. `BY_HEAD` is whether to try [`Near::compare_head`], which is for ways down that are short. +/// could have been compared. With [`First::Alike`], [`Near::compare_head`] is tried before [`Near::compare_chunks`], which +/// is for ways down that are short. #[inline(always)] -pub(crate) fn bt_walk( +pub(crate) fn bt_walk( bt: &mut Nodes, input: &TreeInput<'_>, ip: u32, @@ -1441,7 +1490,11 @@ pub(crate) fn bt_walk( } let near = input.near; let dict_limit = input.dict_limit; - let head = near.chunk16(ip).filter(|_| BY_HEAD); + let by_head = match FIRST { + First::Alike => true, + First::Apart | First::Words => false, + }; + let head = near.chunk16(ip).filter(|_| by_head); let mut match_index = root; while nb_compares != 0 && match_index >= low { let children = bt_children(bt, match_index); @@ -1461,11 +1514,7 @@ pub(crate) fn bt_walk( let common = match head.and_then(|head| near.compare_head(head, match_pos, from)) { Some(common) => common, None => { - let by_chunks = if BY_HEAD { - near.compare_chunks::(ip, match_pos, from) - } else { - near.compare_chunks::(ip, match_pos, from) - }; + let by_chunks = near.compare_chunks::(ip, match_pos, from); match by_chunks { Ok(common) => common, Err(from) => input.compare_rest(ip, match_index, from)?, @@ -1609,9 +1658,12 @@ impl<'a> DictTree<'a> { self.content.prefetch(pos.wrapping_add(from)); } let match_pos = dict_match_index.wrapping_sub(self.low_limit); - let chunks = input - .near - .compare_chunks_with::(ip, self.content, match_pos, from); + let chunks = input.near.compare_chunks_with::<{ First::Alike }>( + ip, + self.content, + match_pos, + from, + ); let common = match chunks { Ok(common) => common, Err(from) => self.compare_rest(input, ip, dict_match_index, from)?, @@ -2732,8 +2784,9 @@ mod tests { let head = near.chunk16(POS).ok_or(BUG)?; for from in [0, len / 2, len] { for common in [ - near.compare_chunks::(POS, 0, from), - near.compare_chunks::(POS, 0, from), + near.compare_chunks::<{ First::Alike }>(POS, 0, from), + near.compare_chunks::<{ First::Apart }>(POS, 0, from), + near.compare_chunks::<{ First::Words }>(POS, 0, from), ] { assert_eq!( common.map(|common| (common.len, common.match_is_smaller)), @@ -2868,15 +2921,22 @@ mod tests { let match_index = DICT_LIMIT - in_older; for from in [0, len / 2, len] { let match_pos = match_index.wrapping_sub(DICT_LIMIT); - let common = match input.near.compare_chunks::(IP, match_pos, from) { - Ok(common) => common, - Err(from) => input.compare_rest(IP, match_index, from)?, - }; - assert_eq!( - (common.len, common.match_is_smaller), - (len, (IP + len < END).then_some(match_is_smaller)), - "{in_older} {len} {from}" - ); + let near = input.near; + for by_chunks in [ + near.compare_chunks::<{ First::Alike }>(IP, match_pos, from), + near.compare_chunks::<{ First::Apart }>(IP, match_pos, from), + near.compare_chunks::<{ First::Words }>(IP, match_pos, from), + ] { + let common = match by_chunks { + Ok(common) => common, + Err(from) => input.compare_rest(IP, match_index, from)?, + }; + assert_eq!( + (common.len, common.match_is_smaller), + (len, (IP + len < END).then_some(match_is_smaller)), + "{in_older} {len} {from}" + ); + } } } } diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 35dfce6f110a..397666643d08 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -5,9 +5,9 @@ use core::num::NonZeroU32; use core::ops::Range; use super::internal::{ - BlockSrc, CompressionParameters, Descent, DictMatchState, DictMode, DictTree, Hash8, Hasher, - Hints, K_SEARCH_STRENGTH, MatchState, Near, Nodes, Prefix, REPCODE1_TO_OFFBASE, RowTags, - SecondTagStats, SeqSink, SeqStore, Strategy, TreeInput, ZSTD_DUBT_UNSORTED_MARK, + BlockSrc, CompressionParameters, Descent, DictMatchState, DictMode, DictTree, First, Hash8, + Hasher, Hints, K_SEARCH_STRENGTH, MatchState, Near, Nodes, Prefix, REPCODE1_TO_OFFBASE, + RowTags, SecondTagStats, SeqSink, SeqStore, Strategy, TreeInput, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, bt_children, bt_look_ahead, bt_low, bt_nodes, bt_set_children, bt_walk, count_2segments, hash_ptr, hashed_swap, index_overlap_check, offbase_is_offset, offbase_to_offset, offset_to_offbase, table_get, table_put, @@ -219,7 +219,7 @@ impl<'a> Dubt<'a> { let mut search_dict = true; match_index = table_get(self.hash_table, h); table_put(self.hash_table, h, curr); - let nb_compares = bt_walk::( + let nb_compares = bt_walk::<{ First::Alike }, { Hints::All }>( self.bt, &self.input, ip, @@ -420,7 +420,7 @@ fn sort_dubt1_in_dict( bt_low: u32, window_low: u32, ) -> Result<()> { - let older = input.dict; + let older = input.dict(); let nothing = Prefix { buf: &[], base: 0 }; let older = TreeInput::new(older, nothing, low_limit, older.pos_of(input.dict_limit))?; sort_dubt1(bt, &older, curr, nb_compares, bt_low, window_low) @@ -438,7 +438,7 @@ fn sort_dubt1( ) -> Result<()> { // The next sorted candidate is in the first slot. The other one has the previous unsorted candidate. let [next_sorted, _] = bt_children(bt, curr); - bt_walk::( + bt_walk::<{ First::Alike }, { Hints::All }>( bt, input, curr.wrapping_sub(input.dict_limit), diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index c99ca47c80b8..2580c0ee28f6 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -6,11 +6,11 @@ use core::mem; use super::hist; use super::huf::HufRepeat; use super::internal::{ - BlockSrc, Common, Descent, DictMatchState, DictMode, DictTree, EntropyCTables, Hasher, Hints, - Match, MatchState, Near, Nodes, OptState, ParamSwitch, Prefix, PriceTable, RawSeq, RawSeqStore, - SeqStore, Strategy, TreeInput, ZSTD_OPT_NUM, bt_look_ahead, bt_nodes, bt_set_children, bt_walk, - hash3_of, index_overlap_check, ll_code, ml_code, new_rep, offset_to_offbase, table_get, - table_put, + BlockSrc, Common, Descent, DictMatchState, DictMode, DictTree, EntropyCTables, First, Hasher, + Hints, Match, MatchState, Near, Nodes, OptState, ParamSwitch, Prefix, PriceTable, RawSeq, + RawSeqStore, SeqStore, Strategy, TreeInput, ZSTD_OPT_NUM, bt_look_ahead, bt_nodes, + bt_set_children, bt_walk, hash3_of, index_overlap_check, ll_code, ml_code, new_rep, + offset_to_offbase, table_get, table_put, }; use crate::common::{ LL_BITS, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, @@ -588,9 +588,9 @@ impl<'a> Trees<'a> { /// Adds the position `ip` of `near`, which starts with `bytes`, to its tree. `visit` is called with each position /// that is met on the way, which are all at least `low`, and what it has in common with `ip`, and returns whether - /// to go on. Returns how many more positions could have been compared. + /// to go on. Returns how many more positions could have been compared. `FIRST` and `HINTS` are those of [`bt_walk`]. #[inline(always)] - fn walk( + fn walk( &mut self, ip: u32, bytes: u64, @@ -612,7 +612,7 @@ impl<'a> Trees<'a> { } let descent = Descent::new(self.bt, curr); - bt_walk::( + bt_walk::( self.bt, &self.input, ip, @@ -635,7 +635,7 @@ impl<'a> Trees<'a> { let mut match_end_idx = curr.wrapping_add(8 + 1); let mut best_length = 8u32; - self.walk::(ip, bytes, window_low, |match_index, common| { + self.walk::<{ First::Words }, HINTS>(ip, bytes, window_low, |match_index, common| { if common.len > best_length { cold_path(); best_length = common.len; @@ -734,7 +734,7 @@ impl<'a> Trees<'a> { { return Ok(None); } - let dict = input.dict; + let dict = input.dict(); ( dict.buf, dict.pos_of(rep_index), @@ -754,7 +754,7 @@ impl<'a> Trees<'a> { if input.dict_limit.wrapping_sub(index) < 4 { return Ok(true); } - let there = read_le32(input.dict.buf, input.dict.pos_of(index))?; + let there = read_le32(input.dict().buf, input.dict().pos_of(index))?; Ok(minmatch_differs(there, bytes, 3) == 0) } @@ -901,7 +901,7 @@ impl<'a> Trees<'a> { ) -> Result { let curr = ip.wrapping_add(self.input.dict_limit); let mut is_over = false; - let left = self.walk::<{ Hints::All }>(ip, bytes, match_low, |match_index, common| { + let visit = |match_index: u32, common: Common| -> Result { if common.len > found.best_length { if common.len > found.match_end_idx.wrapping_sub(match_index) { found.match_end_idx = match_index.wrapping_add(common.len); @@ -922,7 +922,8 @@ impl<'a> Trees<'a> { } } Ok(true) - })?; + }; + let left = self.walk::<{ First::Apart }, { Hints::All }>(ip, bytes, match_low, visit)?; Ok(if is_over { 0 } else { left }) } From f9f4dec21a84b8c27dc726ff6592df4f8fb09d75 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 13:13:42 +0000 Subject: [PATCH 195/199] zstd: a copy of get_all_matches for a dictionary in place and one for an older segment At levels 13 to 19, what took longest relative to C was an input with something before its current segment: a dictionary that is searched in place, or an older segment, which is what a dictionary whose tables are copied is, or the part of a stream before the place where its buffer starts again. Two things cost there. A repeat offset that leads before the current segment was a candidate without a look at what is there, so `get_all_matches` called `rep_len` for it, up to three times a position. `far_rep`, which the loops over literals use, tells whether it starts like the position. A step down a tree that gets to the older segment first failed in `Near::compare_chunks` and then called `compare_rest`. For 32 KB of records of 100 bytes at level 16 with a copied dictionary of 64 KB, 86% of the time is `get_all_matches` and the `compare_rest` that its walk calls, which alone is 39%. `get_all_matches` did both for all modes, and a change to it moves the times of all of them. Now there is a copy for each `Before`, as there is of the loops over literals and of the parsers: `get_all_matches_in_place` and `get_all_matches_older`. `get_all_matches` is the same instructions as before. - Both copies ask `starts_like_rep`, which has `far_rep`, for the three repeat offsets. - `bt_walk` takes an `Older`. `Called` is what it did. `TriedLast` is the same without the call. `TriedFirst` looks at whether the position is in the older segment before anything else, which takes a cycle for one that is not. - `get_all_matches_older` looks for matches with `TriedFirst`, and adds the positions that a match skipped with `TriedLast`, but for a long run in a current segment of more than 64 KB. That is one more `update_tree_in_part`. - All else stays `Called`: the loops over literals, and the binary tree finder of levels 13 to 15, which has not been tried with the others. Time relative to before, means of 16 orders of the functions in the binary for each, 1,031 rows: 197 were more than 3% behind C, 116 are. 499 take less time by more than 3 standard errors, 185 by more than 5%, 64 by more than 10%. By what there is before the input (rows, how many behind C, mean relative to C): - a dictionary of 64 KB whose tables are copied, 11 kinds, 16 to 256 KB (99): 24 and 7, 0.928 and 0.880. Records of 100 bytes 0.81 of before (0.97 of C, now 0.80), of 1 KB 0.88 (1.14, now 1.01), HTML 0.94 (0.93, now 0.87), JSON 0.94, hex digits 0.96, text 0.97. - the same, pieces of the dictionary and copies with bytes altered (40): 23 and 15, 1.006 and 0.933 - a dictionary of 64 or 30 KB, 1 to 32 KB, 11 kinds (286): where it is searched in place (198) 17 and 13, 0.862 and 0.851; where it is copied (88) 22 and 2, 0.906 and 0.842 - the same, the pieces and copies (176): in place (132) 50 and 35, 0.982 and 0.940; copied (44) 23 and 12, 0.981 and 0.901 - a stream that is longer than its window (255): 16 and 10, 0.809 and 0.800 - nothing, 1 and 16 KB (175), the same instructions: 22 and 22, 0.922 and 0.923 Three rows take longer by more than 3 standard errors. Each was run three more times with more rounds. 64 KB with a period of 60 at level 16 with a copied dictionary 0.2%, then 0.2 to 0.4% (it is at 0.88 of C). Two of streams, 3.7 and 1.4%, then 0.976 to 1.003. 54 more rows after a review, which has 2% more instructions where there is a copied dictionary but the trees lead to the current segment: six kinds of other data, 32 to 256 KB, levels 13, 16 and 19. Records of 100, 300 and 1,000 bytes of which each is the one before with bytes altered: 0.993 to 1.006, one row beyond 3 standard errors, 0.4%. A period of 60: 0.998 to 1.005, two rows, 0.4 and 0.5%. JSON 0.92 to 1.00, logs 0.99 to 1.00. All of these rows but three are ahead of C. The same instructions: `get_all_matches`, `compare_rest`, the loops over literals, the two parsers without a mode, the binary tree finder, the three `update_tree_in_part` there were. The four parsers with a mode call other functions. It is 11,958 bytes more code. The output does not change. What else was tried, 8 or 16 orders each unless it says one: - `TriedFirst` for all walks where there is an older segment: more for records (a copied dictionary: 3 rows behind C, 0.873 in the mean; 256 KB of records of 1 KB 0.85 where this has 1.00), but 24 rows take 0.5 to 2.5% longer, one 3.5%: a period of 60 at all sizes, 256 KB of noise, HTML, JSON and logs, streams of records with a window of 256 KB. - `TriedLast` for all of them (one order): two thirds of that gain, and 256 KB of noise takes 4% longer. - With `TriedFirst`, no call either for what `compare_chunks` leaves of a match in the current segment: a period of 60 takes 0.3 to 1.6% longer from level 16 on, 7 rows. - Only the copies, with the call: records of 100 bytes with a copied dictionary take 1.1 to 3.8% longer from level 16 on, 15 rows. The walk of the copy came out with 3% more instructions. - The dictionary handed to `starts_like_rep` by the caller: hex digits take 0.6 to 2.6% longer with a dictionary in place. - The far case in the closure that `rep_offsets.map` takes: the closure is not inlined any more, and all data takes 10 to 18% longer (one order). - `Older` as a const parameter of `bt_walk`, so that `tree_matches` has two calls: other instructions for `get_all_matches` and the loops in place. Not timed. - A function for the comparison without the call, which returns a `Result`: 1 to 2% more instructions (one order). - Two loops, since the indices only go down on the way: one for the current segment, then one for the older. 2% more instructions where few walks get there, and no less time (one order). The second loop out of line: what the closure captures is in memory then, 3 to 10% more instructions (one order). - `cold_path` before the code for the older segment: nothing, or slower (one order). Of 15 faults put into the new code, runs against the C library with random parameters see 9. The other six cannot change the output: five only choose another way to the same comparison, one is about an offset that `far_rep` refuses. All 9 change a digest of the tests in `zstd.test.ts` for literals while a repeat offset leads before the current segment. `a_way_down_a_tree_is_the_same_whatever_is_done_with_the_older_segment` is new: it fills a tree over two segments with each `Older` and each `First`, and compares each step with the bytes. It sees the two faults in `bt_walk`. It is not run under Miri, where it takes minutes. `a_match_that_leaves_the_older_segment_goes_on_in_the_current_one` does the comparison without the call too. All times are from x86-64. --- src/zstd/compress/internal.rs | 127 ++++++++++++++++++++-- src/zstd/compress/lazy.rs | 4 +- src/zstd/compress/opt.rs | 193 +++++++++++++++++++++++++++------- 3 files changed, 277 insertions(+), 47 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index eb45ba767ec2..923a44c11099 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -1377,7 +1377,7 @@ impl<'a> TreeInput<'a> { } /// [`Near::compare_chunks`] for the next 16 bytes alone, if the match has them in the older segment. In words, - /// whatever the walk does in the current segment: there is one copy of this, out of line. + /// whatever the walk does in the current segment. #[inline(always)] fn compare_older( &self, @@ -1469,11 +1469,23 @@ pub(crate) enum Hints { None, } +/// What [`bt_walk`] does with a position of the older segment. With a dictionary of 64 KB there, records take 12 to 19% +/// less time without the call, HTML and JSON 6%. +#[derive(Clone, Copy, PartialEq, Eq, ConstParamTy)] +pub(crate) enum Older { + /// A call, once the position is found not to be in the current segment. + Called, + /// The same without a call. + TriedLast, + /// Before the current segment is tried, which takes a cycle for a position that is there. + TriedFirst, +} + /// Goes down the tree whose root is `root` for the position `ip` of `input.near`, which `descent` is for and which /// becomes the root. `visit` is called with each position that is met on the way, which are all at least `low` and at /// most `nb_compares`, and what it has in common with `ip`, and returns whether to go on. Returns how many more positions /// could have been compared. With [`First::Alike`], [`Near::compare_head`] is tried before [`Near::compare_chunks`], which -/// is for ways down that are short. +/// is for ways down that are short. `older` is a constant. #[inline(always)] pub(crate) fn bt_walk( bt: &mut Nodes, @@ -1483,6 +1495,7 @@ pub(crate) fn bt_walk( low: u32, mut nb_compares: u32, mut descent: Descent, + older: Older, mut visit: impl FnMut(u32, Common) -> Result, ) -> Result { if bt.is_empty() { @@ -1514,9 +1527,21 @@ pub(crate) fn bt_walk( let common = match head.and_then(|head| near.compare_head(head, match_pos, from)) { Some(common) => common, None => { - let by_chunks = near.compare_chunks::(ip, match_pos, from); + let at_once = matches!(older, Older::TriedFirst) && match_index < dict_limit; + let by_chunks = if at_once { + Err(from) + } else { + near.compare_chunks::(ip, match_pos, from) + }; match by_chunks { Ok(common) => common, + Err(from) if at_once || matches!(older, Older::TriedLast) => { + // As a function that `compare_rest` calls too, it takes 1 to 2% more instructions. + match input.compare_older(ip, match_index, from) { + Ok(common) => common, + Err(from) => input.compare_to_the_end(ip, match_index, from)?, + } + } Err(from) => input.compare_rest(ip, match_index, from)?, } } @@ -2922,15 +2947,21 @@ mod tests { for from in [0, len / 2, len] { let match_pos = match_index.wrapping_sub(DICT_LIMIT); let near = input.near; - for by_chunks in [ + let called = [ near.compare_chunks::<{ First::Alike }>(IP, match_pos, from), near.compare_chunks::<{ First::Apart }>(IP, match_pos, from), near.compare_chunks::<{ First::Words }>(IP, match_pos, from), - ] { - let common = match by_chunks { - Ok(common) => common, - Err(from) => input.compare_rest(IP, match_index, from)?, - }; + ] + .map(|by_chunks| match by_chunks { + Ok(common) => Ok(common), + Err(from) => input.compare_rest(IP, match_index, from), + }); + let inline = match input.compare_older(IP, match_index, from) { + Ok(common) => Ok(common), + Err(from) => input.compare_to_the_end(IP, match_index, from), + }; + for common in called.into_iter().chain([inline]) { + let common = common?; assert_eq!( (common.len, common.match_is_smaller), (len, (IP + len < END).then_some(match_is_smaller)), @@ -2944,6 +2975,84 @@ mod tests { Ok(()) } + /// Adds the positions from `from` up to `to` to one tree, and compares what each step finds with `window`, which + /// starts at the index `low`. Returns how many matches start in the older segment and end in the current one. + fn fill_a_tree( + bt: &mut Nodes, + input: &TreeInput<'_>, + window: &[u8], + low: u32, + (from, to): (u32, u32), + older: Older, + ) -> Result { + let mut leave_the_older_segment = 0; + for curr in from..to { + let visit = |index: u32, common: Common| { + let here = window.sub((curr - low) as usize..)?; + let there = window.sub((index - low) as usize..)?; + let len = here.iter().zip(there).take_while(|(a, b)| a == b).count(); + let is_smaller = here.get(len).map(|byte| there.get(len) < Some(byte)); + assert_eq!( + (common.len as usize, common.match_is_smaller), + (len, is_smaller), + "{curr} {index}" + ); + let limit = input.dict_limit; + leave_the_older_segment += u32::from(index < limit && index + common.len > limit); + Ok(is_smaller.is_some()) + }; + let (ip, descent) = (curr - input.dict_limit, Descent::new(bt, curr)); + let root = curr - 1; + bt_walk::(bt, input, ip, root, low, 64, descent, older, visit)?; + } + Ok(leave_the_older_segment) + } + + #[test] + #[cfg_attr(miri, ignore = "takes minutes")] + fn a_way_down_a_tree_is_the_same_whatever_is_done_with_the_older_segment() -> Result<()> { + const LOW: u32 = 100; + const OLDER: u32 = 300; + const DICT_LIMIT: u32 = LOW + OLDER; + const END: u32 = OLDER; + // A period with a byte altered now and then: what two positions have in common is often more than 16 bytes. + let mut seed = 5u32; + let window = Vec::from_iter((0..OLDER + END).map(|at| { + seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (at % 37) as u8 ^ u8::from(seed >> 27 == 0) + })); + let (older, current) = window.split_at(OLDER as usize); + // The older segment gets into the tree while it is the current one. + let mut at_first = vec![[0; 2]; 1024]; + let input = two_segments(&[], LOW, older, LOW, OLDER)?; + let positions = (LOW + 1, DICT_LIMIT); + fill_a_tree::<{ First::Words }>( + &mut at_first, + &input, + older, + LOW, + positions, + Older::Called, + )?; + + let input = two_segments(older, LOW, current, DICT_LIMIT, END)?; + let positions = (DICT_LIMIT, DICT_LIMIT + END); + let mut trees = Vec::new(); + for how in [Older::Called, Older::TriedLast, Older::TriedFirst] { + let mut by_first = [(); 3].map(|()| at_first.clone()); + let [alike, apart, words] = &mut by_first; + let counts = [ + fill_a_tree::<{ First::Alike }>(alike, &input, &window, LOW, positions, how)?, + fill_a_tree::<{ First::Apart }>(apart, &input, &window, LOW, positions, how)?, + fill_a_tree::<{ First::Words }>(words, &input, &window, LOW, positions, how)?, + ]; + assert!(counts.iter().all(|&count| count > 30), "{counts:?}"); + trees.extend(by_first); + } + assert!(trees.iter().all(|tree| Some(tree) == trees.first())); + Ok(()) + } + #[test] fn matches_are_counted_up_to_where_they_end() -> Result<()> { const DISTANCE: u32 = 7; diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 397666643d08..0e446941cbca 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -6,7 +6,7 @@ use core::ops::Range; use super::internal::{ BlockSrc, CompressionParameters, Descent, DictMatchState, DictMode, DictTree, First, Hash8, - Hasher, Hints, K_SEARCH_STRENGTH, MatchState, Near, Nodes, Prefix, REPCODE1_TO_OFFBASE, + Hasher, Hints, K_SEARCH_STRENGTH, MatchState, Near, Nodes, Older, Prefix, REPCODE1_TO_OFFBASE, RowTags, SecondTagStats, SeqSink, SeqStore, Strategy, TreeInput, ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, bt_children, bt_look_ahead, bt_low, bt_nodes, bt_set_children, bt_walk, count_2segments, hash_ptr, hashed_swap, index_overlap_check, offbase_is_offset, @@ -227,6 +227,7 @@ impl<'a> Dubt<'a> { low, self.nb_compares, Descent::new(self.bt, curr), + Older::Called, |match_index, common| { if common.len > best_length { if common.len > match_end_idx.wrapping_sub(match_index) { @@ -446,6 +447,7 @@ fn sort_dubt1( window_low.saturating_add(1), nb_compares, Descent::down_to(curr, bt_low), + Older::Called, // If they are equal, there is no way to know which is smaller. Dropped, to keep the tree consistent. |_, common| Ok(common.match_is_smaller.is_some()), )?; diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index 2580c0ee28f6..dc1ffc5899ed 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -7,8 +7,8 @@ use super::hist; use super::huf::HufRepeat; use super::internal::{ BlockSrc, Common, Descent, DictMatchState, DictMode, DictTree, EntropyCTables, First, Hasher, - Hints, Match, MatchState, Near, Nodes, OptState, ParamSwitch, Prefix, PriceTable, RawSeq, - RawSeqStore, SeqStore, Strategy, TreeInput, ZSTD_OPT_NUM, bt_look_ahead, bt_nodes, + Hints, Match, MatchState, Near, Nodes, Older, OptState, ParamSwitch, Prefix, PriceTable, + RawSeq, RawSeqStore, SeqStore, Strategy, TreeInput, ZSTD_OPT_NUM, bt_look_ahead, bt_nodes, bt_set_children, bt_walk, hash3_of, index_overlap_check, ll_code, ml_code, new_rep, offset_to_offbase, table_get, table_put, }; @@ -588,13 +588,14 @@ impl<'a> Trees<'a> { /// Adds the position `ip` of `near`, which starts with `bytes`, to its tree. `visit` is called with each position /// that is met on the way, which are all at least `low`, and what it has in common with `ip`, and returns whether - /// to go on. Returns how many more positions could have been compared. `FIRST` and `HINTS` are those of [`bt_walk`]. + /// to go on. Returns how many more positions could have been compared. The rest is for [`bt_walk`]. #[inline(always)] fn walk( &mut self, ip: u32, bytes: u64, low: u32, + older: Older, visit: impl FnMut(u32, Common) -> Result, ) -> Result { let curr = ip.wrapping_add(self.input.dict_limit); @@ -620,6 +621,7 @@ impl<'a> Trees<'a> { low, self.nb_compares, descent, + older, visit, ) } @@ -627,7 +629,11 @@ impl<'a> Trees<'a> { /// `ZSTD_insertBt1`: adds `curr`, which is at least 8 bytes before the end, to its tree. `target` is up to where /// the caller is filling them. Returns how many positions to move forward. #[inline(always)] - fn insert_bt1(&mut self, curr: u32, target: u32) -> Result { + fn insert_bt1( + &mut self, + curr: u32, + target: u32, + ) -> Result { let ip = curr.wrapping_sub(self.input.dict_limit); let bytes = self.input.near.read64(ip).ok_or(BUG)?; // Only the positions that are still in the window once the trees are filled are of use. @@ -635,7 +641,7 @@ impl<'a> Trees<'a> { let mut match_end_idx = curr.wrapping_add(8 + 1); let mut best_length = 8u32; - self.walk::<{ First::Words }, HINTS>(ip, bytes, window_low, |match_index, common| { + let visit = |match_index: u32, common: Common| -> Result { if common.len > best_length { cold_path(); best_length = common.len; @@ -645,7 +651,8 @@ impl<'a> Trees<'a> { } // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. Ok(common.match_is_smaller.is_some()) - })?; + }; + self.walk::<{ First::Words }, HINTS>(ip, bytes, window_low, OLDER, visit)?; let positions = if best_length > 384 { 192.min(best_length - 384) @@ -655,16 +662,17 @@ impl<'a> Trees<'a> { Ok(positions.max(match_end_idx.wrapping_sub(curr.wrapping_add(8)))) } - /// `ZSTD_updateTree_internal`: adds the positions from `next_to_update` up to `target` to the trees. + /// `ZSTD_updateTree_internal`: adds the positions from `next_to_update` up to `target` to the trees. `OLDER` is not + /// for a long run in a long current segment, where HTML takes 1 to 2% longer with anything but a call. #[inline(always)] - fn update_tree(&mut self, target: u32) -> Result<()> { + fn update_tree(&mut self, target: u32) -> Result<()> { if target.saturating_sub(self.next_to_update) < LONG_RUN || !self.input.far { - self.update_tree_in_part::<{ Hints::All }>(target, target)?; + self.update_tree_in_part::<{ Hints::All }, OLDER>(target, target)?; // A match skipped them: they go down past its positions, which follow each other too and so are mostly at hand. } else if self.nb_compares > SHORT_WAY_DOWN { - self.update_tree_in_part::<{ Hints::Bytes }>(target, target)?; + self.update_tree_in_part::<{ Hints::Bytes }, { Older::Called }>(target, target)?; } else { - self.update_tree_in_part::<{ Hints::None }>(target, target)?; + self.update_tree_in_part::<{ Hints::None }, { Older::Called }>(target, target)?; } self.next_to_update = target; Ok(()) @@ -672,10 +680,14 @@ impl<'a> Trees<'a> { /// The part of [`Self::update_tree`] that gets to `up_to`, or somewhat beyond. #[inline(never)] - fn update_tree_in_part(&mut self, up_to: u32, target: u32) -> Result<()> { + fn update_tree_in_part( + &mut self, + up_to: u32, + target: u32, + ) -> Result<()> { let mut idx = self.next_to_update; while idx < up_to { - idx = idx.wrapping_add(self.insert_bt1::(idx, target)?); + idx = idx.wrapping_add(self.insert_bt1::(idx, target)?); } self.next_to_update = idx; Ok(()) @@ -768,6 +780,89 @@ impl<'a> Trees<'a> { ip: u32, rep: &[u32; ZSTD_REP_NUM], ll0: bool, + ) -> Result { + self.get_all_matches_generic(matches, ip, rep, ll0, Before::Nothing) + } + + /// [`Self::get_all_matches`] where a dictionary is searched in place. + #[inline(never)] + fn get_all_matches_in_place( + &mut self, + matches: &mut [Match], + ip: u32, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + ) -> Result { + self.get_all_matches_generic(matches, ip, rep, ll0, Before::Dictionary) + } + + /// [`Self::get_all_matches`] where the window has an older segment. + #[inline(never)] + fn get_all_matches_older( + &mut self, + matches: &mut [Match], + ip: u32, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + ) -> Result { + self.get_all_matches_generic(matches, ip, rep, ll0, Before::OlderSegment) + } + + /// The one of these three for what there is before the current segment. + #[inline(always)] + fn get_all_matches_after( + &mut self, + before: Before, + matches: &mut [Match], + ip: u32, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + ) -> Result { + match before { + Before::Nothing => self.get_all_matches(matches, ip, rep, ll0), + Before::OlderSegment => self.get_all_matches_older(matches, ip, rep, ll0), + Before::Dictionary => self.get_all_matches_in_place(matches, ip, rep, ll0), + } + } + + /// Whether `ip`, which starts with `bytes`, starts like what is at `rep_offset`, before the current segment too. + /// `None` is a bug. + #[inline(always)] + fn starts_like_rep( + &self, + before: Before, + ip: u32, + bytes: u32, + rep_offset: u32, + most_in_prefix: u32, + window_low: u32, + ) -> Option { + if rep_offset.wrapping_sub(1) < most_in_prefix { + let rep_bytes = self.input.near.read32(ip.wrapping_sub(rep_offset))?; + Some(minmatch_differs(rep_bytes, bytes, self.min_match) == 0) + } else if rep_offset.wrapping_sub(1) >= ip { + let far = match before { + Before::Dictionary => self.far_rep(self.dms, ip, rep_offset, window_low), + Before::Nothing | Before::OlderSegment => { + debug_assert!(self.dms.is_none()); + self.far_rep(None, ip, rep_offset, window_low) + } + }; + Some(far.ok()?.is_some()) + } else { + Some(false) + } + } + + /// [`Self::get_all_matches`], of which there is a copy for each `before`. + #[inline(always)] + fn get_all_matches_generic( + &mut self, + matches: &mut [Match], + ip: u32, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + before: Before, ) -> Result { let near = self.input.near; let dict_limit = self.input.dict_limit; @@ -777,7 +872,13 @@ impl<'a> Trees<'a> { return Ok(0); } if self.next_to_update < curr { - self.update_tree(curr)?; + match before { + // Periodic data takes 1 to 2% longer with `TriedFirst`. + Before::OlderSegment => self.update_tree::<{ Older::TriedLast }>(curr)?, + Before::Nothing | Before::Dictionary => { + self.update_tree::<{ Older::Called }>(curr)?; + } + } } let min_match = self.min_match; @@ -802,15 +903,26 @@ impl<'a> Trees<'a> { }; // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. let most_in_prefix = ip.min(curr.wrapping_sub(window_low)); - let candidates = rep_offsets.map(|rep_offset| { - // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. - if rep_offset.wrapping_sub(1) < most_in_prefix { - let rep_bytes = near.read32(ip.wrapping_sub(rep_offset))?; - Some(minmatch_differs(rep_bytes, bytes as u32, min_match) == 0) - } else { - Some(rep_offset.wrapping_sub(1) >= ip) + let [a, b, c] = rep_offsets; + let candidates = match before { + Before::OlderSegment | Before::Dictionary => [ + self.starts_like_rep(before, ip, bytes as u32, a, most_in_prefix, window_low), + self.starts_like_rep(before, ip, bytes as u32, b, most_in_prefix, window_low), + self.starts_like_rep(before, ip, bytes as u32, c, most_in_prefix, window_low), + ], + // With the other case in this closure, it is not inlined: all data takes 10 to 18% longer. + Before::Nothing => { + rep_offsets.map(|rep_offset| { + // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. + if rep_offset.wrapping_sub(1) < most_in_prefix { + let rep_bytes = near.read32(ip.wrapping_sub(rep_offset))?; + Some(minmatch_differs(rep_bytes, bytes as u32, min_match) == 0) + } else { + Some(rep_offset.wrapping_sub(1) >= ip) + } + }) } - }); + }; if candidates != [Some(false); 3] { for ((off, rep_offset), is_candidate) in (1..).zip(rep_offsets).zip(candidates) { if !is_candidate.ok_or(BUG)? { @@ -867,7 +979,11 @@ impl<'a> Trees<'a> { } let mut found = Found::new(mnum, best_length, curr); - let nb_compares = self.tree_matches(matches, ip, bytes, match_low, &mut found)?; + let older = match before { + Before::OlderSegment => Older::TriedFirst, + Before::Nothing | Before::Dictionary => Older::Called, + }; + let nb_compares = self.tree_matches(matches, ip, bytes, match_low, &mut found, older)?; if dict_root != 0 && nb_compares != 0 { // A copy, for the one that the loop above works on to need no place in memory. let mut with_dict = found; @@ -898,6 +1014,7 @@ impl<'a> Trees<'a> { bytes: u64, match_low: u32, found: &mut Found, + older: Older, ) -> Result { let curr = ip.wrapping_add(self.input.dict_limit); let mut is_over = false; @@ -923,7 +1040,8 @@ impl<'a> Trees<'a> { } Ok(true) }; - let left = self.walk::<{ First::Apart }, { Hints::All }>(ip, bytes, match_low, visit)?; + let left = + self.walk::<{ First::Apart }, { Hints::All }>(ip, bytes, match_low, older, visit)?; Ok(if is_over { 0 } else { left }) } @@ -1120,7 +1238,10 @@ impl<'a> Trees<'a> { } let mut found = Found::new(0, min_match - 1, curr); - let nb_compares = self.tree_matches(matches, ip, bytes, match_low, &mut found)?; + // Noise takes 2 to 4% longer with anything but a call. + let older = Older::Called; + let nb_compares = + self.tree_matches(matches, ip, bytes, match_low, &mut found, older)?; if dict_root != 0 && nb_compares != 0 { // A copy, so that `found` need not be in memory: 1 to 2% where there are no matches. let mut with_dict = found; @@ -1233,7 +1354,7 @@ pub(crate) fn update_tree( ms.check_cancelled()?; let up_to = ms.next_to_update.saturating_add(1 << 12).min(target); let mut trees = Trees::new(ms, prefix, no_dict, None, iend, mls)?; - trees.update_tree_in_part::<{ Hints::All }>(up_to, target)?; + trees.update_tree_in_part::<{ Hints::All }, { Older::Called }>(up_to, target)?; ms.next_to_update = trees.next_to_update; } ms.next_to_update = target; @@ -1467,7 +1588,7 @@ fn try_lengths( } } -/// `ZSTD_compressBlock_opt_generic`. `before` is what `dict_mode` says, as a constant. +/// `ZSTD_compressBlock_opt_generic`. `before` is a constant: what [`compress_block_opt`] finds before the input. #[inline(always)] fn compress_block_opt_generic( ms: &mut MatchState, @@ -1586,7 +1707,8 @@ fn parse_block( let litlen = (ip - anchor) as u32; if nb_matches == 0 { let pos = (ip - prefix_start) as u32; - nb_matches = trees.get_all_matches(matches, pos, rep, litlen == 0)?; + nb_matches = + trees.get_all_matches_after(before, matches, pos, rep, litlen == 0)?; } opt_ldm.process_match_candidate( matches, @@ -1727,12 +1849,9 @@ fn parse_block( } { - let mut nb_matches = trees.get_all_matches( - matches, - (inr - prefix_start) as u32, - opt_rep.get(at).ok_or(BUG)?, - ll0, - )?; + let (pos, rep) = ((inr - prefix_start) as u32, opt_rep.get(at).ok_or(BUG)?); + let mut nb_matches = + trees.get_all_matches_after(before, matches, pos, rep, ll0)?; opt_ldm.process_match_candidate( matches, &mut nb_matches, @@ -1886,8 +2005,8 @@ fn compress_block_opt0( compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, Before::Nothing) } -/// [`compress_block_opt0`] with [`Trees::skip_matchless_in_place`]. A copy: one parser that chooses takes 0.4 to 1.4% -/// longer on most data without a dictionary. +/// [`compress_block_opt0`] with what [`Trees`] has for a dictionary in place. A copy: one parser that chooses takes 0.4 +/// to 1.4% longer on most data without a dictionary. #[inline(never)] fn compress_block_opt0_in_place( ms: &mut MatchState, @@ -1899,7 +2018,7 @@ fn compress_block_opt0_in_place( compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, Before::Dictionary) } -/// [`compress_block_opt0`] with [`Trees::skip_matchless_older`], likewise. +/// [`compress_block_opt0`] with what [`Trees`] has for an older segment, likewise. #[inline(never)] fn compress_block_opt0_older( ms: &mut MatchState, From 209068dfd1fdc8e7fd945a8863b89deb022e0128 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 15:05:03 +0000 Subject: [PATCH 196/199] zstd: a position that is added to the binary trees does not wait for the one before At levels 13 to 19, records were still behind C where nothing comes before the input: 16 KB of records of 100 bytes took 1.13 and 1.15 times its time at levels 13 and 16, with fewer instructions (8.57 G for 9.03 G) and fewer of them a cycle (4.4 for 5.25). 83% of the time is `update_tree_in_part`, which adds the positions that a match skipped. Five times as many cycles as in C go by with a scheduler of the integer units full (0.27 G for 0.05 G of `int_sch0_token_stall`), and 2.7 times as many places to dispatch to stay empty for want of room behind. `insert_bt1` returns how many positions to move forward, which comes from the longest match that the walk met and from how far a match reached. That is a chain from one position to the next: the last comparison, the length, two conditional moves, the index, the 8 bytes there, the hash, the cell of the table, the first comparison. C has the same chain. The answer is 1 but for matches that overlap the position or are longer than 384 bytes. Now `update_tree_in_part` has a branch for 1, so the processor goes on with the next position while the walk of this one ends. The compiler folds such a branch into the addition that it stands for, so the other arm goes through `core::hint::black_box`. Also, `update_tree` hands `OLDER` to the instance without hints too, which is for long runs in a current segment of more than 64 KB where ways down are short. So far only short runs had it. That is one more `update_tree_in_part`. The one with hints at the bytes alone stays as it is: without the call, 256 KB of records of 1 KB at level 16 take 15% less time, but HTML 2.5% and a period of 60 1.3% more (one order). Time relative to before, means of 16 orders of the functions in the binary for each, 1,206 rows: 144 were more than 3% behind C, 97 are. 332 take less time by more than 3 standard errors, 68 by more than 5%, 18 by more than 10%. By kind, all rows of levels 13 to 19 but those with pieces of a dictionary (mean, best row, how many are more than 3% behind C): - records of 100 bytes (63 rows) 0.955, 0.80, 21 before and 4 now - records of 1 KB (63) 0.960, 0.85, 19 and 3 - words drawn from 2,000 at random (63) 0.976, 0.93; HTML (63) 0.983, 0.92, 3 and none - CSV (13) 0.983, 0.95; a period of 60 (63) 0.985, 0.94 By what there is before the input (rows, how many behind C, mean relative to C): - nothing, 1 and 16 KB (175): 22 and 15, 0.921 and 0.912; 128 and 512 KB (175): 21 and 17, 0.894 and 0.889 - a dictionary of 64 or 30 KB, 1 to 32 KB, 11 kinds (286): where it is searched in place (198) 13 and none, 0.850 and 0.826; where it is copied (88) 2 and 1, 0.837 and 0.834 - the same, pieces of the dictionary and copies with bytes altered (176): 48 and 44, 0.930 and 0.922 - a dictionary of 64 KB whose tables are copied, 16 to 256 KB (99): 8 and 3, 0.880 and 0.869. 256 KB of records of 100 bytes at level 13 0.85 of before (1.22 of C, now 1.04), of 1 KB 0.90 at level 13 (1.12, now 1.00) and 0.85 at level 19 (1.12, now 0.95). - the same, the pieces and copies (40): 15 and 9, 0.931 and 0.920 - a stream that is longer than its window (255): 15 and 8, 0.805 and 0.798 Four rows take longer by more than 3 standard errors, by 0.7 to 1.4%. Each was run three more times with more rounds: 0.995 to 1.005, none beyond 3 standard errors. Other instructions: the four `update_tree_in_part` there were, and nothing else. It is 1,954 bytes more code. The output does not change. What else was tried: - `cold_path` where `insert_bt1` notes how far a match reaches and where it asks for more than 384 bytes, so that both stay branches and the answer is a constant on the usual way (16 orders): as much for records, but the loop of the walk comes out with 5% more instructions. With a copied dictionary from level 16 on HTML takes 2.2 to 4.7% longer and records of 1 KB 3.1 to 3.7%; 50 rows in all. - The other arm as a function that is cold and not inlined (one order): 3% more for records, but 3% more instructions where walks are long, and HTML with a copied dictionary takes 2% longer at level 16. - The branch with `cold_path` alone: folded, the same instructions as before. - `Descent::step` reads the one child that it takes, 4 bytes, where `bt_children` reads both as 8: a third fewer loads that wait for a store of 4 bytes, 3% fewer instructions, 0.5 to 1.8% less time for records and HTML, 1% more for noise and JSON (one order). Left for later: it changes all walks. - Hints at the older segment for the children that are there: 5 to 13% more instructions, slower in all rows (one order). - `rep_len` inlined in the two copies of `get_all_matches`: 2 to 3% more instructions and time (one order). Of 6 faults put into the new code, runs against the C library with random parameters see 5. The sixth only chooses the call for the older segment. The test "at level %i" in `zstd.test.ts` sees the 5: with its data, in one call as there, all change what level 16 makes, and 4 what levels 17 to 19 make. `black_box` is documented as best effort. If a compiler folds the branch one day, the output stays the same and no test fails: only the time is lost. The branch is there in the assembly for x86-64 and for aarch64. All times are from x86-64. --- src/zstd/compress/opt.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs index dc1ffc5899ed..1c2ba4957957 100644 --- a/src/zstd/compress/opt.rs +++ b/src/zstd/compress/opt.rs @@ -1,6 +1,6 @@ //! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. -use core::hint::{Locality, cold_path, prefetch_read}; +use core::hint::{Locality, black_box, cold_path, prefetch_read}; use core::mem; use super::hist; @@ -663,7 +663,7 @@ impl<'a> Trees<'a> { } /// `ZSTD_updateTree_internal`: adds the positions from `next_to_update` up to `target` to the trees. `OLDER` is not - /// for a long run in a long current segment, where HTML takes 1 to 2% longer with anything but a call. + /// for [`Hints::Bytes`], where HTML takes 2.5% longer without the call, though records of 1 KB 15% less. #[inline(always)] fn update_tree(&mut self, target: u32) -> Result<()> { if target.saturating_sub(self.next_to_update) < LONG_RUN || !self.input.far { @@ -672,7 +672,7 @@ impl<'a> Trees<'a> { } else if self.nb_compares > SHORT_WAY_DOWN { self.update_tree_in_part::<{ Hints::Bytes }, { Older::Called }>(target, target)?; } else { - self.update_tree_in_part::<{ Hints::None }, { Older::Called }>(target, target)?; + self.update_tree_in_part::<{ Hints::None }, OLDER>(target, target)?; } self.next_to_update = target; Ok(()) @@ -687,7 +687,15 @@ impl<'a> Trees<'a> { ) -> Result<()> { let mut idx = self.next_to_update; while idx < up_to { - idx = idx.wrapping_add(self.insert_bt1::(idx, target)?); + let forward = self.insert_bt1::(idx, target)?; + // A branch, so that the next position does not wait for what this one met: on x86-64, records take up to + // 20% less time. Without `black_box` it is folded into one addition. + idx = if forward == 1 { + idx.wrapping_add(1) + } else { + cold_path(); + idx.wrapping_add(black_box(forward)) + }; } self.next_to_update = idx; Ok(()) From fc13be22b7d790629c1240c650b6091d09526a06 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 18:04:32 +0000 Subject: [PATCH 197/199] zstd: sequences that are taken one at a time are decoded in registers, and ahead where the history is long A survey of the port against C without a dictionary, 25 kinds of data from 1 to 512 KB: of 1,200 rows of compression at levels 1 to 12, 116 are more than 3% behind, of 682 of decompression into a buffer that is there 139, of 700 into a new one 201. So this is about the decoder. What took longest relative to C was HTML, 1.09 times its time in the mean and up to 1.24, and records. Where the table of match lengths says that matches are long, and for blocks of up to 32 sequences, `Executor::run` takes one sequence at a time: it decodes it and executes it. Two things were wrong with that loop. It worked on `seq_state`, whose address `decode_batch` takes further down in the function. So all of it was in memory: the bits, how many are consumed, the three states, the last offsets. For 128 KB of HTML of level 3, 17% of the samples are on one reload of the bits, and there are 0.53 G loads that get their value from a store that is not done, for 0.42 G in C. Now the loop of `run::` has a copy that nothing takes the address of, as it has of the executor (`fork`): 0.28 G such loads, 8% fewer cycles. With `LONG`, which is for a history of more than 16 MB with far offsets, the same loop ran, and it asks for no match ahead of time. That had not been timed. 48 MB of which the second half is pieces of the first taken at random, with a window of 64 MB, took 1.18 to 1.64 times the time of C. Now `run_long_matches_ahead` does what `ZSTD_decompressSequencesLong_body` does: a sequence is executed once 8 more are decoded and their matches asked for, up to the end of the block. `LONG` is also for the first block after a dictionary that counts as cold, which is each one in a new context. Such a dictionary has just been copied, so it is in a cache. There the loop stays as it was, without the copy either: 128 KB of HTML, of records of 100 bytes and of copies of the dictionary with bytes altered take 5 to 7% longer with the copy, and 14 to 16% with 8 ahead (one order). So the new function is for a history of more than 16 MB alone. Time relative to before, means of 16 orders of the functions in the binary for each. Rows, how many are more than 3% behind C, mean relative to C, how many take less time by more than 3 standard errors: - into a buffer that is there (682): 123 and 107, 0.875 and 0.869, 107, of which 33 by more than 5%. HTML 0.950 of before (1.081 of C, now 1.027), records of 100 bytes 0.971 (1.012, now 0.982), a period of 60 0.972, words drawn from 200 at random 0.983, records of 1 KB 0.985. - into a new buffer (700): 191 and 190, 0.908 and 0.903, 105. HTML 0.953 (1.137, now 1.083), a period of 60 0.973, records of 100 bytes 0.977. - frames of 48 MB with a window of 64 MB (19, not timed so far): 14 and 14, 1.206 and 1.048, 14. At level 3 with long distance matching and at level 9, pieces from far back of 100 bytes 0.794 and 0.644 (1.39 and 1.64 of C, now 1.11 and 1.06); of 300 bytes 0.794 and 0.724 (1.35 and 1.52, now 1.07 and 1.10); of 1 KB 0.926 and 0.893 (1.18, now 1.09 and 1.05); of 70 to 330 bytes 0.781 and 0.718; of 8 to 2,000 bytes 0.945 and 0.926; of 20 to 90 bytes 0.971 and 0.868; of 40 bytes at level 9 0.680 (1.57, now 1.07). Of 20 bytes, which go in batches, 0.998. - a new context, a dictionary of 64 KB is loaded, a frame of 1 to 128 KB is decompressed, 26 kinds, levels 1, 3, 9 and 19 (356, not timed so far): 80 and 79, 0.928 and 0.929, 2. Eleven rows take longer by more than 3 standard errors. All but the first and the last were run three more times with more rounds. - 17 MB of source code three times over, with a byte altered every 200 bytes in the mean, at level 3 with long distance matching: 3.7% (0.897 of C, now 0.929). Its matches are far but one after the other, which the processor sees by itself. There the loop without anything ahead took a quarter less time than C's, and this one takes what C's takes. Two thirds of it is the delay, not the hints (one order). Nothing cheap tells such a block from one with pieces taken at random: long offsets have the same share of the table, 125 of 256. - 512 KB with a period of 1 KB, the frame of levels 9 and 13, into either buffer, four rows: 1.9 to 2.2%, then 1.3 to 2.6% (0.91 to 0.94 of C, and 1.01 to 1.02). A block is two sequences, and takes 181 more instructions: what is in registers is saved around the calls that copy. - 512 KB with a period of 100 KB of level 3 into a new buffer 1.2%, then 0.6 to 1.2%. 1 KB of skewed bytes of level 9 into a new buffer 2.7%, then 1.7 to 2.4%, within 3 standard errors (0.98 of C). - 16 KB of a binary of level 5 0.7%, then 0.998 to 1.001. With a dictionary, 128 KB of HTML of level 1 0.6% and 32 KB of records of 500 bytes of level 9 0.8%, then 0.0 to 0.4%, within 3 standard errors. - 16 KB of integers of level 19 into a new buffer 0.1%. Other instructions: `run::` and `run::`, which has the test for the new function. It is 5,736 bytes more code. The output and the errors do not change: the sequences are decoded in the same order, and the same ones are executed before the end of the stream is looked at. What else was tried, one order unless it says more: - The new loop in `run` itself (16 orders): as much for far matches, or 2% more. But `run::` comes out with other instructions, which gain no more, and 512 KB with a period of 100 KB of level 3 take 2.5% longer. - The call in `decompress_block`: that has 992 lines of assembly for 921. - The new function for all blocks with a long history, not only where the table says that matches are long: pieces of 20 bytes take 9% longer, of 20 to 90 bytes 3%. So blocks with the predefined table, which has few long lengths, stay in batches whatever their matches are. - With a long history, batches for long matches too: 1.14 to 1.37 of C. Batches of 16, 32 or 64 for 128: 1.15 to 1.31. - The 8 that wait are executed before the last 8 are decoded: 1.04 to 1.18 of C. Blocks of such data have 200 to 1,600 sequences. - 16 ahead: nothing. - Hints at all of a match, up to 512 or 1,024 bytes: pieces of 1 KB 0.92 to 0.97 of C, but those of 300 bytes take 2% longer, and so does the source code. - No hint for a match with the offset of the one before: nothing. - The loop as it was for blocks of up to 32 sequences: a third of what HTML gains is lost, most of what records of 100 bytes gain, and the period of 1 KB keeps two thirds of what it loses. - `History::prefetch` on top of what takes a `Sequence`, so that nothing is converted to and fro: `decode_batch` comes out with other instructions. Not timed. - Batches for all blocks, with a short history: HTML as here or 2 to 4% less, records of 100 bytes 13% more than before. - Matches of 17 to 64 bytes without a call of `memcpy`: HTML 1 to 2.5% less, records of 1 KB 2% more. Left for later. - For a new buffer, where 15 to 19% of the time of records of 1 KB is `memset`: blocks with long matches grow the buffer as they go, where now all that a block may write is zeroed first: records of 1 KB take 14 to 22% longer, HTML 3%. Of 13 faults put into the new code, 9 are seen. The state that is not written back after the loop of `run::`: runs against the C library with random input see it at once. Eight in the new function: 7 that wait for 8, 7 decoded ahead, no rotation, rotation the other way, the newest executed for the oldest, the last sequence not told from the others, the offsets of the start returned, the position not written back. Random input does not get there, as it takes 16 MB. The test "long matches from more than 16 MB back" in `zstd.test.ts` is new: its frame, whose blocks have 592 to 857 sequences, sees all eight, in one call and as a stream, and so do frames that were timed. Not seen: the end of the stream is not looked at, which takes a damaged frame; the last sequence is not told from the others among the first 8, which takes a block of up to 8 sequences; literals that would be in the output; a history of any length, which only costs time. 642 results of the debug build are those of Bun 1.4.4: the 19 frames, each also damaged at three places and cut short, and frames with a dictionary, with it, without it and damaged. All times are from x86-64. --- src/zstd/decompress/block.rs | 99 ++++++++++++++++++++++++++++++----- test/js/bun/util/zstd.test.ts | 19 +++++++ 2 files changed, 104 insertions(+), 14 deletions(-) diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs index bf683537f68c..419c9685e13d 100644 --- a/src/zstd/decompress/block.rs +++ b/src/zstd/decompress/block.rs @@ -733,7 +733,7 @@ impl BlockDecoder { small_output_used, } = &mut **scratch; let dict_tables = dict_entropy.map(|dict| &dict.seq); - let is_history_long = total_history_size > (1 << 24); + let is_history_long = total_history_size > MIN_FAR_HISTORY; let tables = if nb_seq <= MAX_ONE_BY_ONE && !use_prefetch_decoder && !is_history_long { entropy.seq.pick(self.seq_sources, dict_tables) } else { @@ -952,6 +952,16 @@ impl History { prefetch_read(from.wrapping_add(64), Locality::L1); self.position += (match_length + MINMATCH) as usize; } + + /// The same, whatever its offset. + #[inline(always)] + fn prefetch_match(&mut self, sequence: Sequence) { + self.prefetch( + sequence.lit_length, + sequence.match_length - MINMATCH, + (sequence.offset as u32).saturating_add(3), + ); + } } /// A copy of the start of a stream after zeroes, where it can be read as if its start were far. That it was not shows @@ -999,6 +1009,10 @@ impl PaddedStart { /// Up to how many sequences it is not worth it to decode them by the batch. const MAX_ONE_BY_ONE: usize = 32; +/// How many sequences `ZSTD_decompressSequencesLong_body` decodes ahead of the one that it executes. +const ADVANCED_SEQS: usize = 8; +/// A history that is longer is not all in a cache. +const MIN_FAR_HISTORY: usize = 1 << 24; /// `seqState_t` #[derive(Clone, Copy)] @@ -1874,6 +1888,48 @@ impl<'a> Executor<'_, 'a> { Ok(index) } + /// `ZSTD_decompressSequencesLong_body` the way it is in C: a sequence is executed once [`ADVANCED_SEQS`] more are + /// decoded and their matches asked for. It is for long matches, which [`Self::run`] takes one at a time too, and knows + /// nothing of literals that would be in the output. Returns the last offsets. + #[inline(never)] + fn run_long_matches_ahead( + &mut self, + seq_state: &SeqState<'_>, + nb_seq: usize, + ) -> Result<[usize; ZSTD_REP_NUM]> { + let mut ahead = [Sequence::default(); ADVANCED_SEQS]; + let ahead = ahead.sub_mut(..nb_seq.min(ADVANCED_SEQS))?; + let mut history = self.history(); + let mut exec = self.fork(); + let mut seq_state: SeqState<'_> = *seq_state; + let mut remaining = nb_seq; + for sequence in &mut *ahead { + *sequence = seq_state.decode_sequence(remaining == 1); + remaining -= 1; + history.prefetch_match(*sequence); + } + let mut oldest = 0; + while remaining > 0 { + let next = seq_state.decode_sequence(remaining == 1); + remaining -= 1; + history.prefetch_match(next); + let slot = ahead.at_mut(oldest)?; + exec.exec_sequence::(*slot)?; + *slot = next; + oldest = (oldest + 1) % ADVANCED_SEQS; + } + if !seq_state.bit_d.is_at_end() { + return Err(Error::CorruptionDetected); + } + ahead.rotate_left(oldest); + for &sequence in &*ahead { + exec.exec_sequence::(sequence)?; + } + let exec = (exec.op, exec.lits, exec.split); + self.join(exec); + Ok(seq_state.prev_offset) + } + /// `ZSTD_decompressSequences_body`, `ZSTD_decompressSequences_bodySplitLitBuffer` and, with `LONG`, /// `ZSTD_decompressSequencesLong_body`. Returns the last offsets. fn run( @@ -1882,10 +1938,17 @@ impl<'a> Executor<'_, 'a> { nb_seq: usize, batch: &mut [RawSequence], ) -> Result<[usize; ZSTD_REP_NUM]> { - // The C code then decodes 8 sequences ahead, so it sees whether the stream ends where it should before it - // executes the last ones. - let mut ahead = [Sequence::default(); 8]; - let ahead = ahead.sub_mut(..if LONG { nb_seq.min(8) } else { 0 })?; + if LONG + && self.op - self.prefix_start + self.ext_dict.len() > MIN_FAR_HISTORY + && seq_state.tables.matches_are_long() + && !self.split + { + return self.run_long_matches_ahead(seq_state, nb_seq); + } + // With `LONG`, the C code decodes that many sequences ahead, so it sees whether the stream ends where it should + // before it executes the last ones. + let mut ahead = [Sequence::default(); ADVANCED_SEQS]; + let ahead = ahead.sub_mut(..if LONG { nb_seq.min(ADVANCED_SEQS) } else { 0 })?; let mut start = None; // There until decoding goes on from it. let mut start = (!LONG).then_some(&mut start); @@ -1899,10 +1962,22 @@ impl<'a> Executor<'_, 'a> { if (seq_state.tables.matches_are_long() || nb_seq <= MAX_ONE_BY_ONE && !LONG) && !self.split { let mut exec = self.fork(); - while remaining > ahead.len() { - let sequence = seq_state.decode_sequence(remaining == 1); - remaining -= 1; - exec.exec_sequence::(sequence)?; + if LONG { + while remaining > ahead.len() { + let sequence = seq_state.decode_sequence(remaining == 1); + remaining -= 1; + exec.exec_sequence::(sequence)?; + } + } else { + // As for `fork`: `decode_batch` takes the address of `seq_state`. With `LONG`, HTML after a cold dictionary + // takes 7% longer with the copy. + let mut state = seq_state; + while remaining > ahead.len() { + let sequence = state.decode_sequence(remaining == 1); + remaining -= 1; + exec.exec_sequence::(sequence)?; + } + seq_state = state; } let exec = (exec.op, exec.lits, exec.split); self.join(exec); @@ -1962,11 +2037,7 @@ impl<'a> Executor<'_, 'a> { for sequence in &mut *ahead { *sequence = seq_state.decode_sequence(remaining == 1); remaining -= 1; - history.prefetch( - sequence.lit_length, - sequence.match_length - MINMATCH, - (sequence.offset as u32).saturating_add(3), - ); + history.prefetch_match(*sequence); } let is_at_end = match left_at { Some(_) => PaddedStart::is_reached(&seq_state.bit_d), diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index adf6c8a28331..cd7308908fb9 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -513,6 +513,25 @@ describe("decompressing frames whose size is known", () => { expect(decoded.equals(Buffer.concat([data, data, data]))).toBe(true); }); + // Then a sequence waits until 8 more are decoded, so that what it copies can be fetched by then. + it("long matches from more than 16 MB back", () => { + const c = zlib.constants; + const piece = (i: number) => pieces[0].subarray((i * 7919) % 139_000).subarray(0, 40 + ((i * 31) % 300)); + const data = Buffer.concat([ + pieces[0], + Buffer.alloc(17 << 20), + ...Array.from({ length: 4_000 }, (_, i) => piece(i)), + ]); + const params = { [c.ZSTD_c_windowLog]: 25, [c.ZSTD_c_enableLongDistanceMatching]: 1 }; + const frame = zlib.zstdCompressSync(data, { params }); + const sequences = blocks(frame).filter(block => typeof block === "number" && block > 300); + expect(new Set(sequences.map(count => Number(count) % 8)).size).toBeGreaterThan(2); + expect(zstdDecompressSync(frame).equals(data)).toBe(true); + // In one go to where it is wanted, and by way of the decoder's own buffer. + expect(zlib.zstdDecompressSync(frame, { chunkSize: data.length }).equals(data)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + }); + // The C library keeps some of them in the output buffer, unless that has room for all of them. describe("blocks of more than 64 KB of literals", () => { // Noise of 6 bits, and after every 6 to 30 bytes of it, up to `end`, 5 to 8 bytes from `nearest` to `furthest` back. From 1e789521ec0f64797a5ce5ae618c4b4141e7f09d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 02:09:55 +0000 Subject: [PATCH 198/199] zstd: a worker stops while it indexes its prefix in rows, and a dictionary is indexed faster at levels 5 to 10 When a frame with workers is given up, they are told to stop, and the next frame waits until they have. A worker looked at that flag before each block, and every 4,096 positions while it adds the prefix of its job to the binary trees. It did not look while it adds it to rows. 3 workers, level 12, jobs of 1 MB, long distance matching, which makes the prefix up to 4 MB: with 4.6 MB of noise fed, from the reset of the session to the end of a next frame of 5 bytes it took 254 to 324 ms, where the C library, which waits for all jobs to end, takes 200 to 270 ms. `row_update` now takes 65,536 positions at a time and looks at the flag before each part, as `update_tree` does: 19 to 35 ms. Level 8 with the longest overlap and 8 MB: 12 to 19 ms, now 0.7 to 2.6 ms. Not changed: - Hash chains. Rows take over from a window of 32 KB on, so there the prefix of a job is 16 KB at most. - `fast` and `dfast`, which index every third position. With long distance matching, or a window of 128 MB and the longest overlap, and the tables of the level: 0.2 to 4.6 ms. With a hash table of 2^27 entries as well, a job indexes 128 MB first, which a reviewer timed at 1.6 s. To index that in parts takes more care than a loop, as what is done near the end of a part depends on where it ends. To see what that costs, a dictionary is loaded into a context and 1 KB compressed, which had not been timed: 11 dictionaries of 8 KB to 1 MB, levels 5 to 12, 88 rows, 8 orders of the functions in the binary for each. The port was behind C: 1.08 in the mean, 69 rows by more than 3%. Two loops were the reason. - Hash chains (a dictionary of 8 KB at levels 5 to 8: 1.43 to 1.55 of C). `insert_and_find_first_index` chose the hash by the least length of a match at each position, and checked three indices. It now calls `Chains::insert_all`, which the search has for the same thing. `hash_ptr`, `hash_ptr_salted` and `hash5` to `hash8` have no caller left and are gone; `hash3` and `hash4`, without a salt now, are in their one caller each. - Rows. `row_update` checked at each position that 8 bytes can be read. It now takes the bytes of a part once and goes through windows of 8. It is also a function of its own: within `load_dictionary_content`, its loops came out 5 to 10% slower or faster with what else is in there. Time relative to before, and relative to C before and after: - all 88 rows: 0.856; 1.080, now 0.925. 69 rows more than 3% behind C, now 7. 67 take less time by more than 3 standard errors, none takes more. - by level: 5 to 8 0.80 to 0.81 (1.12 to 1.14, now 0.89 to 0.92); 9 and 10 0.85 (1.08, now 0.91 and 0.92); 11 and 12, where most of these are binary trees, 0.98. - by dictionary: 110 KB of JSON 0.79, 64 KB of HTML 0.81, of JSON 0.82, of text 0.85, of noise 0.91; 200 KB of HTML 0.82; 8 KB of HTML 0.84; 400 KB of text 0.86, of noise 0.94; 30 KB of text 0.88; 1 MB of a binary 0.93. - still more than 3% behind: 64 to 200 KB of HTML and JSON at levels 11 and 12 (1.04 to 1.12, binary trees), 400 KB of noise at level 12 (1.06), 8 KB of HTML at level 5 (1.06). These times are from an Intel Xeon of the Skylake family alone. On AMD Zen 4, where all earlier times are from, this has not been run. Other instructions: `load_dictionary_content`, and `row_update`, which is new as a function. It is 172 bytes less code. The output does not change: both loops add the same positions in the same order. What else was tried: - The loop over parts in `load_dictionary_content`, for rows and hash chains (8 orders): the wait is as short. 57 rows take 4 to 10% less time, only because the loops of `row_update` come out otherwise, and the 8 KB dictionary at levels 5 to 8 takes 13 to 14% longer. - That with `Chains::insert_all` and `row_update` as a function of its own, reading 8 bytes at a time as before (8 orders): 0.930 of before, 0.999 of C. Tests: - The unit test of a worker that stops now also runs with rows. Without the look at the flag it fails. Not tested: that the flag is looked at again between parts, which takes a race. - "after a dictionary that is indexed in several parts" in `zstd.test.ts` is new: 200,000 bytes of 3 bits each, and pieces from all over it, from 9 positions before to 2 after where each part ends, and from its end; hash chains with a least length of 3 to 7, and rows of levels 5, 8 and 10, with values from Bun 1.4.4. Like the rest of that block it passes before the change too. - Of 12 faults put into the two loops, runs against the C library with random input see all 12, the new test 9: only the first part is indexed; the first or the last position of a part is left out; one or 7 windows too few; the bytes of the next position; indices one too high; a hash of 4 bytes always, or of 6 at most. It does not see: where parts meet is indexed twice; one position more or less at the end of the hash chains. - 342 results of the debug build are those of Bun 1.4.4: dictionaries of 65,543 to 400,000 bytes at levels 5 to 12, with hash chains too, and 4 MB with 3 workers, with and without long distance matching and overlap. --- src/zstd/compress/internal.rs | 64 ++++------------------------ src/zstd/compress/lazy.rs | 78 ++++++++++++++++++++--------------- src/zstd/compress/mt.rs | 10 ++++- test/js/bun/util/zstd.test.ts | 32 ++++++++++++++ 4 files changed, 93 insertions(+), 91 deletions(-) diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs index 923a44c11099..2fd7276be78f 100644 --- a/src/zstd/compress/internal.rs +++ b/src/zstd/compress/internal.rs @@ -14,8 +14,7 @@ use crate::common::{ }; use crate::error::{Error, Result}; use crate::util::{ - BUG, SliceExt, append_at, copy, highbit32, make_room, read_le32, read_le64, reset_table, - try_zeroed, write_le24, + BUG, SliceExt, append_at, copy, highbit32, make_room, reset_table, try_zeroed, write_le24, }; pub(crate) const K_SEARCH_STRENGTH: u32 = 8; @@ -1801,42 +1800,16 @@ const PRIME_4_BYTES: u32 = 2_654_435_761; const PRIME_5_BYTES: u64 = 889_523_592_379; const PRIME_6_BYTES: u64 = 227_718_039_650_203; const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; -pub(crate) const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; - -/// `ZSTD_hash3` -#[inline(always)] -const fn hash3(u: u32, h: u32, s: u32) -> u32 { - (((u << (32 - 24)).wrapping_mul(PRIME_3_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash4(u: u32, h: u32, s: u32) -> u32 { - ((u.wrapping_mul(PRIME_4_BYTES)) ^ s) >> ((32 - h) & 31) -} -#[inline(always)] -const fn hash5(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 40)).wrapping_mul(PRIME_5_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash6(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 48)).wrapping_mul(PRIME_6_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -#[inline(always)] -const fn hash7(u: u64, h: u32, s: u64) -> usize { - ((((u << (64 - 56)).wrapping_mul(PRIME_7_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} -/// `ZSTD_hash8` -#[inline(always)] -pub(crate) const fn hash8(u: u64, h: u32, s: u64) -> usize { - (((u.wrapping_mul(PRIME_8_BYTES)) ^ s) >> ((64 - h) & 63)) as usize -} +const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; /// `ZSTD_hash3Ptr` of the position that the 4 bytes `bytes` were read at. `h` is from 1 to 32, as for all these. #[inline(always)] pub(crate) const fn hash3_of(bytes: u32, h: u32) -> usize { - hash3(bytes, h, 0) as usize + ((bytes << (32 - 24)).wrapping_mul(PRIME_3_BYTES) >> ((32 - h) & 31)) as usize } -/// [`hash_ptr`], out of the 8 bytes at the position. +/// `ZSTD_hashPtr`, out of the 8 bytes at the position: a hash of `h_bits` bits of the first `mls` of them, or 4 if that is +/// less. #[derive(Clone, Copy)] pub(crate) struct Hasher { /// Takes the bytes that do not count out, on the left. @@ -1888,33 +1861,14 @@ impl Hasher { } } - /// [`hash_ptr`] of the position that the 8 bytes `bytes` were read at. + /// The hash of the position that the 8 bytes `bytes` were read at. #[inline(always)] pub(crate) const fn hash(self, bytes: u64) -> usize { (bytes.wrapping_mul(self.factor) >> self.shift) as usize } } -/// `ZSTD_hashPtr`: a hash of `h_bits` bits of the `mls` bytes of `buf` at `pos`. Reads 4 bytes if `mls` is at most 4, -/// else 8. -#[inline(always)] -pub(crate) fn hash_ptr(buf: &[u8], pos: usize, h_bits: u32, mls: u32) -> Result { - hash_ptr_salted(buf, pos, h_bits, mls, 0) -} - -/// `ZSTD_hashPtrSalted` -#[inline(always)] -fn hash_ptr_salted(buf: &[u8], pos: usize, h_bits: u32, mls: u32, hash_salt: u64) -> Result { - Ok(match mls { - 5 => hash5(read_le64(buf, pos)?, h_bits, hash_salt), - 6 => hash6(read_le64(buf, pos)?, h_bits, hash_salt), - 7 => hash7(read_le64(buf, pos)?, h_bits, hash_salt), - 8 => hash8(read_le64(buf, pos)?, h_bits, hash_salt), - _ => hash4(read_le32(buf, pos)?, h_bits, hash_salt as u32) as usize, - }) -} - -/// [`hash_ptr`] of 4 bytes. To read no more than these crosses fewer lines of the cache than a [`Hasher`] does. +/// What a [`Hasher`] gives for 4 bytes. To read no more than these crosses fewer lines of the cache than a [`Hasher`] does. #[derive(Clone, Copy)] pub(crate) struct Hash4 { h_bits: u32, @@ -1923,11 +1877,11 @@ pub(crate) struct Hash4 { impl Hash4 { #[inline(always)] pub(crate) const fn hash(self, bytes: u32) -> usize { - hash4(bytes, self.h_bits, 0) as usize + (bytes.wrapping_mul(PRIME_4_BYTES) >> ((32 - self.h_bits) & 31)) as usize } } -/// [`hash_ptr_salted`] of 4 to 6 bytes, out of the 8 bytes at the position: the multiplication also does the shift that +/// `ZSTD_hashPtrSalted` of 4 to 6 bytes, out of the 8 bytes at the position: the multiplication also does the shift that /// leaves the other bytes out. #[derive(Clone, Copy)] pub(crate) struct Hash8 { diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs index 0e446941cbca..fa0f42397c4c 100644 --- a/src/zstd/compress/lazy.rs +++ b/src/zstd/compress/lazy.rs @@ -5,16 +5,16 @@ use core::num::NonZeroU32; use core::ops::Range; use super::internal::{ - BlockSrc, CompressionParameters, Descent, DictMatchState, DictMode, DictTree, First, Hash8, - Hasher, Hints, K_SEARCH_STRENGTH, MatchState, Near, Nodes, Older, Prefix, REPCODE1_TO_OFFBASE, - RowTags, SecondTagStats, SeqSink, SeqStore, Strategy, TreeInput, ZSTD_DUBT_UNSORTED_MARK, - ZSTD_ROW_HASH_CACHE_SIZE, bt_children, bt_look_ahead, bt_low, bt_nodes, bt_set_children, - bt_walk, count_2segments, hash_ptr, hashed_swap, index_overlap_check, offbase_is_offset, - offbase_to_offset, offset_to_offbase, table_get, table_put, + BlockSrc, CompressionParameters, Descent, DictMatchState, DictMode, DictTree, First, + HASH_READ_SIZE, Hash8, Hasher, Hints, K_SEARCH_STRENGTH, MatchState, Near, Nodes, Older, + Prefix, REPCODE1_TO_OFFBASE, RowTags, SecondTagStats, SeqSink, SeqStore, Strategy, TreeInput, + ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, bt_children, bt_look_ahead, bt_low, + bt_nodes, bt_set_children, bt_walk, count_2segments, hashed_swap, index_overlap_check, + offbase_is_offset, offbase_to_offset, offset_to_offbase, table_get, table_put, }; use crate::common::{ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM}; use crate::error::Result; -use crate::util::{BUG, SliceExt, highbit32, read_le32, read_le64}; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; const K_LAZY_SKIPPING_STEP: u32 = 8; @@ -876,26 +876,25 @@ impl<'a, const DICT_MODE: u8, const FAR: bool> Finder<'a> for TreeFinder<'a, DIC // ── hash chains ─────────────────────────────────────────────────────────── /// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash -/// chains. +/// chains. `ip` is not before that, and there are 7 bytes from it on. For a job that is 16 KB at most, as rows take over +/// from a window of 32 KB on. pub(crate) fn insert_and_find_first_index( ms: &mut MatchState, prefix: Prefix<'_>, ip: usize, ) -> Result<()> { - let hash_log = ms.c_params.hash_log; - let mls = ms.c_params.min_match; - let chain_mask = (1u32 << ms.c_params.chain_log) - 1; - let target = prefix.index_of(ip); - let hash_table = ms.hash_table.as_mut_slice(); - let chain_table = ms.chain_table.as_mut_slice(); - - for idx in ms.next_to_update..target { - let h = hash_ptr(prefix.buf, prefix.pos_of(idx), hash_log, mls)?; - let next_in_chain = core::mem::replace(hash_table.at_mut(h)?, idx); - chain_table.put((idx & chain_mask) as usize, next_in_chain)?; + if ms.hash_table.is_empty() || ms.chain_table.is_empty() { + return Err(BUG); } - - ms.next_to_update = target; + let first = prefix.pos_of(ms.next_to_update); + let positions = prefix.buf.sub(first..ip + HASH_READ_SIZE - 1)?; + let chains = Chains { + hash_table: &mut ms.hash_table, + chain_table: &mut ms.chain_table, + }; + let hasher = Hasher::new(ms.c_params.hash_log, ms.c_params.min_match); + chains.insert_all(hasher, positions, ms.next_to_update); + ms.next_to_update = prefix.index_of(ip); Ok(()) } @@ -1460,7 +1459,8 @@ fn row_prefetch( } /// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the -/// row-based match finder. +/// row-based match finder. There are 7 bytes from `ip` on. +#[inline(never)] pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { #[inline(always)] fn update( @@ -1469,8 +1469,10 @@ pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> prefix: Prefix<'_>, indices: Range, ) -> Result<()> { - for idx in indices { - let hash = hasher.of::(read_le64(prefix.buf, prefix.pos_of(idx))?); + let first = prefix.pos_of(indices.start); + let positions = (prefix.buf).span(first, indices.len() + HASH_READ_SIZE - 1)?; + for (idx, bytes) in indices.zip(positions.array_windows()) { + let hash = hasher.of::(u64::from_le_bytes(*bytes)); rows.row::(hash.hash) .ok_or(BUG)? .insert(hash, idx); @@ -1478,22 +1480,30 @@ pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Ok(()) } + /// The prefix of a job can take 0.1 s, so a worker looks every so many positions whether there is any use in going on. + const PART: u32 = 1 << 16; + row_flatten_tags(&mut ms.row_tags, &mut ms.tag_table)?; let mls = ms.c_params.min_match.min(6); let hash_bits = ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; let hasher = RowHasher(Hash8::new(mls, hash_bits, ms.hash_salt)); - let rows = Rows { - indices: &mut ms.hash_table, - tags: &mut ms.tag_table, - }; let target = prefix.index_of(ip); - let indices = ms.next_to_update..target; - ms.next_to_update = target; - match ms.c_params.search_log.clamp(4, 6) { - 4 => update::<16>(rows, hasher, prefix, indices), - 5 => update::<32>(rows, hasher, prefix, indices), - _ => update::<64>(rows, hasher, prefix, indices), + while ms.next_to_update < target { + ms.check_cancelled()?; + let indices = ms.next_to_update..ms.next_to_update.saturating_add(PART).min(target); + ms.next_to_update = indices.end; + let rows = Rows { + indices: &mut ms.hash_table, + tags: &mut ms.tag_table, + }; + match ms.c_params.search_log.clamp(4, 6) { + 4 => update::<16>(rows, hasher, prefix, indices)?, + 5 => update::<32>(rows, hasher, prefix, indices)?, + _ => update::<64>(rows, hasher, prefix, indices)?, + } } + ms.next_to_update = target; + Ok(()) } /// Bit `n` is set if the entry `n` has the tag `tag`. This becomes a comparison of vectors. diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs index 0c98cc88cf10..6c6548e9fa53 100644 --- a/src/zstd/compress/mt.rs +++ b/src/zstd/compress/mt.rs @@ -909,7 +909,13 @@ mod tests { #[test] fn a_worker_stops_at_a_prefix_or_a_block_of_a_frame_that_is_given_up() { - let params = small_params(Strategy::BtOpt, 6); + a_worker_stops(&small_params(Strategy::BtOpt, 6)); + let mut with_rows = small_params(Strategy::Lazy, 6); + with_rows.use_row_match_finder = ParamSwitch::Enable; + a_worker_stops(&with_rows); + } + + fn a_worker_stops(params: &CCtxParams) { let data = [0u8; 64]; let input = Input { bufs: Buffers { @@ -927,7 +933,7 @@ mod tests { let cancelled = Arc::new(AtomicBool::new(false)); let mut cctx = FrameCompressor::for_worker(Arc::clone(&cancelled)); let begin = |cctx: &mut FrameCompressor| { - cctx.begin(&input, Some(span(0, 32)), ¶ms, 32, false) + cctx.begin(&input, Some(span(0, 32)), params, 32, false) .map(|_| ()) }; let mut dst = [0u8; 128]; diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index cd7308908fb9..1f30e444e49e 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -1669,6 +1669,38 @@ describe("the output is that of zstd 1.5.7", () => { ]); }); + it("after a dictionary that is indexed in several parts", () => { + // Rows take 65,536 positions at a time. The pieces are from all over it, then from where a part ends and from its end, + // whose last 8 positions are not indexed. 3 bits to a byte make for many candidates. + const next = xorshift(9); + const dictionary = Buffer.alloc(200_000); + for (let at = 0; at < dictionary.length; at++) dictionary[at] = next() >>> 29; + const fromAllOver = (i: number) => dictionary.subarray((i * 104_729) % 199_000).subarray(0, 8 + (i % 30)); + const atTheEndOfAPart = (i: number) => dictionary.subarray(65_536 * (1 + (i % 3)) + (i >> 2) - 9).subarray(0, 40); + const data = Buffer.concat([ + ...Array.from({ length: 400 }, (_, i) => fromAllOver(i)), + ...Array.from({ length: 48 }, (_, i) => atTheEndOfAPart(i)), + ...Array.from({ length: 6 }, (_, i) => dictionary.subarray(-7 - i)), + ]); + // Hash chains, which are indexed by as many bytes as the least length of a match, then rows. + const chains = { [c.ZSTD_c_windowLog]: 14, [c.ZSTD_c_chainLog]: 17, [c.ZSTD_c_hashLog]: 17 }; + const all = [ + ...[5, 8].map(n => ({ ...level(n), ...chains })), + ...[3, 6, 7].map(minMatch => ({ ...level(5), ...chains, [c.ZSTD_c_minMatch]: minMatch })), + ...[5, 8, 10].map(level), + ]; + expect(all.map(params => digest(zlib.zstdCompressSync(data, { params, dictionary })))).toEqual([ + "1781 f75b9689d167d764", + "3049 d39dde15a799187e", + "2458 6a9eaca5014ab4c7", + "1454 2f96fdade038991b", + "3566 626fb7b8cb1df2dd", + "1480 cf7dbfcd5f5f068f", + "3077 7c69c63d013fb421", + "1385 c99be48f15f658e8", + ]); + }); + it("where there are literals before a match in the trees of a dictionary that is searched in place", () => { // Few trees and few steps down them, so that it counts how many are left for the dictionary. const crowded = (n: number) => ({ ...level(n), [c.ZSTD_c_hashLog]: 10, [c.ZSTD_c_searchLog]: 2 }); From 98c81e35c63d6c50e0ef705902b7a6ec488d7319 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 02:42:27 +0000 Subject: [PATCH 199/199] test: compile-trim-capabilities scans only the Mach-O sections that hold instructions The change of #47613, as it is there at eb43446e8, so that it is nothing once main has it. On macOS arm64 the test fails for some layouts of the executable, and for that of 1e789521e (Buildkite #136744, all four attempts): `placesThatCall` finds a place that calls `execve` in an executable without what starts programs. Its bytes are 0f0f0f0f0f0f0f0f0f10101010..., a table in `__TEXT,__const`: `readMachO` gave whole segments to the scan, which reads every word of an executable segment as an instruction. Now it lists sections, and only one that says that it holds instructions is read as such. The two tests "a Mach-O file" pass here and fail with the helper as it was. The rest of the file is skipped in a build with AddressSanitizer, so that it does what is expected of it on macOS is left to CI. --- .../bundler/compile-trim-capabilities.test.ts | 110 +++++++++++++++++- test/bundler/what-an-executable-can-call.ts | 19 ++- 2 files changed, 122 insertions(+), 7 deletions(-) diff --git a/test/bundler/compile-trim-capabilities.test.ts b/test/bundler/compile-trim-capabilities.test.ts index a1cc983c181f..31aef220df4c 100644 --- a/test/bundler/compile-trim-capabilities.test.ts +++ b/test/bundler/compile-trim-capabilities.test.ts @@ -74,6 +74,16 @@ async function build(without: string[]) { } const namesOf = (places: { name: string }[]) => [...new Set(places.map(place => place.name))].sort(); + +// What every executable calls, whatever --trim removed. A scan that finds neither has not read the code of the file. +const alwaysCalled = ["mmap", "write"]; + +function placesLeftThatCall(trimmed: Uint8Array, names: readonly string[]) { + const places = placesThatCall(trimmed, [...names, ...alwaysCalled]); + expect(namesOf(places.filter(place => alwaysCalled.includes(place.name)))).toEqual(alwaysCalled); + return places.filter(place => !alwaysCalled.includes(place.name)); +} + const numbersOf = (places: { number: number }[]) => [...new Set(places.map(place => place.number))].sort((a, b) => a - b); @@ -93,7 +103,7 @@ if (isASAN) if (isLinux) expect(numbersOf(placesThatMakeSystemCalls(whole.file, startsAProgram.systemCalls))).toContain(435); const cut = await build(startsAProgram.features); - expect(placesThatCall(cut.file, startsAProgram.functions)).toEqual([]); + expect(placesLeftThatCall(cut.file, startsAProgram.functions)).toEqual([]); expect(placesThatMakeSystemCalls(cut.file, startsAProgram.systemCalls)).toEqual([]); expect(await run([cut.executable], folder)).toEqual(saysHello); }); @@ -107,7 +117,7 @@ if (isASAN) }); const file = readFileSync(join(folder, "named")); expect( - namesOf(placesThatCall(file, startsAProgram.functions)).filter(name => !sendsACrashReport.includes(name)), + namesOf(placesLeftThatCall(file, startsAProgram.functions)).filter(name => !sendsACrashReport.includes(name)), ).toEqual([]); expect(placesThatMakeSystemCalls(file, startsAProgram.systemCalls)).toEqual([]); expect(await run([join(folder, "named")], folder)).toEqual(saysHello); @@ -125,7 +135,7 @@ if (isASAN) const cut = await build(loadsNativeCode.features); expect( - placesThatCall(cut.file, loadsNativeCode.functions).map(place => ({ + placesLeftThatCall(cut.file, loadsNativeCode.functions).map(place => ({ ...place, keptBy: featuresThatKeep(place.address, loadsNativeCode.features), })), @@ -134,3 +144,97 @@ if (isASAN) }); }, ); + +// A Mach-O file that imports `execve`. __TEXT is its one executable segment: __text is at 0x100000400 (up to 16 bytes), +// __stubs at 0x100000410 and __const, with the flags given, at 0x100000480 (up to 128 bytes). The sections of __DATA are +// the three kinds that have no bytes in the file. +function machO(cpu: number, text: string, stub: string, constants: string, flagsOfConstants: number): Uint8Array { + type Section = [name: string, address: number, size: number, offset: number, flags: number, indirect?: number]; + const file = new Uint8Array(0x530); + const view = new DataView(file.buffer); + let at = 32; + const u32 = (...values: number[]) => values.forEach(value => (view.setUint32(at, value, true), (at += 4))); + const u64 = (...values: number[]) => values.forEach(value => (view.setBigUint64(at, BigInt(value), true), (at += 8))); + const name = (value: string) => (file.set(Buffer.from(value), at), (at += 16)); + const segment = (segmentName: string, [address, size, offset, bytes, protection]: number[], sections: Section[]) => { + u32(0x19, 72 + 80 * sections.length); + name(segmentName); + u64(0x100000000 + address, size, offset, bytes); + u32(protection, protection, sections.length, 0); + for (const [sectionName, sectionAddress, sectionSize, sectionOffset, flags, indirect = 0] of sections) { + name(sectionName); + name(segmentName); + u64(0x100000000 + sectionAddress, sectionSize); + u32(sectionOffset, 0, 0, 0, flags, indirect, sectionName === "__stubs" ? sectionSize : 0, 0); + } + }; + segment( + "__TEXT", + [0, 0x4000, 0, 0x500, 5], + [ + ["__text", 0x400, text.length / 2, 0x400, 0x80000400], + ["__stubs", 0x410, stub.length / 2, 0x410, 0x80000408], + ["__const", 0x480, constants.length / 2, 0x480, flagsOfConstants], + ], + ); + segment("__DATA_CONST", [0x4000, 0x4000, 0x500, 8, 3], [["__got", 0x4000, 8, 0x500, 6, 1]]); + segment( + "__DATA", + [0x8000, 0xc000, 0, 0, 3], + [ + ["__bss", 0x8000, 0x4000, 0, 1], + ["__huge", 0xc000, 0x4000, 0, 12], + ["__thread_bss", 0x10000, 0x4000, 0, 18], + ], + ); + segment("__LINKEDIT", [0x14000, 0x4000, 0x508, 0x28, 1], []); + // LC_SYMTAB: one symbol at 0x508, its name at 0x520. LC_DYSYMTAB: it is undefined, and at 0x518 the stub and the slot + // of __got name it. + u32(2, 24, 0x508, 1, 0x520, 9); + u32(11, 80, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0x518, 2, 0, 0, 0, 0); + const sizeOfCommands = at - 32; + at = 0; + u32(0xfeedfacf, cpu, cpu === 0x01000007 ? 3 : 0, 2, 6, sizeOfCommands, 0, 0); + file.set(Buffer.from(text, "hex"), 0x400); + file.set(Buffer.from(stub, "hex"), 0x410); + file.set(Buffer.from(constants, "hex"), 0x480); + at = 0x508; + u32(1, 1, 0, 0); + file.set(Buffer.from("\0_execve\0"), 0x520); + return file; +} + +describe("a Mach-O file", () => { + test.each([ + { + cpu: "arm64", + cpuType: 0x0100000c, + text: "04000094" + "c0035fd6", // bl 0x100000410; ret + stub: "30000090" + "100240f9" + "00021fd6", // adrp x16, 0x100004000; ldr x16, [x16]; br x16 + constants: "e4ffff97" + "f1db0110", // as instructions: bl 0x100000410; adr x17, 0x100004000 + call: 0x100000400, + callsInConstants: [0x100000480, 0x100000484], + }, + { + cpu: "x64", + cpuType: 0x01000007, + text: "e80b000000" + "c3" + "9090", // call 0x100000410; ret; nop; nop + stub: "ff25ea3b0000", // jmp *0x100004000(%rip) + constants: "e98bffffff" + "000000", // as an instruction: jmp 0x100000410 + call: 0x100000401, + callsInConstants: [0x100000481], + }, + ])( + "for $cpu is read as instructions only where a section says that it holds them", + ({ cpuType, text, stub, constants, call, callsInConstants }) => { + const callsWith = (flagsOfConstants: number) => + placesThatCall(machO(cpuType, text, stub, constants, flagsOfConstants), ["execve"]).map(place => place.address); + // S_ATTR_PURE_INSTRUCTIONS is 0x80000000, S_ATTR_SOME_INSTRUCTIONS 0x400. + expect({ data: callsWith(0), pure: callsWith(0x80000000), some: callsWith(0x400) }).toEqual({ + data: [call], + pure: [call, ...callsInConstants], + some: [call, ...callsInConstants], + }); + }, + ); +}); diff --git a/test/bundler/what-an-executable-can-call.ts b/test/bundler/what-an-executable-can-call.ts index e64836c2e11a..c2bc31362f6b 100644 --- a/test/bundler/what-an-executable-can-call.ts +++ b/test/bundler/what-an-executable-can-call.ts @@ -5,6 +5,7 @@ export interface Place { code: string; } +// What is scanned as a whole: a segment of an ELF file, a section of a Mach-O file. interface Segment { offset: number; end: number; @@ -26,6 +27,9 @@ const jumpSlot = { x64: 7, arm64: 1026 }; const globalData = { x64: 6, arm64: 1025 }; const absolute = { x64: 1, arm64: 257 }; const operandOpcodes = [0x8d, 0x8b, 0xff, 0x3b, 0x39, 0x89]; +// Of a Mach-O section: the attributes S_ATTR_PURE_INSTRUCTIONS and S_ATTR_SOME_INSTRUCTIONS, and the types S_ZEROFILL, S_GB_ZEROFILL and S_THREAD_LOCAL_ZEROFILL, which have no bytes in the file. +const instructions = 0x80000400; +const zeroFill = [1, 12, 18]; function signed(value: number, bits: number): number { return value >= 2 ** (bits - 1) ? value - 2 ** bits : value; @@ -49,11 +53,16 @@ function readMachO(file: Uint8Array): Image { if (command === 2) [symbols, strings] = [u32(at + 8), u32(at + 16)]; if (command === 11) indirect = u32(at + 56); if (command !== 0x19) continue; - if (u64(at + 48) > 0) image.segments.push({ offset: u64(at + 40), end: u64(at + 40) + u64(at + 48), address: u64(at + 24), isExecutable: (u32(at + 60) & 4) !== 0 }); for (let section = at + 72; section < at + 72 + u32(at + 64) * 80; section += 80) { - const type = u32(section + 64) & 0xff; - if (type === 6 || type === 7) pointers.push({ address: u64(section + 32), size: u64(section + 40), first: u32(section + 68), each: 8, isStub: false }); - if (type === 8) pointers.push({ address: u64(section + 32), size: u64(section + 40), first: u32(section + 68), each: u32(section + 72), isStub: true }); + const address = u64(section + 32); + const size = u64(section + 40); + const offset = u32(section + 48); + const flags = u32(section + 64); + const type = flags & 0xff; + // __TEXT is executable as a whole, and __const and __cstring are in it. Only a section says whether its bytes are instructions. + if (!zeroFill.includes(type)) image.segments.push({ offset, end: offset + size, address, isExecutable: (flags & instructions) !== 0 }); + if (type === 6 || type === 7) pointers.push({ address, size, first: u32(section + 68), each: 8, isStub: false }); + if (type === 8) pointers.push({ address, size, first: u32(section + 68), each: u32(section + 72), isStub: true }); } } for (const { address, size, first, each, isStub } of pointers) { @@ -202,6 +211,8 @@ function placesIn(elf: Image, wanted: (name: string) => boolean): Place[] { } continue; } + // A call can be the first instruction of a section. The loop sees a displacement only with two bytes before it. + if (offset + 5 <= end && (file[offset] === 0xe8 || file[offset] === 0xe9)) found(nameAt(base + offset + 5 + view.getInt32(offset + 1, true)), base + offset + 1); for (let at = offset + 2; at + 4 <= end; at++) { const word = view.getInt32(at, true); const before = file[at - 1]!;